s396341/mass-scraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

done proxy.py

Browse Source
This commit is contained in:
siulkilulki 2017-06-11 00:00:22 +02:00
parent c205e1b627
commit de56ecb253
1 changed files with 22 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
proxy.py
View File

@ -1,8 +1,12 @@
from selenium import webdriver from selenium import webdriver
import re import re
import random
# TODO: export path with geckodriver or chromedriver automatically and put driver in project files
# TODO: automatically download geckodriver or chromedriver
class Proxy(): class Proxy():
def __init__(self, proxies = None): def __init__(self, proxies=None):
"docstring" "docstring"
self.proxies = [] if proxies is None else proxies self.proxies = [] if proxies is None else proxies
@ -10,21 +14,32 @@ class Proxy():
driver = webdriver.Chrome() driver = webdriver.Chrome()
driver.maximize_window() driver.maximize_window()
driver.get('http://www.gatherproxy.com/proxylist/anonymity/?t=elite') driver.get('http://www.gatherproxy.com/proxylist/anonymity/?t=elite')
full_list_button = driver.find_element_by_xpath('//input[@type="submit" and @value="Show Full List"]') full_list_button = driver.find_element_by_xpath(
'//input[@type="submit" and @value="Show Full List"]')
full_list_button.click() full_list_button.click()
print(driver.page_source) print(driver.page_source)
for match in re.finditer('<a href="#(.*?)" class="inactive" onclick="gp.pageClick', driver.page_source): for match in re.finditer(
'<a href="#(.*?)" class="inactive" onclick="gp.pageClick',
driver.page_source):
pass pass
pages_nr = int(match.group(1)) pages_nr = int(match.group(1))
for i in range(2, pages_nr+1): for i in range(1, pages_nr + 1):
self._get_proxies(driver.page_source)
driver.execute_script('gp.pageClick(' + str(i) + ')') driver.execute_script('gp.pageClick(' + str(i) + ')')
print(i) print(i)
def random(self):
return random.choice(self.proxies)
def _get_proxies(self, html): def _get_proxies(self, html):
for match in re.findall("<td><script>document.write\('(.*?)'[\w\W]*?<td><script>document.write\(gp.dep\('(.*?)'", html): for match in re.findall(
proxy = (match[0], str(int(match[1],16))) "<td><script>document.write\('(.*?)'[\w\W]*?<td><script>document.write\(gp.dep\('(.*?)'",
html):
proxy = (match[0], str(int(match[1], 16)))
self.proxies.append(proxy) self.proxies.append(proxy)
if __name__ == '__main__': if __name__ == '__main__':
p = Proxy() p = Proxy()
p.download() p.download()
print(p.random())