s396341/mass-scraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fix proxy conditional statement

Browse Source
This commit is contained in:
siulkilulki 2017-06-18 21:44:12 +02:00
parent f54e01581c
commit 4ae6cd24c0
2 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scraper/duckduckgo.py
View File

@ -45,7 +45,9 @@ class DuckDuckGo(object):
proxy_dict = self._proxy_to_dict(proxy) proxy_dict = self._proxy_to_dict(proxy)
self.falitures += 1 self.falitures += 1
if self.falitures > 0.95 * len(self.proxy_obj.proxies): total_nr_of_proxies = len(
self.proxy_obj.proxies) + self.falitures
if self.falitures > 0.95 * total_nr_ofproxies:
if self.download: if self.download:
self.download_proxies() self.download_proxies()
self.download = False self.download = False

1
urls_checked_a.txt
View File

@ -1 +0,0 @@
http://www.adamowice.katowice.opoka.org.pl/