s444415/wikisource-crawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

header update

Browse Source
This commit is contained in:
Michał Kozłowski 2023-01-07 14:32:45 +01:00
parent 5c3611f972
commit bf6014ba98
1 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
image_download.py
View File

@ -8,8 +8,7 @@ import pickle
import time
from pprint import pprint
session = requests.Session()
session.headers.update({'User-Agent': 'ImageDownloadOcrBot/1.0 (micha9@op.pl)'})
headers = {'User-Agent': 'ImageDownloadOcrBot/1.0 (micha9@op.pl) requests/2.28.1'}
def save_state(index, offset):
with open("./state.pickle", "wb") as state_file:
@ -31,7 +30,7 @@ def main(args):
for n, row in enumerate(tqdm(df.iterrows(), total=len(df))):
try:
time.sleep(0.2)
r = requests.get(f"https:{row[1]['image_url']}", stream=True)
r = requests.get(f"https:{row[1]['image_url']}", stream=True, headers=headers)
if r.status_code != 200:
pprint(r.__dict__)
save_state(n, offset)