s444415/wikisource-crawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix save

Browse Source
This commit is contained in:
Michał Kozłowski 2023-01-07 11:42:54 +01:00
parent 68231dce0a
commit 526eb7d5e4
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
crawler.py
View File

@ -38,6 +38,7 @@ def main(args):
with tqdm(total=max_len) as pbar: with tqdm(total=max_len) as pbar:
if args.start_page_number: if args.start_page_number:
pbar.update(data_number) pbar.update(data_number)
pbar.refresh()
while data_number < max_len: while data_number < max_len:
pbar.set_description(f"Page number: {page_number}") pbar.set_description(f"Page number: {page_number}")
@ -64,6 +65,8 @@ def main(args):
except Exception as e: except Exception as e:
print(e) print(e)
save_data(f"./{args.output_file_name}-{args.type}.tsv", result_list) save_data(f"./{args.output_file_name}-{args.type}.tsv", result_list)
except KeyboardInterrupt:
save_data(f"./{args.output_file_name}-{args.type}.tsv", result_list)
save_data(f"./{args.output_file_name}-{args.type}.tsv", result_list) save_data(f"./{args.output_file_name}-{args.type}.tsv", result_list)