yellow data, crawler fixes, .gitignore added out files

2023-01-04 20:32:26 +00:00 · 2023-01-04 20:32:26 +00:00 · a16d660cf7
commit a16d660cf7
parent f7ff1acec3
3 changed files with 1253134 additions and 1 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1 +1,2 @@
 env
+*.out
--- a/crawler.py
+++ b/crawler.py
@ -41,7 +41,7 @@ def main(args):

                    if r.status_code != 200:
                        print(r.__dict__)
-                        time.sleep(10)
+                        time.sleep(30)
                        r = requests.get(MAIN_URL + next_page)
                        if r.status_code != 200:
                            break
--- a/yellow.tsv
+++ b/yellow.tsv