s444415/wikisource-crawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

crawler fix

Browse Source
This commit is contained in:
Michał Kozłowski 2023-01-07 14:59:07 +01:00
parent 9ee285cf95
commit 1f08432fbf
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crawler.py
View File

@ -11,7 +11,7 @@ MAIN_URL = "https://pl.wikisource.org/"
def get_page_data(page_element):
time.sleep(0.5)
doc = requests.get(MAIN_URL + page_element['href'])
doc_soup = BeautifulSoup(doc.text, 'lxml', from_encoding="utf-8")
doc_soup = BeautifulSoup(doc.text, 'lxml')
text = doc_soup.find("div", {"class": "pagetext"}).next_element
image_url = doc_soup.find("div", {"class": "prp-page-image"}).next_element['src']
return {"title": page_element['title'], "href": MAIN_URL + page_element['href'], "image_url": image_url, "text": text.text}