diff --git a/scrape_data.py b/scrape_data.py new file mode 100644 index 0000000..6214254 --- /dev/null +++ b/scrape_data.py @@ -0,0 +1,27 @@ +import requests +from bs4 import BeautifulSoup +from lxml import etree +import re + +ROOT_URL = "https://myanimelist.net/character.php" + +def get_page_xpath_result(url, xpath_str): + HEADERS = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'} + page = requests.get(url, headers=HEADERS) + soup = BeautifulSoup(page.content, "html.parser") + dom = etree.HTML(str(soup)) + return dom.xpath(xpath_str) + +# 1. face image +# 2. character name +# 3. link + +character_links = get_page_xpath_result(ROOT_URL, '//div[@class="information di-ib mt24"]/a/@href') +character_names = get_page_xpath_result(ROOT_URL, '//div[@class="information di-ib mt24"]/a') +character_names = [link.text for link in character_names] + +print("character_links") +print(character_links) + +print("character_names") +print(character_names) \ No newline at end of file