diff --git a/.gitignore b/.gitignore index bd48140..24fee33 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -images +data # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/scrape_data.py b/scrape_data.py index af05b51..bb9678c 100644 --- a/scrape_data.py +++ b/scrape_data.py @@ -2,6 +2,7 @@ import requests from bs4 import BeautifulSoup from lxml import etree import os +import csv ROOT_URL = "https://myanimelist.net/character.php" @@ -12,6 +13,12 @@ def get_page_xpath_result(url, xpath_str): dom = etree.HTML(str(soup)) return dom.xpath(xpath_str) +def createDirectory(path): + isExist = os.path.exists(path) + if not isExist: + os.makedirs(path) + print(f"The {path} is created!") + character_links = get_page_xpath_result(ROOT_URL, '//div[@class="information di-ib mt24"]/a/@href') character_names = get_page_xpath_result(ROOT_URL, '//div[@class="information di-ib mt24"]/a') character_names = [link.text for link in character_names] @@ -19,17 +26,33 @@ character_names = [link.text for link in character_names] character_img_urls = [get_page_xpath_result(link, '//td/div/a/img/@data-src')[0] for link in character_links] -path = './images' -isExist = os.path.exists(path) -if not isExist: - os.makedirs(path) - print("The images directory is created!") +createDirectory('./data') +createDirectory('./data/images') -for name, url in zip(character_names, character_img_urls): +csv_header = ['name', 'file', 'link'] +data_rows = [] + +for name, url, link in zip(character_names, character_img_urls, character_links): img_data = requests.get(url).content - with open(f'images/{name}.jpg', 'wb') as handler: - handler.write(img_data) - handler.close() - print(f'{name}.jpg downloaded') + img_path = f'./data/images/{name}.jpg' + img_path_exist = os.path.exists(img_path) + if not img_path_exist: + with open(img_path, 'wb') as handler: + handler.write(img_data) + handler.close() + print(f'{name}.jpg downloaded') + data_row = [name, f'{name}.jpg', link] + data_rows.append(data_row) + +csv_path = './data/anime-faces.csv' +if len(data_rows) > 0: + csv_file_exist = os.path.exists(csv_path) + with open(csv_path, 'a', encoding='UTF8') as f: + writer = csv.writer(f) + if not csv_file_exist: + writer.writerow(csv_header) + writer.writerows(data_rows) + +print("finish!") \ No newline at end of file