s444498/wko_anime-face-similarity
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

scrap n images depends on argv param

Browse Source
This commit is contained in:
Mateusz 2023-01-15 14:17:33 +01:00
parent a27a7e612a
commit 1f4a2d9dfc
1 changed files with 39 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
scrape_data.py
View File

@ -3,8 +3,12 @@ from bs4 import BeautifulSoup
from lxml import etree from lxml import etree
import os import os
import csv import csv
import sys
ROOT_URL = "https://myanimelist.net/character.php" if len(sys.argv) > 1:
IMG_N = int(sys.argv[1])
else:
IMG_N = 50
def get_page_xpath_result(url, xpath_str): def get_page_xpath_result(url, xpath_str):
HEADERS = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'} HEADERS = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
@ -19,40 +23,46 @@ def createDirectory(path):
os.makedirs(path) os.makedirs(path)
print(f"The {path} is created!") print(f"The {path} is created!")
character_links = get_page_xpath_result(ROOT_URL, '//div[@class="information di-ib mt24"]/a/@href') n = IMG_N // 50
character_names = get_page_xpath_result(ROOT_URL, '//div[@class="information di-ib mt24"]/a')
character_names = [link.text for link in character_names]
character_img_urls = [get_page_xpath_result(link, '//td/div/a/img/@data-src')[0] for i in range(n):
for link in character_links] ROOT_URL = f'https://myanimelist.net/character.php?limit={(i)*50}'
print(ROOT_URL)
createDirectory('./data') character_links = get_page_xpath_result(ROOT_URL, '//div[@class="information di-ib mt24"]/a/@href')
createDirectory('./data/images') character_names = get_page_xpath_result(ROOT_URL, '//div[@class="information di-ib mt24"]/a')
character_names = [link.text for link in character_names]
csv_header = ['name', 'file', 'link'] character_img_urls = [get_page_xpath_result(link, '//td/div/a/img/@data-src')[0]
data_rows = [] for link in character_links]
for name, url, link in zip(character_names, character_img_urls, character_links): createDirectory('./data')
img_data = requests.get(url).content createDirectory('./data/images')
img_path = f'./data/images/{name}.jpg'
img_path_exist = os.path.exists(img_path)
if not img_path_exist: csv_header = ['name', 'file', 'link']
with open(img_path, 'wb') as handler: data_rows = []
handler.write(img_data)
handler.close()
print(f'{name}.jpg downloaded')
data_row = [name, f'{name}.jpg', link] for name, url, link in zip(character_names, character_img_urls, character_links):
data_rows.append(data_row) img_data = requests.get(url).content
img_path = f'./data/images/{name}.jpg'
img_path_exist = os.path.exists(img_path)
csv_path = './data/anime-faces.csv' if not img_path_exist:
if len(data_rows) > 0: with open(img_path, 'wb') as handler:
csv_file_exist = os.path.exists(csv_path) handler.write(img_data)
with open(csv_path, 'a', encoding='UTF8') as f: handler.close()
writer = csv.writer(f) print(f'{name}.jpg downloaded')
if not csv_file_exist:
writer.writerow(csv_header) data_row = [name, f'{name}.jpg', link]
writer.writerows(data_rows) data_rows.append(data_row)
csv_path = './data/anime-faces.csv'
if len(data_rows) > 0:
csv_file_exist = os.path.exists(csv_path)
with open(csv_path, 'a', encoding='UTF8') as f:
writer = csv.writer(f)
if not csv_file_exist:
writer.writerow(csv_header)
writer.writerows(data_rows)
print("finish!") print("finish!")