2018-03-01 18:16:11 +01:00
|
|
|
SHELL := /bin/bash
|
|
|
|
|
PREPARE_ENVIRONMENT := $(shell ./prepare-environment.sh > /tmp/makeenv)
|
|
|
|
|
include /tmp/makeenv
|
|
|
|
|
JOBS := 6
|
|
|
|
|
|
|
|
|
|
.PHONY: all clean data
|
|
|
|
|
|
|
|
|
|
all: data
|
|
|
|
|
|
|
|
|
|
data: parishwebsites/spider-commands.txt
|
|
|
|
|
rm -rf data
|
|
|
|
|
parallel --jobs $(JOBS) < $<
|
|
|
|
|
|
|
|
|
|
parishwebsites/spider-commands.txt: parishes-with-urls.tsv
|
|
|
|
|
cut -f3 $< | tail -n +2 | grep http | parishwebsites/generate_spider_commands.sh | sort -u > $@
|
|
|
|
|
|
|
|
|
|
parishes-with-urls.tsv: apikey.txt parishes-deon.tsv scraper/get_parishes_urls.py
|
|
|
|
|
scraper/get_parishes_urls.py -a $< -p $(word 2,$^) > $@ 2> get-parishes-urls.log
|
|
|
|
|
|
|
|
|
|
parishes-deon.tsv: scraper/crawl_deon.py
|
|
|
|
|
scraper/crawl_deon.py > $@ 2> crawl-deon.log
|
|
|
|
|
|
|
|
|
|
clean:
|
|
|
|
|
rm -rf parishes-deon.tsv parishes-with-urls.tsv spider-commands.txt
|
