mass-scraper/parishwebsites/generate_spider_commands.sh

6 lines
203 B
Bash
Raw Normal View History

#!/usr/bin/env bash
while IFS='$\n' read -r url; do
echo "scrapy crawl parishes -a url=\"$url\" -t jsonlines -o data/`echo "$url" | sed -Ee 's@/|:@@g' | sed 's/^http//g' | sed 's/^www\.//g'`"
done