37 lines
710 B
Bash
Executable File
37 lines
710 B
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -xe -o pipefail
|
|
|
|
make normalize csv2tsv/csv2tsv
|
|
|
|
keep=(stops.txt trips.txt stop_times.txt)
|
|
|
|
mkdir -p data && cd data
|
|
parallel -j 8 -- wget --no-verbose --no-clobber <../ztm-data.txt
|
|
|
|
for file in $(find . -name 'index*.zip'); do
|
|
dir="${file##*=}"
|
|
dir="${dir%.zip}"
|
|
if [ ! -d "$dir" ]; then
|
|
mkdir "$dir"
|
|
unzip "$file" -d "$dir" "${keep[@]}"
|
|
fi
|
|
done
|
|
|
|
cd ..
|
|
|
|
for k in "${keep[@]}"; do
|
|
csv="${k%.txt}.csv"
|
|
tsv="${k%.txt}.tsv"
|
|
if [ ! -f "$tsv" ]; then
|
|
cat $(find data -name "$k") > "$csv"
|
|
csv2tsv/csv2tsv <"$csv" >"$tsv"
|
|
fi
|
|
done
|
|
|
|
if [ ! -f "stop_times.normalized.tsv" ]; then
|
|
./normalize <stop_times.tsv >stop_times.normalized.tsv
|
|
./split_train_valid_test.py
|
|
fi
|
|
./stats.py
|