Support docker; restore previous implementaiton

.gitignore

@@ -3,7 +3,7 @@ columns.pruned.tsv
 columns.original.tsv
 *.csv
 *.tsv
-csv2tsv/csv2tsv
+src/csv2tsv/csv2tsv
 .cache
 compile_commands.json
 tsv2json/tsv2json

Dockerfile

@@ -0,0 +1,5 @@
+FROM ubuntu:22.04
+
+RUN apt update && apt install -y vim make python3 python3-pip python-is-python3 gcc g++ golang wget unzip
+RUN pip install pandas matplotlib scikit-learn
+CMD "bash"

aitech-ium

@@ -1 +1 @@
-Subproject commit 855ca79293d0bfaec6d144651030d304db9242af
+Subproject commit b88e776291ee8cae62537e96845f4e1a37457840

prepare-ztm-data.sh

@@ -1,47 +0,0 @@
-#!/usr/bin/env bash
-
-set -xe -o pipefail
-
-# Disable to allow to work in bare jenkins
-# make normalize csv2tsv/csv2tsv
-
-keep=stop_times.txt
-
-mkdir -p data && cd data
-xargs -- wget --no-verbose --no-clobber <../ztm-data.txt
-
-for file in $(find . -name 'index*.zip'); do
-	dir="${file##*=}"
-	dir="${dir%.zip}"
-	if [ ! -d "$dir" ]; then
-		mkdir "$dir"
-		unzip "$file" -d "$dir" "${keep[@]}"
-	fi
-done
-
-cd ..
-
-k=$keep
-csv="${k%.txt}.csv"
-cat $(find data -name "$k") | shuf > "$csv"
-train_size=$(( $(wc -l "$csv" | cut -f1 -d' ') * 8 / 10 ))
-echo $train_size
-
-head -n $train_size  $csv >train.csv
-tail -n +$train_size $csv >test.csv
-
-# Disable to allow to work in bare jenkins
-# for k in "${keep[@]}"; do
-# 	csv="${k%.txt}.csv"
-# 	tsv="${k%.txt}.tsv"
-# 	if [ ! -f "$tsv" ]; then
-# 		cat $(find data -name "$k") > "$csv"
-# 		csv2tsv/csv2tsv <"$csv" >"$tsv"
-# 	fi
-# done
-#
-# if [ ! -f "stop_times.normalized.tsv" ]; then
-# 	./normalize <stop_times.tsv >stop_times.normalized.tsv
-# 	./split_train_valid_test.py
-# fi
-# ./stats.py

run.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+set -xe
+
+docker build -t ium .
+docker run -it ium

src/prepare-ztm-data.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+set -xe -o pipefail
+
+make normalize csv2tsv/csv2tsv
+
+keep=stop_times.txt
+
+mkdir -p data && cd data
+xargs -- wget --no-verbose --no-clobber <../ztm-data.txt
+
+for file in $(find . -name 'index*.zip'); do
+	dir="${file##*=}"
+	dir="${dir%.zip}"
+	if [ ! -d "$dir" ]; then
+		mkdir "$dir"
+		unzip "$file" -d "$dir" "${keep[@]}"
+	fi
+done
+
+cd ..
+
+for k in "${keep[@]}"; do
+	csv="${k%.txt}.csv"
+	tsv="${k%.txt}.tsv"
+	if [ ! -f "$tsv" ]; then
+		cat $(find data -name "$k") > "$csv"
+		csv2tsv/csv2tsv <"$csv" >"$tsv"
+	fi
+done
+
+if [ ! -f "stop_times.normalized.tsv" ]; then
+	./normalize <stop_times.tsv >stop_times.normalized.tsv
+fi
+
+if [ ! \( -f "stop_times.train.tsv" -a -f "stop_times.test.tsv" -a -f "stop_times.valid.tsv" \) ]; then
+	./split_train_valid_test.py
+fi
+
+
+./stats.py