.gitignore

@@ -3,7 +3,7 @@ columns.pruned.tsv
 columns.original.tsv
 *.csv
 *.tsv
-src/csv2tsv/csv2tsv
+csv2tsv/csv2tsv
 .cache
 compile_commands.json
 tsv2json/tsv2json

Dockerfile

@@ -1,5 +0,0 @@
-FROM ubuntu:22.04
-
-RUN apt update && apt install -y vim make python3 python3-pip python-is-python3 gcc g++ golang wget unzip
-RUN pip install pandas matplotlib scikit-learn
-CMD "bash"

aitech-ium

@@ -1 +1 @@
-Subproject commit b88e776291ee8cae62537e96845f4e1a37457840
+Subproject commit 855ca79293d0bfaec6d144651030d304db9242af

prepare-ztm-data.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+set -xe -o pipefail
+
+# Disable to allow to work in bare jenkins
+# make normalize csv2tsv/csv2tsv
+
+keep=stop_times.txt
+
+mkdir -p data && cd data
+xargs -- wget --no-verbose --no-clobber <../ztm-data.txt
+
+for file in $(find . -name 'index*.zip'); do
+	dir="${file##*=}"
+	dir="${dir%.zip}"
+	if [ ! -d "$dir" ]; then
+		mkdir "$dir"
+		unzip "$file" -d "$dir" "${keep[@]}"
+	fi
+done
+
+cd ..
+
+k=$keep
+csv="${k%.txt}.csv"
+cat $(find data -name "$k") | shuf > "$csv"
+train_size=$(( $(wc -l "$csv" | cut -f1 -d' ') * 8 / 10 ))
+echo $train_size
+
+head -n $train_size  $csv >train.csv
+tail -n +$train_size $csv >test.csv
+
+# Disable to allow to work in bare jenkins
+# for k in "${keep[@]}"; do
+# 	csv="${k%.txt}.csv"
+# 	tsv="${k%.txt}.tsv"
+# 	if [ ! -f "$tsv" ]; then
+# 		cat $(find data -name "$k") > "$csv"
+# 		csv2tsv/csv2tsv <"$csv" >"$tsv"
+# 	fi
+# done
+#
+# if [ ! -f "stop_times.normalized.tsv" ]; then
+# 	./normalize <stop_times.tsv >stop_times.normalized.tsv
+# 	./split_train_valid_test.py
+# fi
+# ./stats.py

run.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-
-set -xe
-
-docker build -t ium .
-docker run -it ium

src/prepare-ztm-data.sh

@@ -1,41 +0,0 @@
-#!/usr/bin/env bash
-
-set -xe -o pipefail
-
-make normalize csv2tsv/csv2tsv
-
-keep=stop_times.txt
-
-mkdir -p data && cd data
-xargs -- wget --no-verbose --no-clobber <../ztm-data.txt
-
-for file in $(find . -name 'index*.zip'); do
-	dir="${file##*=}"
-	dir="${dir%.zip}"
-	if [ ! -d "$dir" ]; then
-		mkdir "$dir"
-		unzip "$file" -d "$dir" "${keep[@]}"
-	fi
-done
-
-cd ..
-
-for k in "${keep[@]}"; do
-	csv="${k%.txt}.csv"
-	tsv="${k%.txt}.tsv"
-	if [ ! -f "$tsv" ]; then
-		cat $(find data -name "$k") > "$csv"
-		csv2tsv/csv2tsv <"$csv" >"$tsv"
-	fi
-done
-
-if [ ! -f "stop_times.normalized.tsv" ]; then
-	./normalize <stop_times.tsv >stop_times.normalized.tsv
-fi
-
-if [ ! \( -f "stop_times.train.tsv" -a -f "stop_times.test.tsv" -a -f "stop_times.valid.tsv" \) ]; then
-	./split_train_valid_test.py
-fi
-
-
-./stats.py