dvc

.dvc/.gitignore

@@ -0,0 +1,3 @@
+/config.local
+/tmp
+/cache

.dvc/config

@@ -0,0 +1,6 @@
+[core]
+    analytics = false
+    autostage = true
+    remote = ium_ssh_remote
+['remote "ium_ssh_remote"']
+    url = ssh://ium-sftp@tzietkiewicz.vm.wmi.amu.edu.pl

.dvcignore

@@ -0,0 +1,3 @@
+# Add patterns of files dvc should ignore, which could improve
+# the performance. Learn more at
+# https://dvc.org/doc/user-guide/dvcignore

Dockerfile

@@ -1,5 +1,5 @@
 FROM ubuntu:22.04
 
 RUN apt update && apt install -y vim make python3 python3-pip python-is-python3 gcc g++ golang wget unzip git
-RUN pip install pandas matplotlib scikit-learn tensorflow
+RUN pip install pandas matplotlib scikit-learn tensorflow dvc dvc[ssh] paramiko
 CMD "bash"

dvc.jenkinsfile

@@ -0,0 +1,19 @@
+node {
+	checkout scm
+
+	docker.build("s452639-image-dvc").inside {
+		stage('Build') {
+			checkout([$class: 'GitSCM', branches: [[name: "dvc"]], extensions: [], userRemoteConfigs: [[url: 'https://git.wmi.amu.edu.pl/s452639/ium_452639']]])
+
+			withCredentials(
+				[sshUserPrivateKey(credentialsId: '48ac7004-216e-4260-abba-1fe5db753e18', keyFileVariable: 'IUM_SFTP_KEY', passphraseVariable: '', usernameVariable: '')]
+			) {
+				sh 'dvc remote add -d ium_ssh_remote ssh://ium-sftp@tzietkiewicz.vm.wmi.amu.edu.pl/ium-sftp'
+				sh 'dvc remote modify --local ium_ssh_remote keyfile $IUM_SFTP_KEY'
+				sh 'dvc pull'
+				sh 'dvc repro'
+			}
+		}
+	}
+}
+

dvc.lock

@@ -0,0 +1,37 @@
+schema: '2.0'
+stages:
+  train:
+    cmd: python3 tf_train.py
+    deps:
+    - path: stop_times.categories.tsv
+      md5: d88f6a43257758c19a8cd8d708459958
+      size: 3693
+    - path: stop_times.train.tsv
+      md5: 4f95b09a81a43d946f97ea05ff83dae0
+      size: 59369274
+    - path: stop_times.valid.tsv
+      md5: 179d6d81c2d284ae84af89c66dcf378d
+      size: 133596844
+    outs:
+    - path: model.keras
+      md5: f7136946f9bafe2b6d8a794379292631
+      size: 19874552
+  test:
+    cmd: python3 tf_test.py
+    deps:
+    - path: model.keras
+      md5: f7136946f9bafe2b6d8a794379292631
+      size: 19874552
+    - path: stop_times.categories.tsv
+      md5: d88f6a43257758c19a8cd8d708459958
+      size: 3693
+    - path: stop_times.test.tsv
+      md5: 2b6b49382efb97c511755f21453f8726
+      size: 44527766
+    outs:
+    - path: stop_times.accuracy.tsv
+      md5: dd1d9835fa82bf98491de73c74e42ec3
+      size: 20
+    - path: stop_times.predictions.tsv
+      md5: 5e18d5b87fce01e5f87c9939ec84fdca
+      size: 16681428

dvc.yaml

@@ -0,0 +1,20 @@
+stages:
+  train:
+    cmd: python3 tf_train.py
+    wdir: src
+    deps:
+    - stop_times.categories.tsv
+    - stop_times.train.tsv
+    - stop_times.valid.tsv
+    outs:
+    - model.keras
+  test:
+    cmd: python3 tf_test.py
+    wdir: src
+    deps:
+    - stop_times.categories.tsv
+    - stop_times.test.tsv
+    - model.keras
+    outs:
+    - stop_times.accuracy.tsv
+    - stop_times.predictions.tsv

src/stop_times.categories.tsv.dvc

@@ -0,0 +1,4 @@
+outs:
+- md5: d88f6a43257758c19a8cd8d708459958
+  size: 3693
+  path: stop_times.categories.tsv

src/stop_times.test.tsv.dvc

@@ -0,0 +1,4 @@
+outs:
+- md5: 2b6b49382efb97c511755f21453f8726
+  size: 44527766
+  path: stop_times.test.tsv

src/stop_times.train.tsv.dvc

@@ -0,0 +1,4 @@
+outs:
+- md5: 4f95b09a81a43d946f97ea05ff83dae0
+  size: 59369274
+  path: stop_times.train.tsv

src/stop_times.valid.tsv.dvc

@@ -0,0 +1,4 @@
+outs:
+- md5: 179d6d81c2d284ae84af89c66dcf378d
+  size: 133596844
+  path: stop_times.valid.tsv