From 29329d7efc6ba3ee85ce682ff8bf6a1c7f842977 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Szymon=20Parafin=CC=81ski?= <sparafinski@fandom.com>
Date: Sun, 5 Jun 2022 22:53:44 +0200
Subject: [PATCH] add Jenkinsfile_dvc, dvc.yaml, modify Dockerfile

---
 Dockerfile                |  6 ++++++
 Jenkinsfile_dvc           | 31 +++++++++++++++++++++++++++++++
 biblioteka_DL/dllib.py    |  5 +++--
 biblioteka_DL/evaluate.py |  2 +-
 dvc.yaml                  |  9 +++++++++
 5 files changed, 50 insertions(+), 3 deletions(-)
 create mode 100644 Jenkinsfile_dvc
 create mode 100644 dvc.yaml

diff --git a/Dockerfile b/Dockerfile
index fb84dd0..88f2dc1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -35,4 +35,10 @@ COPY registry.py .
 
 RUN chmod +x ./download.sh
 RUN ./download.sh
+
+RUN pip3 install dvc
+RUN pip3 install dvc[ssh] paramiko
+RUN apt install -y sshpass openssh-client
+RUN useradd -r -u 111 jenkins
+
 #CMD python3 ./dllib.py
diff --git a/Jenkinsfile_dvc b/Jenkinsfile_dvc
new file mode 100644
index 0000000..9ef397e
--- /dev/null
+++ b/Jenkinsfile_dvc
@@ -0,0 +1,31 @@
+pipeline {
+    agent {
+        docker {
+			image 'docker_image'
+		}
+    }
+
+stages {
+
+	stage('checkout'){
+            steps{
+                checkout([$class: 'GitSCM', branches: [[name: '*/master']], extensions: [], userRemoteConfigs: [[credentialsId: 's444018', url: 'https://git.wmi.amu.edu.pl/s444018/ium_444018.git']]])
+            }
+        }
+
+        stage("DVC"){
+		steps {
+            		withCredentials([sshUserPrivateKey(credentialsId: '48ac7004-216e-4260-abba-1fe5db753e18', keyFileVariable: 'IUM_SFTP_KEY')]) {
+                	sh 'dvc remote modify --local ium_ssh_remote keyfile $IUM_SFTP_KEY'
+			sh 'dvc remote modify --local ium_ssh_remote password IUM@2021'
+			sh 'dvc remote list'
+			sh 'cat .dvc/config'
+			sh 'cat .dvc/config.local'
+                	sh 'dvc pull'
+			sh 'dvc repro'
+			sh 'ls -al'
+		}
+            }
+        }
+    }
+}
diff --git a/biblioteka_DL/dllib.py b/biblioteka_DL/dllib.py
index b89ad6d..e080d26 100644
--- a/biblioteka_DL/dllib.py
+++ b/biblioteka_DL/dllib.py
@@ -65,7 +65,7 @@ def normalize_gross(imbd_data):
 
 
 def prepare_dataset():
-    df = pd.read_csv('biblioteka_DL/imdb_top_1000.csv')
+    df = pd.read_csv('Data/imdb_top_1000.csv')
     df = drop_relevant_columns(df)
     df_lowercase = lowercase_columns_names(df)
     df = data_to_numeric(df_lowercase)
@@ -167,4 +167,5 @@ def my_main(epochs):
 
 
 with mlflow.start_run() as run:
-    my_main(epochs)
\ No newline at end of file
+    my_main(epochs)
+    
diff --git a/biblioteka_DL/evaluate.py b/biblioteka_DL/evaluate.py
index 34b6acd..b469bc3 100644
--- a/biblioteka_DL/evaluate.py
+++ b/biblioteka_DL/evaluate.py
@@ -56,7 +56,7 @@ def normalize_gross(imbd_data):
 
 
 def prepare_dataset():
-    df = pd.read_csv('biblioteka_DL/imdb_top_1000.csv')
+    df = pd.read_csv('Data/imdb_top_1000.csv')
     df = drop_relevant_columns(df)
     df_lowercase = lowercase_columns_names(df)
     df = data_to_numeric(df_lowercase)
diff --git a/dvc.yaml b/dvc.yaml
new file mode 100644
index 0000000..2cb93b6
--- /dev/null
+++ b/dvc.yaml
@@ -0,0 +1,9 @@
+stages:
+  train:
+    cmd: python3 ./biblioteka_DL/dllib.py 100
+    deps:
+    - Data/imdb_top_1000.csv
+  evaluate:
+    cmd: python3 ./biblioteka_DL/evaluate.py
+    deps:
+    - Data/imdb_top_1000.csv