add Jenkinsfile_dvc, dvc.yaml, modify Dockerfile

Dockerfile

@@ -35,4 +35,10 @@ COPY registry.py .
 
 RUN chmod +x ./download.sh
 RUN ./download.sh
+
+RUN pip3 install dvc
+RUN pip3 install dvc[ssh] paramiko
+RUN apt install -y sshpass openssh-client
+RUN useradd -r -u 111 jenkins
+
 #CMD python3 ./dllib.py

Jenkinsfile_dvc

@@ -0,0 +1,31 @@
+pipeline {
+    agent {
+        docker {
+			image 'docker_image'
+		}
+    }
+
+stages {
+
+	stage('checkout'){
+            steps{
+                checkout([$class: 'GitSCM', branches: [[name: '*/master']], extensions: [], userRemoteConfigs: [[credentialsId: 's444018', url: 'https://git.wmi.amu.edu.pl/s444018/ium_444018.git']]])
+            }
+        }
+
+        stage("DVC"){
+		steps {
+            		withCredentials([sshUserPrivateKey(credentialsId: '48ac7004-216e-4260-abba-1fe5db753e18', keyFileVariable: 'IUM_SFTP_KEY')]) {
+                	sh 'dvc remote modify --local ium_ssh_remote keyfile $IUM_SFTP_KEY'
+			sh 'dvc remote modify --local ium_ssh_remote password IUM@2021'
+			sh 'dvc remote list'
+			sh 'cat .dvc/config'
+			sh 'cat .dvc/config.local'
+                	sh 'dvc pull'
+			sh 'dvc repro'
+			sh 'ls -al'
+		}
+            }
+        }
+    }
+}

biblioteka_DL/dllib.py

@@ -65,7 +65,7 @@ def normalize_gross(imbd_data):
 
 
 def prepare_dataset():
-    df = pd.read_csv('biblioteka_DL/imdb_top_1000.csv')
+    df = pd.read_csv('Data/imdb_top_1000.csv')
     df = drop_relevant_columns(df)
     df_lowercase = lowercase_columns_names(df)
     df = data_to_numeric(df_lowercase)
@@ -167,4 +167,5 @@ def my_main(epochs):
 
 
 with mlflow.start_run() as run:
-    my_main(epochs)
+    my_main(epochs)
+    

biblioteka_DL/evaluate.py

@@ -56,7 +56,7 @@ def normalize_gross(imbd_data):
 
 
 def prepare_dataset():
-    df = pd.read_csv('biblioteka_DL/imdb_top_1000.csv')
+    df = pd.read_csv('Data/imdb_top_1000.csv')
     df = drop_relevant_columns(df)
     df_lowercase = lowercase_columns_names(df)
     df = data_to_numeric(df_lowercase)

dvc.yaml

@@ -0,0 +1,9 @@
+stages:
+  train:
+    cmd: python3 ./biblioteka_DL/dllib.py 100
+    deps:
+    - Data/imdb_top_1000.csv
+  evaluate:
+    cmd: python3 ./biblioteka_DL/evaluate.py
+    deps:
+    - Data/imdb_top_1000.csv