update learning

.dvc/config

@@ -1,6 +1,5 @@
 [core]
     remote = ium_ssh_remote
-['remote "my_local_remote"']
-    url = /Users/adamwojdyla/Documents/Studia/Magisterskie/1_sem/IUM/ium_444507/dvcstore
 ['remote "ium_ssh_remote"']
-    url = ssh://ium-sftp@tzietkiewicz.vm.wmi.amu.edu.pl
+    url = ssh://tzietkiewicz.vm.wmi.amu.edu.pl:/home/ium-sftp
+    user = ium-sftp

.gitignore

@@ -153,11 +153,9 @@ fabric.properties
 # kaggle
 kaggle.json
 Car_Prices_Poland_Kaggle*.csv
-CarPrices*
 IUM08/*
 .DS_store
 *.db
 mlruns
 my_model
 dvcstore
-/prediction_results.csv

Jenkinsfile_dvc

@@ -10,9 +10,9 @@ pipeline {
     stages {
         stage('DVC') {
             steps {
+                checkout([$class: 'GitSCM', branches: [[name: '*/master']], extensions: [], userRemoteConfigs: [[credentialsId: 'S444507_cred', url: 'https://git.wmi.amu.edu.pl/s444507/ium_444507.git']]])
                 withCredentials(
-                    [sshUserPrivateKey(credentialsId: '48ac7004-216e-4260-abba-1fe5db753e18', keyFileVariable: 'IUM_SFTP_KEY', passphraseVariable: '', usernameVariable: '')]) {
-                        sh 'dvc remote add -d ium_ssh_remote ssh://ium-sftp@tzietkiewicz.vm.wmi.amu.edu.pl/ium-sftp'
+                    [sshUserPrivateKey(credentialsId: '48ac7004-216e-4260-abba-1fe5db753e18', keyFileVariable: 'IUM_SFTP_KEY', passphraseVariable: '', usernameVariable: 'USER')]) {
                         sh 'dvc remote modify --local ium_ssh_remote keyfile $IUM_SFTP_KEY'
                         sh 'dvc pull'
                         sh 'dvc repro'
@@ -20,9 +20,4 @@ pipeline {
             }
         }
     }
-    post {
-        success {
-            archiveArtifacts artifacts: 'prediction_results.csv, *.pkl', followSymlinks: false
-        }
-    }
 }

data/.gitignore

@@ -1 +1,5 @@
 /prepared
+/Car_Prices_Poland_Kaggle_test.csv
+/Car_Prices_Poland_Kaggle_train.csv
+/Car_Prices_Poland_Kaggle_dev.csv
+/Car_Prices_Poland_Kaggle.csv

data/Car_Prices_Poland_Kaggle_dev.csv.dvc

@@ -0,0 +1,4 @@
+outs:
+- md5: 099e43435758084862777c03cc2feb02
+  size: 1648018
+  path: Car_Prices_Poland_Kaggle_dev.csv

data/Car_Prices_Poland_Kaggle_test.csv.dvc

@@ -0,0 +1,4 @@
+outs:
+- md5: 099e43435758084862777c03cc2feb02
+  size: 1648018
+  path: Car_Prices_Poland_Kaggle_test.csv

data/Car_Prices_Poland_Kaggle_train.csv.dvc

@@ -0,0 +1,4 @@
+outs:
+- md5: 47632fa91312b676baf3053a7a1b4f2b
+  size: 6598832
+  path: Car_Prices_Poland_Kaggle_train.csv

dvc.lock

@@ -9,38 +9,21 @@ stages:
     - path: script_prepare.py
       md5: f1dfe33a503f5acc687c53dee448f71b
       size: 1899
-    outs:
-    - path: data/Car_Prices_Poland_Kaggle_dev.csv
-      md5: cf9355749edc79f588e264de5b2bf1f0
-      size: 1648309
-    - path: data/Car_Prices_Poland_Kaggle_test.csv
-      md5: cf9355749edc79f588e264de5b2bf1f0
-      size: 1648309
-    - path: data/Car_Prices_Poland_Kaggle_train.csv
-      md5: 8818f758e2de344a4b9ad712379b81e1
-      size: 6597472
   train:
-    cmd: python3 lab05_deepLearning.py 50
+    cmd: python3 lab05_deepLearning.py 70
     deps:
     - path: data/Car_Prices_Poland_Kaggle_dev.csv
-      md5: cf9355749edc79f588e264de5b2bf1f0
-      size: 1648309
+      md5: 113fad808a17e2aa5131832ecaa2e640
+      size: 1647374
     - path: data/Car_Prices_Poland_Kaggle_test.csv
-      md5: cf9355749edc79f588e264de5b2bf1f0
-      size: 1648309
+      md5: 113fad808a17e2aa5131832ecaa2e640
+      size: 1647374
     - path: data/Car_Prices_Poland_Kaggle_train.csv
-      md5: 8818f758e2de344a4b9ad712379b81e1
-      size: 6597472
-    outs:
-    - path: CarPrices_pytorch_model.pkl
-      md5: cff6a79945bbf839058a4fd1b2dcc98f
-      size: 30039
-    - path: prediction_results.csv
-      md5: 62b9e54cdfebc7f1dfb060e18e9b8738
-      size: 585197
+      md5: dd41429d2b3285cc85b94a9b0ec8cf91
+      size: 6597186
   evaluate:
     cmd: python3 lab10_evaluate.py
     deps:
     - path: CarPrices_pytorch_model.pkl
-      md5: cff6a79945bbf839058a4fd1b2dcc98f
+      md5: a73485a169b6185b0161d75bc5c883a3
       size: 30039

dvc.yaml

@@ -4,19 +4,12 @@ stages:
     deps:
     - data/Car_Prices_Poland_Kaggle.csv
     - script_prepare.py
-    outs:
-    - data/Car_Prices_Poland_Kaggle_dev.csv
-    - data/Car_Prices_Poland_Kaggle_train.csv
-    - data/Car_Prices_Poland_Kaggle_test.csv
   train:
     cmd:  python3 lab05_deepLearning.py 70
     deps:
     - data/Car_Prices_Poland_Kaggle_dev.csv
     - data/Car_Prices_Poland_Kaggle_train.csv
     - data/Car_Prices_Poland_Kaggle_test.csv
-    outs:
-    - CarPrices_pytorch_model.pkl
-    - prediction_results.csv
   evaluate:
     cmd:  python3 lab10_evaluate.py
     deps:

lab05_deepLearning.py

@@ -11,7 +11,12 @@ import torch.nn.functional as F
 import pandas as pd
 from sklearn import preprocessing
 import sys
+import os
 
+path = '.'
+files = os.listdir(".")
+if not "Car_Prices_Poland_Kaggle.csv" in files:
+    path = "data"
 
 class Model(nn.Module):
     def __init__(self, input_dim):
@@ -29,15 +34,15 @@ class Model(nn.Module):
 
 def load_dataset_raw():
     """ Load data from .csv file. """
-    cars = pd.read_csv('./Car_Prices_Poland_Kaggle.csv', usecols=[1, 4, 5, 6, 10], sep=',')
+    cars = pd.read_csv(f'{path}/Car_Prices_Poland_Kaggle.csv', usecols=[1, 4, 5, 6, 10], sep=',')
     return cars
 
 
 def load_dataset_files():
     """ Load shuffled, splitted dev and train files from .csv files. """
 
-    cars_dev = pd.read_csv('./Car_Prices_Poland_Kaggle_dev.csv', usecols=[1, 4, 5, 6, 10], sep=',', names= [str(i) for i in range(5)])
-    cars_train = pd.read_csv('./Car_Prices_Poland_Kaggle_train.csv', usecols=[1, 4, 5, 6, 10], sep=',', names= [str(i) for i in range(5)])
+    cars_dev = pd.read_csv(f'{path}/Car_Prices_Poland_Kaggle_dev.csv', usecols=[1, 4, 5, 6, 10], sep=',', names= [str(i) for i in range(5)])
+    cars_train = pd.read_csv(f'{path}/Car_Prices_Poland_Kaggle_train.csv', usecols=[1, 4, 5, 6, 10], sep=',', names= [str(i) for i in range(5)])
 
     return cars_dev, cars_train
 
@@ -65,15 +70,6 @@ def prepare_labels_features(dataset):
     return lab, feat
 
 
-
-
-
-# def draw_plot(lbl):
-# need to import matplotlib to work
-#     plt.hist(lbl, bins=[i for i in range(len(set(lbl)))], edgecolor="black")
-#     plt.xticks(np.arange(0, len(set(lbl)), 1))
-#     plt.show()
-
 # Prepare dataset
 print("Loading dataset...")
 dev, train = load_dataset_files()