lab7 exercise start

lab7/Dockerfile

@@ -0,0 +1,18 @@
+FROM ubuntu:latest
+
+WORKDIR /ium
+
+RUN apt update && apt install -y python3-pip
+
+RUN pip3 install pandas
+RUN pip3 install numpy
+RUN pip3 install sklearn
+RUN pip3 install tensorflow
+RUN pip3 install matplotlib
+RUN pip3 install keras
+run pip3 install sacred
+run pip3 install pymongo
+
+COPY ./lab7/simple_regression.py ./
+COPY ./evaluate.py ./
+COPY ./plot.py ./

lab7/Jenkinsfile_eval

@@ -0,0 +1,59 @@
+pipeline {
+    agent {
+        dockerfile true
+        }
+	parameters {
+        gitParameter branchFilter: 'origin/(.*)', defaultValue: 'master', name: 'BRANCH', type: 'PT_BRANCH'
+        buildSelector(
+			defaultSelector: lastSuccessful(),
+			description: 'Which build to use for copying artifacts',
+			name: 'BUILD_SELECTOR'
+		)
+    }
+    stages {
+        stage('Stage 1') {
+            steps {
+                    git branch: "${params.BRANCH}", url: 'https://git.wmi.amu.edu.pl/s449288/ium_s449288.git'
+				    sh 'chmod u+x ./evaluate.py'
+					echo 'Copying datasets from the create-dataset job...'
+					copyArtifacts filter: 'lego_sets_clean_test.csv', projectName: 's449288-create-dataset'
+					echo 'Datasets copied'					
+					echo 'Copying model from the training job...'
+					copyArtifacts filter: 'lego_reg_model.tar.gz', projectName: "s449288-training/${BRANCH}/", selector: buildParameter('BUILD_SELECTOR') 				
+					echo 'Model copied'
+					sh 'tar xvzf lego_reg_model.tar.gz'				
+					echo 'Optional copying of the metrics file from previous build...'
+					copyArtifacts filter: 'eval_results.txt', projectName: 's449288-evaluation/master/', optional: true
+					echo 'Metrics file copied if it did not exist'
+					echo 'Evaluating model...'
+					sh 'python3 evaluate.py'
+					echo 'Model evaluated. Metrics saved. Plot saved.'
+                    sh 'head eval_results.txt'	
+					sh 'file error_plot.jpg'
+                    echo 'Archiving metrics file...'
+                    archiveArtifacts 'eval_results.txt'
+                    echo 'File archived'
+                    script {
+                        LAST_MAE = sh (
+                        script: 'tail -1 eval_results.txt',
+                        returnStdout: true
+                        ).trim()
+                    }
+				}
+            }
+        }
+	post {
+        success {
+            emailext body: "SUCCESS - ${LAST_MAE} MAE", subject: 's449288-evaluation build status', to: 'e19191c5.uam.onmicrosoft.com@emea.teams.ms'
+        }
+        failure {
+            emailext body: "FAILURE - ${LAST_MAE} MAE", subject: 's449288-evaluation build status', to: 'e19191c5.uam.onmicrosoft.com@emea.teams.ms'
+        }
+        unstable {
+            emailext body: "UNSTABLE - ${LAST_MAE} MAE", subject: 's449288-evaluation build status', to: 'e19191c5.uam.onmicrosoft.com@emea.teams.ms'
+        }
+        changed {
+            emailext body: "CHANGED - ${LAST_MAE} MAE", subject: 's449288-evaluation build status', to: 'e19191c5.uam.onmicrosoft.com@emea.teams.ms'
+        }		
+    }
+}

lab7/Jenkinsfile_train

@@ -0,0 +1,52 @@
+pipeline {
+    agent {
+        dockerfile true
+        }
+    parameters {
+        string(
+			defaultValue: '100',
+			description: 'Example training parameter',
+			name: 'EPOCHS_NUM'
+		)
+    }
+    stages {
+        stage('Stage 1') {
+            steps {
+				    sh 'chmod u+x ./simple_regression.py'
+					echo 'Copying datasets from create-dataset...'
+					copyArtifacts filter: '*', projectName: 's449288-create-dataset'
+					echo 'Datasets copied'
+					echo 'Conducting simple regression model test'
+					sh 'python3 simple_regression.py $EPOCHS_NUM'
+					echo 'Model and predictions saved'
+                    sh 'head lego_reg_results.csv'
+					sh 'ls -lh lego_reg_model'
+                    echo 'Archiving model...'
+                    sh 'tar -czf lego_reg_model.tar.gz lego_reg_model/'
+                    archiveArtifacts 'lego_reg_model.tar.gz'
+                    echo 'Model archived'
+					echo 'Archiving Sacred's output repo...'
+					sh 'ls -lh runs/*/'
+					sh 'tar -czf sacred_runs.tar.gz runs/'
+                    archiveArtifacts 'sacred_runs.tar.gz'
+					echo 'Sacred's repo archived'
+					echo 'Launching the s449288-evaluation job...'
+					build job: 's449288-evaluation/master/'
+				}
+            }
+        }
+    post {
+        success {
+            emailext body: 'SUCCESS', subject: 's449288-training build status', to: 'e19191c5.uam.onmicrosoft.com@emea.teams.ms'
+        }
+        failure {
+            emailext body: 'FAILURE', subject: 's449288-training build status', to: 'e19191c5.uam.onmicrosoft.com@emea.teams.ms'
+        }
+        unstable {
+            emailext body: 'UNSTABLE', subject: 's449288-training build status', to: 'e19191c5.uam.onmicrosoft.com@emea.teams.ms'
+        }
+        changed {
+            emailext body: 'CHANGED', subject: 's449288-training build status', to: 'e19191c5.uam.onmicrosoft.com@emea.teams.ms'
+        }		
+    }
+}

lab7/simple_regression.py

@@ -0,0 +1,89 @@
+import tensorflow as tf
+from keras import layers
+from keras.models import save_model
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import sys
+from sacred import Experiment
+from sacred.observers import FileStorageObserver
+from sacred.observers import MongoObserver
+
+# Stworzenie obiektu klasy Experiment do śledzenia przebiegu regresji narzędziem Sacred
+ex = Experiment()
+
+# Dodanie obserwatora FileObserver
+ex.observers.append(FileStorageObserver('runs'))
+
+#Dodanie obserwatora Mongo
+ex.observers.append(MongoObserver(url='mongodb://mongo_user:mongo_password_IUM_2021@localhost:27017', db_name='sacred'))
+
+# Przykładowa modyfikowalna z Sacred konfiguracja wybranych parametrów treningu
+@ex.config
+def config():
+    units = 1
+    learning_rate = 0.1
+
+
+# Reszta kodu wrzucona do udekorowanej funkcji train do wywołania przez Sacred, żeby coś było capture'owane
+@ex.capture
+def train(units, learning_rate, _run):
+    # Pobranie przykładowego argumentu trenowania z poziomu Jenkinsa
+    EPOCHS_NUM = int(sys.argv[1])
+
+    # Wczytanie danych
+    data_train = pd.read_csv('lego_sets_clean_train.csv')
+    data_test = pd.read_csv('lego_sets_clean_test.csv')
+
+    # Wydzielenie zbiorów dla predykcji ceny zestawu na podstawie liczby klocków, którą zawiera
+    train_piece_counts = np.array(data_train['piece_count'])
+    train_prices = np.array(data_train['list_price'])
+    test_piece_counts = np.array(data_test['piece_count'])
+    test_prices = np.array(data_test['list_price'])
+
+    # Normalizacja
+    normalizer = layers.Normalization(input_shape=[1, ], axis=None)
+    normalizer.adapt(train_piece_counts)
+
+    # Inicjalizacja
+    model = tf.keras.Sequential([
+        normalizer,
+        layers.Dense(units=units)
+    ])
+
+    # Kompilacja
+    model.compile(
+        optimizer=tf.optimizers.Adam(learning_rate=learning_rate),
+        loss='mean_absolute_error'
+    )
+
+    # Trening
+    history = model.fit(
+        train_piece_counts,
+        train_prices,
+        epochs=EPOCHS_NUM,
+        verbose=0,
+        validation_split=0.2
+    )
+
+    # Wykonanie predykcji na danych ze zbioru testującego
+    y_pred = model.predict(test_piece_counts)
+
+    # Zapis predykcji do pliku
+    results = pd.DataFrame(
+        {'test_set_piece_count': test_piece_counts.tolist(), 'predicted_price': [round(a[0], 2) for a in y_pred.tolist()]})
+    results.to_csv('lego_reg_results.csv', index=False, header=True)
+
+    # Zapis modelu do pliku standardowo poprzez metodę kerasa i poprzez metodę obiektu Experiment z Sacred
+    model.save('lego_reg_model')
+    ex.add_artifact('lego_reg_model_art')
+
+    # Przykładowo zwracamy coś w charakterze wyników, żeby było widoczne w plikach zapisanych przez obserwator
+    hist = pd.DataFrame(history.history)
+    hist['epoch'] = history.epoch
+    _run.info["train_results"] = str(hist.tail())
+    return hist.tail()
+
+@ex.automain
+def main(units, learning_rate):
+    print(train())