mongoobserver added

Dockerfile

@@ -3,11 +3,6 @@ FROM ubuntu:focal
 RUN apt update
 
 RUN apt install -y python3 python3-pip
-# RUN pip3 install kaggle
-# RUN pip3 install pandas
-# RUN pip3 install tensorflow
-# RUN pip3 install scikit-learn
-# RUN pip3 install pandas
 # RUN apt install -y unzip
 
 RUN mkdir /.kaggle
@@ -19,13 +14,15 @@ COPY ./requirements.txt ./
 COPY ./avocado-preprocessing.py ./
 COPY ./avocado-training.py ./
 COPY ./avocado-evaluation.py ./
-COPY ./sacred-training.py ./
+COPY ./sacred-fileobserver.py ./
+COPY ./sacred-mongoobserver.py ./
 
 RUN chmod +x ./requirements.txt
 RUN chmod +x ./avocado-preprocessing.py 
 RUN chmod +x ./avocado-training.py
 RUN chmod +x ./avocado-evaluation.py
-RUN chmod +x ./sacred-training.py
+RUN chmod +x ./sacred-fileobserver.py
+RUN chmod +x ./sacred-mongoobserver.py
 
 RUN pip3 install -r ./requirements.txt
 # CMD python3 avocado-preprocessing.py

evaluation.Jenkinsfile

@@ -46,7 +46,8 @@ pipeline {
                     sh 'chmod +x avocado-evaluation.py'
                     sh "echo ${env.BUILD_ID}"
                     sh "python3 avocado-evaluation.py ${env.BUILD_ID}"
-                    sh 'python3 sacred-training.py'
+                    sh 'python3 sacred-fileobserver.py'
+                    sh 'python3 sacred-mongoobserver.py'
                 
                 }
 			}
@@ -56,13 +57,14 @@ pipeline {
 			steps{
 				archiveArtifacts 'eval_results.txt'
                 archiveArtifacts 'eval_plot.png'
+                archiveArtifacts '/my_runs'
 			}
 		}
 
 
         stage('sendMail') {
             steps{
-                emailext body: currentBuild.result ?: 'SUCCESS',
+                emailext body: "${currentBuild.currentResult}"",
                 subject: 's434742 evaluation',
                 to: '26ab8f35.uam.onmicrosoft.com@emea.teams.ms'
             }

requirements.txt

@@ -5,4 +5,5 @@ matplotlib==3.3.4
 numpy==1.19.5
 kaggle==1.5.12
 keras==2.4.3
-scikit_learn==0.24.2
+scikit_learn==0.24.2
+pymongo

sacred-fileobserver.py

@@ -63,8 +63,8 @@ def prepare_model(epochs, batch_size):
 
 
 
-@ex.main
+@ex.automain
-def my_main():
+def my_main(epochs, batch_size):
     print(prepare_model())
 
 ex.run()

sacred-mongoobserver.py

@@ -0,0 +1,71 @@
+import sys
+from keras.backend import mean
+import pandas as pd
+import numpy as np
+from sklearn import preprocessing
+from sklearn.linear_model import LinearRegression
+from sklearn.metrics import mean_squared_error
+import tensorflow as tf
+from tensorflow import keras
+from tensorflow.keras.layers import Input, Dense, Activation,Dropout
+from tensorflow.keras.models import Model
+from tensorflow.keras.callbacks import EarlyStopping
+from keras.models import Sequential
+from sacred import Experiment
+from sacred.observers import MongoObserver
+
+ex = Experiment("file_observer", interactive=False, save_git_info=False)
+ex.observers.append(MongoObserver(url='mongodb://mongo_user:mongo_password_IUM_2021@172.17.0.1:27017', db_name='sacred'))
+
+@ex.config
+def my_config():
+    epochs = 10
+    batch_size = 16
+
+@ex.capture
+def prepare_model(epochs, batch_size):
+    # odczytanie danych z plików
+    avocado_train = pd.read_csv('avocado_train.csv')
+    avocado_test = pd.read_csv('avocado_test.csv')
+    avocado_validate = pd.read_csv('avocado_validate.csv')
+
+
+    # podzial na X i y
+    X_train = avocado_train[['average_price', 'total_volume', '4046', '4225', '4770', 'total_bags', 'small_bags', 'large_bags', 'xlarge_bags']]
+    y_train = avocado_train[['type']]
+    X_test = avocado_test[['average_price', 'total_volume', '4046', '4225', '4770', 'total_bags', 'small_bags', 'large_bags', 'xlarge_bags']]
+    y_test = avocado_test[['type']]
+
+    print(X_train.shape[1])
+    # keras model
+    model = Sequential()
+    model.add(Dense(9, input_dim = X_train.shape[1], kernel_initializer='normal', activation='relu'))
+    model.add(Dense(1,kernel_initializer='normal', activation='sigmoid'))
+
+    early_stop = EarlyStopping(monitor="val_loss", mode="min", verbose=1, patience=10)
+
+    # kompilacja
+    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
+
+    # trenowanie modelu
+    model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, validation_data=(X_test, y_test))
+
+    # predykcja
+    prediction = model.predict(X_test)
+
+    # ewaluacja
+    rmse = mean_squared_error(y_test, prediction)
+
+    # zapisanie modelu
+    model.save('avocado_model.h5')
+
+    return rmse
+
+
+
+@ex.automain
+def my_main(epochs, batch_size):
+    print(prepare_model())
+
+ex.run()
+ex.add_artifact('avocado_model.h5')