diff --git a/Dockerfile b/Dockerfile index a1835b2..3d5fd47 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,11 +3,6 @@ FROM ubuntu:focal RUN apt update RUN apt install -y python3 python3-pip -# RUN pip3 install kaggle -# RUN pip3 install pandas -# RUN pip3 install tensorflow -# RUN pip3 install scikit-learn -# RUN pip3 install pandas # RUN apt install -y unzip RUN mkdir /.kaggle @@ -19,13 +14,15 @@ COPY ./requirements.txt ./ COPY ./avocado-preprocessing.py ./ COPY ./avocado-training.py ./ COPY ./avocado-evaluation.py ./ -COPY ./sacred-training.py ./ +COPY ./sacred-fileobserver.py ./ +COPY ./sacred-mongoobserver.py ./ RUN chmod +x ./requirements.txt RUN chmod +x ./avocado-preprocessing.py RUN chmod +x ./avocado-training.py RUN chmod +x ./avocado-evaluation.py -RUN chmod +x ./sacred-training.py +RUN chmod +x ./sacred-fileobserver.py +RUN chmod +x ./sacred-mongoobserver.py RUN pip3 install -r ./requirements.txt # CMD python3 avocado-preprocessing.py diff --git a/evaluation.Jenkinsfile b/evaluation.Jenkinsfile index 8eb860b..0e2a87e 100644 --- a/evaluation.Jenkinsfile +++ b/evaluation.Jenkinsfile @@ -46,7 +46,8 @@ pipeline { sh 'chmod +x avocado-evaluation.py' sh "echo ${env.BUILD_ID}" sh "python3 avocado-evaluation.py ${env.BUILD_ID}" - sh 'python3 sacred-training.py' + sh 'python3 sacred-fileobserver.py' + sh 'python3 sacred-mongoobserver.py' } } @@ -56,13 +57,14 @@ pipeline { steps{ archiveArtifacts 'eval_results.txt' archiveArtifacts 'eval_plot.png' + archiveArtifacts '/my_runs' } } stage('sendMail') { steps{ - emailext body: currentBuild.result ?: 'SUCCESS', + emailext body: "${currentBuild.currentResult}"", subject: 's434742 evaluation', to: '26ab8f35.uam.onmicrosoft.com@emea.teams.ms' } diff --git a/requirements.txt b/requirements.txt index 6ae2cb0..e357e93 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ matplotlib==3.3.4 numpy==1.19.5 kaggle==1.5.12 keras==2.4.3 -scikit_learn==0.24.2 \ No newline at end of file +scikit_learn==0.24.2 +pymongo \ No newline at end of file diff --git a/sacred-training.py b/sacred-fileobserver.py similarity index 96% rename from sacred-training.py rename to sacred-fileobserver.py index f46264b..6d26a1e 100644 --- a/sacred-training.py +++ b/sacred-fileobserver.py @@ -63,8 +63,8 @@ def prepare_model(epochs, batch_size): -@ex.main -def my_main(): +@ex.automain +def my_main(epochs, batch_size): print(prepare_model()) ex.run() diff --git a/sacred-mongoobserver.py b/sacred-mongoobserver.py new file mode 100644 index 0000000..17471c1 --- /dev/null +++ b/sacred-mongoobserver.py @@ -0,0 +1,71 @@ +import sys +from keras.backend import mean +import pandas as pd +import numpy as np +from sklearn import preprocessing +from sklearn.linear_model import LinearRegression +from sklearn.metrics import mean_squared_error +import tensorflow as tf +from tensorflow import keras +from tensorflow.keras.layers import Input, Dense, Activation,Dropout +from tensorflow.keras.models import Model +from tensorflow.keras.callbacks import EarlyStopping +from keras.models import Sequential +from sacred import Experiment +from sacred.observers import MongoObserver + +ex = Experiment("file_observer", interactive=False, save_git_info=False) +ex.observers.append(MongoObserver(url='mongodb://mongo_user:mongo_password_IUM_2021@172.17.0.1:27017', db_name='sacred')) + +@ex.config +def my_config(): + epochs = 10 + batch_size = 16 + +@ex.capture +def prepare_model(epochs, batch_size): + # odczytanie danych z plików + avocado_train = pd.read_csv('avocado_train.csv') + avocado_test = pd.read_csv('avocado_test.csv') + avocado_validate = pd.read_csv('avocado_validate.csv') + + + # podzial na X i y + X_train = avocado_train[['average_price', 'total_volume', '4046', '4225', '4770', 'total_bags', 'small_bags', 'large_bags', 'xlarge_bags']] + y_train = avocado_train[['type']] + X_test = avocado_test[['average_price', 'total_volume', '4046', '4225', '4770', 'total_bags', 'small_bags', 'large_bags', 'xlarge_bags']] + y_test = avocado_test[['type']] + + print(X_train.shape[1]) + # keras model + model = Sequential() + model.add(Dense(9, input_dim = X_train.shape[1], kernel_initializer='normal', activation='relu')) + model.add(Dense(1,kernel_initializer='normal', activation='sigmoid')) + + early_stop = EarlyStopping(monitor="val_loss", mode="min", verbose=1, patience=10) + + # kompilacja + model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) + + # trenowanie modelu + model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, validation_data=(X_test, y_test)) + + # predykcja + prediction = model.predict(X_test) + + # ewaluacja + rmse = mean_squared_error(y_test, prediction) + + # zapisanie modelu + model.save('avocado_model.h5') + + return rmse + + + +@ex.automain +def my_main(epochs, batch_size): + print(prepare_model()) + +ex.run() +ex.add_artifact('avocado_model.h5') \ No newline at end of file