sacred training added
Some checks failed
s434742-training/pipeline/head There was a failure building this commit
s434742-evaluation/pipeline/head There was a failure building this commit

This commit is contained in:
patrycjalazna 2021-05-14 20:48:13 +02:00
parent 88022c0556
commit 0ac6258c67
4 changed files with 80 additions and 2 deletions

View File

@ -19,11 +19,13 @@ COPY ./requirements.txt ./
COPY ./avocado-preprocessing.py ./ COPY ./avocado-preprocessing.py ./
COPY ./avocado-training.py ./ COPY ./avocado-training.py ./
COPY ./avocado-evaluation.py ./ COPY ./avocado-evaluation.py ./
COPY ./sacred-training.py ./
RUN chmod +x ./requirements.txt RUN chmod +x ./requirements.txt
RUN chmod +x ./avocado-preprocessing.py RUN chmod +x ./avocado-preprocessing.py
RUN chmod +x ./avocado-training.py RUN chmod +x ./avocado-training.py
RUN chmod +x ./avocado-evaluation.py RUN chmod +x ./avocado-evaluation.py
RUN chmod +x ./sacred-training.py
RUN pip3 install -r ./requirements.txt RUN pip3 install -r ./requirements.txt
# CMD python3 avocado-preprocessing.py # CMD python3 avocado-preprocessing.py

View File

@ -45,7 +45,7 @@ pipeline {
sh 'chmod +x avocado-evaluation.py' sh 'chmod +x avocado-evaluation.py'
sh "echo ${env.BUILD_ID}" sh "echo ${env.BUILD_ID}"
sh "python3 avocado-evaluation.py ${env.BUILD_ID}" sh "python3 avocado-evaluation.py ${env.BUILD_ID}"
sh 'python3 sacred-training.py'
} }
} }

View File

@ -6,3 +6,4 @@ tensorflow==2.4.1
keras keras
jinja2==2.11.3 jinja2==2.11.3
matplotlib matplotlib
sacred==0.8.2

75
sacred-training.py Normal file
View File

@ -0,0 +1,75 @@
import sys
from keras.backend import mean
import pandas as pd
import numpy as np
from sklearn import preprocessing
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.layers import Input, Dense, Activation,Dropout
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import EarlyStopping
from keras.models import Sequential
from sacred import Experiment
from sacred.observers import FileStorageObserver
ex = Experiment("file_observer")
ex.observers.append(FileStorageObserver('my_runs'))
@ex.config
def my_config():
epochs = 10
batch_size = 16
@ex.capture
def prepare_model():
# odczytanie danych z plików
avocado_train = pd.read_csv('avocado_train.csv')
avocado_test = pd.read_csv('avocado_test.csv')
avocado_validate = pd.read_csv('avocado_validate.csv')
# podzial na X i y
X_train = avocado_train[['average_price', 'total_volume', '4046', '4225', '4770', 'total_bags', 'small_bags', 'large_bags', 'xlarge_bags']]
y_train = avocado_train[['type']]
X_test = avocado_test[['average_price', 'total_volume', '4046', '4225', '4770', 'total_bags', 'small_bags', 'large_bags', 'xlarge_bags']]
y_test = avocado_test[['type']]
print(X_train.shape[1])
# keras model
model = Sequential()
model.add(Dense(9, input_dim = X_train.shape[1], kernel_initializer='normal', activation='relu'))
model.add(Dense(1,kernel_initializer='normal', activation='sigmoid'))
early_stop = EarlyStopping(monitor="val_loss", mode="min", verbose=1, patience=10)
# kompilacja
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
# model fit
epochs = int(sys.argv[1])
batch_size = int(sys.argv[2])
# trenowanie modelu
model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, validation_data=(X_test, y_test))
# predykcja
prediction = model.predict(X_test)
# ewaluacja
rmse = mean_squared_error(y_test, prediction)
# zapisanie modelu
model.save('avocado_model.h5')
return rmse
@ex.main
def my_main():
print(prepare_model())
ex.run()
ex.add_artifact('avocado_model.h5')