s487187/ium_487187
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

f

Browse Source
This commit is contained in:
Jakub Zaręba 2023-05-10 22:49:14 +02:00
parent ba6e518100
commit f562d1b0a4
3 changed files with 41 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Dockerfile
View File

@ -2,7 +2,7 @@ FROM ubuntu
RUN apt-get update && apt-get install -y python3 python3-pip unzip RUN apt-get update && apt-get install -y python3 python3-pip unzip
RUN python3 -m pip install pandas numpy tensorflow imbalanced-learn sklearn sacred pymongo RUN python3 -m pip install pandas numpy tensorflow imbalanced-learn sklearn sacred pymongo mlflow
RUN apt-get install -y git RUN apt-get install -y git
COPY train.py /app/train.py COPY train.py /app/train.py
@ -12,6 +12,6 @@ COPY data.csv /app/data.csv
WORKDIR /app WORKDIR /app
RUN export SACRED_IGNORE_GIT=TRUE RUN export SACRED_IGNORE_GIT=TRUE
RUN python3 train.py RUN mlflow run . -P epochs=10
CMD ["python3", "predictions.py"] CMD ["python3", "predictions.py"]

10
JenkinsfileDL
View File

@ -14,7 +14,7 @@ pipeline {
stages { stages {
stage('Preparation') { stage('Preparation') {
steps { steps {
sh 'pip install pandas tensorflow scikit-learn imbalanced-learn sacred pymongo' sh 'pip install pandas tensorflow scikit-learn imbalanced-learn sacred pymongo mlflow'
} }
} }
stage('Pobierz dane') { stage('Pobierz dane') {
@ -27,13 +27,17 @@ pipeline {
stage('Trenuj model') { stage('Trenuj model') {
steps { steps {
script { script {
sh "python3 train.py" //--epochs $EPOCHS sh 'mlflow run . -P epochs=$EPOCHS'
} }
} }
} }
stage('Zarchiwizuj model') { stage('Zarchiwizuj model') {
steps { steps {
archiveArtifacts artifacts: 'model.h5', fingerprint: true sh '''
mkdir -p model
cp -r mlruns/* model/
'''
archiveArtifacts artifacts: 'model/**', fingerprint: true
} }
} }
} }

15
train.py
View File

@ -1,13 +1,14 @@
from sacred import Experiment from sacred import Experiment
from sacred.observers import MongoObserver, FileStorageObserver from sacred.observers import MongoObserver, FileStorageObserver
import os import os
import mlflow
import mlflow.keras
os.environ["SACRED_NO_GIT"] = "1" os.environ["SACRED_NO_GIT"] = "1"
ex = Experiment('s487187-training', interactive=True, save_git_info=False) ex = Experiment('s487187-training', interactive=True, save_git_info=False)
ex.observers.append(MongoObserver(url='mongodb://admin:IUM_2021@172.17.0.1:27017', db_name='sacred')) ex.observers.append(MongoObserver(url='mongodb://admin:IUM_2021@172.17.0.1:27017', db_name='sacred'))
@ex.config @ex.config
def my_config(): def my_config():
data_file = 'data.csv' data_file = 'data.csv'
@ -25,12 +26,11 @@ def train_model(data_file, model_file, epochs, batch_size, test_size, random_sta
import tensorflow as tf import tensorflow as tf
from imblearn.over_sampling import SMOTE from imblearn.over_sampling import SMOTE
with mlflow.start_run():
smote = SMOTE(random_state=random_state) smote = SMOTE(random_state=random_state)
data = pd.read_csv(data_file, sep=';') data = pd.read_csv(data_file, sep=';')
print('Total rows:', len(data))
print('Rows with medal:', len(data.dropna(subset=['Medal'])))
data = pd.get_dummies(data, columns=['Sex', 'Medal']) data = pd.get_dummies(data, columns=['Sex', 'Medal'])
data = data.drop(columns=['Name', 'Team', 'NOC', 'Games', 'Year', 'Season', 'City', 'Sport', 'Event']) data = data.drop(columns=['Name', 'Team', 'NOC', 'Games', 'Year', 'Season', 'City', 'Sport', 'Event'])
@ -58,10 +58,11 @@ def train_model(data_file, model_file, epochs, batch_size, test_size, random_sta
model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size) model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size)
loss, accuracy = model.evaluate(X_test, y_test) loss, accuracy = model.evaluate(X_test, y_test)
print('Test accuracy:', accuracy)
print('Test loss:', loss)
model.save(model_file) mlflow.log_metric("loss", loss)
mlflow.log_metric("accuracy", accuracy)
mlflow.keras.save_model(model, model_file)
return accuracy return accuracy