s487187/ium_487187
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

s

Browse Source
This commit is contained in:
Jakub Zaręba 2023-05-11 00:11:04 +02:00
parent 162dcf95ab
commit bc646e2982
2 changed files with 36 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
JenkinsfileDL
View File

@ -5,16 +5,13 @@ pipeline {
args '-v /root/.cache:/root/.cache -u root'
}
}
environment {
SACRED_IGNORE_GIT = 'TRUE'
}
parameters {
string(name: 'EPOCHS', defaultValue: '10', description: 'Liczba Epok')
}
stages {
stage('Przygotowania') {
stage('Preparation') {
steps {
sh 'pip install pandas tensorflow scikit-learn imbalanced-learn sacred pymongo mlflow'
sh 'pip install pandas tensorflow scikit-learn imbalanced-learn sacred pymongo'
}
}
stage('Pobierz dane') {
@ -27,17 +24,14 @@ pipeline {
stage('Trenuj model') {
steps {
script {
sh 'mlflow run . -P epochs=$EPOCHS'
// sh "python3 train.py --epochs $EPOCHS"
sh "python3 train.py"
}
}
}
stage('Zarchiwizuj model') {
steps {
sh '''
mkdir -p model
cp -r mlruns/* model/
'''
archiveArtifacts artifacts: 'model/**', fingerprint: true
archiveArtifacts artifacts: 'model.h5', fingerprint: true
}
}
}

67
train.py
View File

@ -1,26 +1,22 @@
from sacred import Experiment
from sacred.observers import MongoObserver
from sacred.observers import MongoObserver, FileStorageObserver
import os
import mlflow
import mlflow.keras
from mlflow.models.signature import infer_signature
from mlflow.models import Model
os.environ["SACRED_NO_GIT"] = "1"
ex = Experiment('s487187-training', interactive=True, save_git_info=False)
ex.observers.append(MongoObserver(url='mongodb://admin:IUM_2021@172.17.0.1:27017', db_name='sacred'))
@ex.config
def my_config():
data_file = 'data.csv'
model_file = 'model'
model_file = 'model.h5'
epochs = 10
batch_size = 32
test_size = 0.2
random_state = 42
@ex.capture
def train_model(data_file, model_file, epochs, batch_size, test_size, random_state):
import pandas as pd
@ -29,48 +25,45 @@ def train_model(data_file, model_file, epochs, batch_size, test_size, random_sta
import tensorflow as tf
from imblearn.over_sampling import SMOTE
with mlflow.start_run():
smote = SMOTE(random_state=random_state)
data = pd.read_csv(data_file, sep=';')
smote = SMOTE(random_state=random_state)
data = pd.read_csv(data_file, sep=';')
print('Total rows:', len(data))
print('Rows with medal:', len(data.dropna(subset=['Medal'])))
data = pd.get_dummies(data, columns=['Sex', 'Medal'])
data = data.drop(columns=['Name', 'Team', 'NOC', 'Games', 'Year', 'Season', 'City', 'Sport', 'Event'])
data = pd.get_dummies(data, columns=['Sex', 'Medal'])
data = data.drop(columns=['Name', 'Team', 'NOC', 'Games', 'Year', 'Season', 'City', 'Sport', 'Event'])
scaler = MinMaxScaler()
data = pd.DataFrame(scaler.fit_transform(data), columns=data.columns)
scaler = MinMaxScaler()
data = pd.DataFrame(scaler.fit_transform(data), columns=data.columns)
X = data.filter(regex='Sex|Age')
y = data.filter(regex='Medal')
y = pd.get_dummies(y)
X = data.filter(regex='Sex|Age')
y = data.filter(regex='Medal')
y = pd.get_dummies(y)
X = X.fillna(0)
y = y.fillna(0)
X = X.fillna(0)
y = y.fillna(0)
y = y.values
y = y.values
X_resampled, y_resampled = smote.fit_resample(X, y)
X_train, X_test, y_train, y_test = train_test_split(X_resampled, y_resampled, test_size=test_size, random_state=random_state)
X_resampled, y_resampled = smote.fit_resample(X, y)
X_train, X_test, y_train, y_test = train_test_split(X_resampled, y_resampled, test_size=test_size, random_state=random_state)
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Dense(64, input_dim=X_train.shape[1], activation='relu'))
model.add(tf.keras.layers.Dense(32, activation='relu'))
model.add(tf.keras.layers.Dense(y.shape[1], activation='softmax'))
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Dense(64, input_dim=X_train.shape[1], activation='relu'))
model.add(tf.keras.layers.Dense(32, activation='relu'))
model.add(tf.keras.layers.Dense(y.shape[1], activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size)
loss, accuracy = model.evaluate(X_test, y_test)
model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size)
loss, accuracy = model.evaluate(X_test, y_test)
print('Test accuracy:', accuracy)
print('Test loss:', loss)
mlflow.log_metric("loss", loss)
mlflow.log_metric("accuracy", accuracy)
model.save(model_file)
signature = infer_signature(X_train, model.predict(X_train))
input_example = Model.log_input_example(X_train.iloc[0])
mlflow.keras.log_model(model, model_file, signature=signature, input_example=input_example)
return accuracy
return accuracy
@ex.main
def run_experiment():