s444356/ium_444356
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update jenkinsfile_train-sacred, dllib-mlflow.py
Some checks failed
s444356-evaluation/pipeline/head This commit looks good
Details
s444356-training/pipeline/head There was a failure building this commit
Details

Browse Source
This commit is contained in:
Maciej Czajka 2022-05-11 15:08:32 +02:00
parent ac87a425b9
commit 531f469e4e
2 changed files with 38 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
Biblioteka_DL/dllib-mlflow.py
View File

@ -1,4 +1,4 @@
import numpy as np 67import numpy as np
import sys import sys
import os import os
import torch import torch
@ -237,6 +237,7 @@ def remove_list(games):
# 'user_review': games['user_review']} # 'user_review': games['user_review']}
# features_g = pd.DataFrame(features_g, dtype=np.float64) # features_g = pd.DataFrame(features_g, dtype=np.float64)
# features_g = features_g.to_numpy() # features_g = features_g.to_numpy()
epochs = int(sys.argv[1]) if len(sys.argv) > 1 else 20
def my_main(epochs): def my_main(epochs):
platform = pd.read_csv('all_games.train.csv', sep=',', usecols=[1], header=None).values.tolist() platform = pd.read_csv('all_games.train.csv', sep=',', usecols=[1], header=None).values.tolist()
@ -290,15 +291,12 @@ def my_main(epochs):
loss_fn = nn.CrossEntropyLoss() loss_fn = nn.CrossEntropyLoss()
# epochs = 1000 # epochs = 1000
# epochs = epochs # epochs = epochs
epochs = int(sys.argv[1]) if len(sys.argv) > 1 else 20
mlflow.log_param("epochs", epochs)
def print_(loss): def print_(loss):
print ("The loss calculated: ", loss) print ("The loss calculated: ", loss)
x_train, y_train = Variable(torch.from_numpy(features_train_g)).float(), Variable(
with mlflow.start_run() as run: torch.from_numpy(labels_train_g)).long()
x_train, y_train = Variable(torch.from_numpy(features_train_g)).float(), Variable(torch.from_numpy(labels_train_g)).long()
for epoch in range(1, epochs + 1): for epoch in range(1, epochs + 1):
print("Epoch #", epoch) print("Epoch #", epoch)
y_pred = model(x_train) y_pred = model(x_train)
@ -318,7 +316,7 @@ def my_main(epochs):
pred = pred.detach().numpy() pred = pred.detach().numpy()
print("The accuracy is", accuracy_score(labels_test_g, np.argmax(pred, axis=1))) print("The accuracy is", accuracy_score(labels_test_g, np.argmax(pred, axis=1)))
mlflow.log_metric("accuracy", accuracy_score(labels_test_g, np.argmax(pred, axis=1))) # mlflow.log_metric("accuracy", accuracy_score(labels_test_g, np.argmax(pred, axis=1)))
pred = pd.DataFrame(pred) pred = pd.DataFrame(pred)
@ -326,5 +324,10 @@ def my_main(epochs):
# save model # save model
torch.save(model, "games_model.pkl") torch.save(model, "games_model.pkl")
return accuracy_score(labels_test_g, np.argmax(pred, axis=1))
with mlflow.start_run() as run:
acc = my_main(epochs)
mlflow.log_param("epochs", epochs)
mlflow.log_metric("accuracy", acc)

33
Jenkinsfile_train-sacred
View File

@ -1,12 +1,12 @@
pipeline { pipeline {
agent { agent {
dockerfile { docker {
additionalBuildArgs "--build-arg KAGGLE_USERNAME=${params.KAGGLE_USERNAME} --build-arg KAGGLE_KEY=${params.KAGGLE_KEY} --build-arg CUTOFF=${params.CUTOFF} -t maciejczajka" image 'maciejczajka'
} }
} }
parameters { parameters {
string( string(
defaultValue: '1000', defaultValue: '100',
description: 'Number of epochs', description: 'Number of epochs',
name: 'EPOCHS', name: 'EPOCHS',
trim: false trim: false
@ -16,28 +16,13 @@ pipeline {
stage('Script'){ stage('Script'){
steps { steps {
copyArtifacts filter: '*', projectName: 's444356-create-dataset' copyArtifacts filter: '*', projectName: 's444356-create-dataset'
sh 'python Biblioteka_DL/dllib-sacred.py with "epochs=$EPOCHS"' sh "python Biblioteka_DL/dllib-mlflow.py -e $EPOCHS"
sh 'ls my_res' archiveArtifacts artifacts: 'games_model.pkl'
sh 'cp -r my_res res' sh 'ls -al'
archiveArtifacts artifacts: 'games_model.pkl, res/**/*.*' sh 'cat MLProject'
sh 'rm -r my_res' sh 'ls mlruns'
sh 'rm -r res' archiveArtifacts artifacts: 'mlruns/**'
build job: 's444356-evaluation/master/'
} }
} }
} }
post {
success {
emailext body: 'SUCCESS', subject: 's444356-training', to: 'e19191c5.uam.onmicrosoft.com@emea.teams.ms'
}
failure {
emailext body: 'FAILURE', subject: 's444356-training', to: 'e19191c5.uam.onmicrosoft.com@emea.teams.ms'
}
unstable {
emailext body: 'UNSTABLE', subject: 's444356-training', to: 'e19191c5.uam.onmicrosoft.com@emea.teams.ms'
}
changed {
emailext body: 'CHANGED', subject: 's444356-training', to: 'e19191c5.uam.onmicrosoft.com@emea.teams.ms'
}
}
} }