s444501/ium_444501
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

sacred

Browse Source
This commit is contained in:
s444501 2022-05-02 17:39:03 +02:00
parent 3c4da67f2d
commit a428cad996
2 changed files with 91 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
biblioteki_ml.py
View File

@ -2,16 +2,10 @@ import sys
import torch import torch
import torch.nn as nn import torch.nn as nn
import torch.nn.functional as F import torch.nn.functional as F
from sacred.observers import FileStorageObserver, MongoObserver
from sklearn.preprocessing import LabelEncoder from sklearn.preprocessing import LabelEncoder
import pandas as pd import pandas as pd
from sacred import Experiment
# Parametry z konsoli
try:
epochs = int(sys.argv[1])
except:
print('No epoch number passed. Defaulting to 100')
epochs = 100
# Model # Model
@ -29,6 +23,32 @@ class Model(nn.Module):
return x return x
# Sacred
ex = Experiment()
ex.observers.append(FileStorageObserver('my_runs'))
# Parametry treningu -> my_runs/X/config.json
# Plik z modelem jako artefakt -> my_runs/X/model.pkl
# Kod źródłowy -> my_runs/_sources/biblioteki_ml_XXXXXXXXXXX.py
# Wyniki (ostateczny loss) -> my_runs/X/metrics.json
ex.observers.append(MongoObserver(url='mongodb://mongo_user:mongo_password_IUM_2021@localhost:27017',
db_name='sacred'))
@ex.config
def my_config():
epochs = 100
@ex.automain
def train_main(epochs, _run):
# Parametry z konsoli
# try:
# epochs = int(sys.argv[1])
# except:
# print('No epoch number passed. Defaulting to 100')
# epochs = 100
# Ładowanie danych # Ładowanie danych
train_set = pd.read_csv('d_train.csv', encoding='latin-1') train_set = pd.read_csv('d_train.csv', encoding='latin-1')
train_set = train_set[['Rating', 'Branch', 'Reviewer_Location']] train_set = train_set[['Rating', 'Branch', 'Reviewer_Location']]
@ -83,6 +103,7 @@ for i in range(epochs):
optimizer.zero_grad() optimizer.zero_grad()
loss.backward() loss.backward()
optimizer.step() optimizer.step()
_run.log_scalar("training.final_loss", losses[-1].item()) # Ostateczny loss
# Testy # Testy
@ -100,3 +121,7 @@ print(f"{df['Correct'].sum() / len(df)} percent of predictions correct")
# Zapis do pliku # Zapis do pliku
df.to_csv('neural_network_prediction_results.csv', index=False) df.to_csv('neural_network_prediction_results.csv', index=False)
torch.save(model, "model.pkl") torch.save(model, "model.pkl")
# Zapis Sacred
ex.add_artifact("model.pkl")
ex.add_artifact("neural_network_prediction_results.csv")

6
training.Jenkinsfile
View File

@ -24,13 +24,13 @@ pipeline {
stage('Train model') { stage('Train model') {
steps { steps {
withEnv(["EPOCH=${params.EPOCH}"]) { withEnv(["EPOCH=${params.EPOCH}"]) {
sh 'python biblioteki_ml.py $EPOCH' sh 'python biblioteki_ml.py with "epochs=$EPOCH"'
} }
} }
} }
stage('Archive model') { stage('Archive artifacts') {
steps { steps {
archiveArtifacts artifacts: 'model.pkl, neural_network_prediction_results.csv' archiveArtifacts artifacts: 'model.pkl, neural_network_prediction_results.csv, my_run'
} }
} }
stage ('Model - evaluation') { stage ('Model - evaluation') {