This commit is contained in:
parent
c6146b9dfc
commit
8172699654
8
Jenkinsfile
vendored
8
Jenkinsfile
vendored
@ -26,7 +26,7 @@ pipeline {
|
|||||||
{
|
{
|
||||||
steps
|
steps
|
||||||
{
|
{
|
||||||
copyArtifacts(fingerprintArtifacts: true, projectName: 's434788-create-dataset', selector: buildParameter('WHICH_BUILD'))
|
copyArtifacts(fingerprintArtifacts: true, projectName: 's434695-create-dataset', selector: buildParameter('WHICH_BUILD'))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
stage('train')
|
stage('train')
|
||||||
@ -34,14 +34,16 @@ pipeline {
|
|||||||
steps
|
steps
|
||||||
{
|
{
|
||||||
catchError {
|
catchError {
|
||||||
sh 'python3.8 Zadanie_06_and_07_training.py ${BATCH_SIZE} ${EPOCHS}'
|
sh 'python3.8 train.py ${BATCH_SIZE} ${EPOCHS}'
|
||||||
|
sh 'python3.8 sacred1.py ${BATCH_SIZE} ${EPOCHS}'
|
||||||
|
sh 'python3.8 sacred2.py ${BATCH_SIZE} ${EPOCHS}'
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
stage('Archive artifacts') {
|
stage('Archive artifacts') {
|
||||||
steps{
|
steps{
|
||||||
archiveArtifacts 'wine_model.h5'
|
archiveArtifacts 'vgsales_model.h5'
|
||||||
archiveArtifacts 'my_runs/**'
|
archiveArtifacts 'my_runs/**'
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
16
sacred1.py
16
sacred1.py
@ -1,12 +1,16 @@
|
|||||||
#! /usr/bin/python3
|
#! /usr/bin/python3
|
||||||
from tensorflow.keras.models import Sequential, load_model
|
import sys
|
||||||
from tensorflow.keras.layers import Dense
|
|
||||||
from sklearn.metrics import accuracy_score, classification_report
|
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from sklearn.model_selection import train_test_split
|
|
||||||
import wget
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import requests
|
from sklearn import preprocessing
|
||||||
|
from sklearn.linear_model import LinearRegression
|
||||||
|
from sklearn.metrics import mean_squared_error
|
||||||
|
import tensorflow as tf
|
||||||
|
from tensorflow import keras
|
||||||
|
from tensorflow.keras.layers import Input, Dense, Activation,Dropout
|
||||||
|
from tensorflow.keras.models import Model
|
||||||
|
from tensorflow.keras.callbacks import EarlyStopping
|
||||||
|
from tensorflow.keras.models import Sequential
|
||||||
from sacred.observers import FileStorageObserver
|
from sacred.observers import FileStorageObserver
|
||||||
from sacred import Experiment
|
from sacred import Experiment
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
16
sacred2.py
16
sacred2.py
@ -1,12 +1,16 @@
|
|||||||
#! /usr/bin/python3
|
#! /usr/bin/python3
|
||||||
from tensorflow.keras.models import Sequential, load_model
|
import sys
|
||||||
from tensorflow.keras.layers import Dense
|
|
||||||
from sklearn.metrics import accuracy_score, classification_report
|
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from sklearn.model_selection import train_test_split
|
|
||||||
import wget
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import requests
|
from sklearn import preprocessing
|
||||||
|
from sklearn.linear_model import LinearRegression
|
||||||
|
from sklearn.metrics import mean_squared_error
|
||||||
|
import tensorflow as tf
|
||||||
|
from tensorflow import keras
|
||||||
|
from tensorflow.keras.layers import Input, Dense, Activation,Dropout
|
||||||
|
from tensorflow.keras.models import Model
|
||||||
|
from tensorflow.keras.callbacks import EarlyStopping
|
||||||
|
from tensorflow.keras.models import Sequential
|
||||||
from sacred.observers import FileStorageObserver
|
from sacred.observers import FileStorageObserver
|
||||||
from sacred import Experiment
|
from sacred import Experiment
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
78
train.py
78
train.py
@ -10,39 +10,63 @@ from tensorflow.keras.layers import Input, Dense, Activation,Dropout
|
|||||||
from tensorflow.keras.models import Model
|
from tensorflow.keras.models import Model
|
||||||
from tensorflow.keras.callbacks import EarlyStopping
|
from tensorflow.keras.callbacks import EarlyStopping
|
||||||
from tensorflow.keras.models import Sequential
|
from tensorflow.keras.models import Sequential
|
||||||
|
from sacred import Experiment
|
||||||
|
from datetime import datetime
|
||||||
|
from sacred.observers import FileStorageObserver
|
||||||
|
from sacred.observers import MongoObserver
|
||||||
|
import pymongo
|
||||||
|
|
||||||
# odczytanie danych z plików
|
ex = Experiment("434695-mongo", interactive=False, save_git_info=False)
|
||||||
vgsales_train = pd.read_csv('train.csv')
|
ex.observers.append(MongoObserver(url='mongodb://mongo_user:mongo_password_IUM_2021@172.17.0.1:27017', db_name='sacred'))
|
||||||
vgsales_test = pd.read_csv('test.csv')
|
ex.observers.append(FileStorageObserver('my_runs'))
|
||||||
vgsales_dev = pd.read_csv('dev.csv')
|
|
||||||
|
|
||||||
vgsales_train['Nintendo'] = vgsales_train['Publisher'].apply(lambda x: 1 if x=='Nintendo' else 0)
|
|
||||||
vgsales_test['Nintendo'] = vgsales_test['Publisher'].apply(lambda x: 1 if x=='Nintendo' else 0)
|
|
||||||
vgsales_dev['Nintendo'] = vgsales_dev['Publisher'].apply(lambda x: 1 if x=='Nintendo' else 0)
|
|
||||||
|
|
||||||
# podzial na X i y
|
@ex.config
|
||||||
X_train = vgsales_train.drop(['Rank','Name','Platform','Year','Genre','Publisher'],axis = 1)
|
def my_config():
|
||||||
y_train = vgsales_train[['Nintendo']]
|
batch_param = int(sys.argv[1])
|
||||||
X_test = vgsales_test.drop(['Rank','Name','Platform','Year','Genre','Publisher'],axis = 1)
|
epoch_param = int(sys.argv[2])
|
||||||
y_test = vgsales_test[['Nintendo']]
|
|
||||||
|
|
||||||
print(X_train.shape[1])
|
def regression_model(epoch_param, batch_param, _run):
|
||||||
# keras model
|
_run.info["prepare_model_ts"] = str(datetime.now())
|
||||||
model = Sequential()
|
# odczytanie danych z plików
|
||||||
model.add(Dense(9, input_dim = X_train.shape[1], kernel_initializer='normal', activation='relu'))
|
vgsales_train = pd.read_csv('train.csv')
|
||||||
model.add(Dense(1,kernel_initializer='normal', activation='sigmoid'))
|
vgsales_test = pd.read_csv('test.csv')
|
||||||
|
vgsales_dev = pd.read_csv('dev.csv')
|
||||||
|
|
||||||
early_stop = EarlyStopping(monitor="val_loss", mode="min", verbose=1, patience=10)
|
vgsales_train['Nintendo'] = vgsales_train['Publisher'].apply(lambda x: 1 if x=='Nintendo' else 0)
|
||||||
|
vgsales_test['Nintendo'] = vgsales_test['Publisher'].apply(lambda x: 1 if x=='Nintendo' else 0)
|
||||||
|
vgsales_dev['Nintendo'] = vgsales_dev['Publisher'].apply(lambda x: 1 if x=='Nintendo' else 0)
|
||||||
|
|
||||||
# kompilacja
|
# podzial na X i y
|
||||||
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
|
X_train = vgsales_train.drop(['Rank','Name','Platform','Year','Genre','Publisher'],axis = 1)
|
||||||
|
y_train = vgsales_train[['Nintendo']]
|
||||||
|
X_test = vgsales_test.drop(['Rank','Name','Platform','Year','Genre','Publisher'],axis = 1)
|
||||||
|
y_test = vgsales_test[['Nintendo']]
|
||||||
|
|
||||||
# model fit
|
print(X_train.shape[1])
|
||||||
epochs = int(sys.argv[1])
|
# keras model
|
||||||
batch_size = int(sys.argv[2])
|
model = Sequential()
|
||||||
|
model.add(Dense(9, input_dim = X_train.shape[1], kernel_initializer='normal', activation='relu'))
|
||||||
|
model.add(Dense(1,kernel_initializer='normal', activation='sigmoid'))
|
||||||
|
|
||||||
# trenowanie modelu
|
early_stop = EarlyStopping(monitor="val_loss", mode="min", verbose=1, patience=10)
|
||||||
model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, validation_data=(X_test, y_test))
|
|
||||||
|
|
||||||
# zapisanie modelu
|
# kompilacja
|
||||||
model.save('vgsales_model.h5')
|
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
|
||||||
|
|
||||||
|
# model fit
|
||||||
|
epochs = int(sys.argv[1])
|
||||||
|
batch_size = int(sys.argv[2])
|
||||||
|
|
||||||
|
# trenowanie modelu
|
||||||
|
model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, validation_data=(X_test, y_test))
|
||||||
|
|
||||||
|
# zapisanie modelu
|
||||||
|
model.save('vgsales_model.h5')
|
||||||
|
|
||||||
|
@ex.main
|
||||||
|
def my_main(epoch_param, batch_param):
|
||||||
|
print(regression_model())
|
||||||
|
|
||||||
|
r = ex.run()
|
||||||
|
ex.add_artifact("vgsales_model.h5")
|
Loading…
Reference in New Issue
Block a user