From 97f05db3b9e9bfee208c3f6c1d4e92674bc6912c Mon Sep 17 00:00:00 2001 From: Kacper Date: Fri, 13 May 2022 14:46:06 +0200 Subject: [PATCH] working on lab8 --- lab8/.MLproject.kate-swp | Bin 5415 -> 0 bytes lab8/Jenkinsfile_train | 8 ++--- lab8/MLproject | 15 ++++++++- lab8/simple_regression_lab8.py | 55 ++++++++++----------------------- simple_regression_lab8.py | 53 ++++++++++--------------------- 5 files changed, 50 insertions(+), 81 deletions(-) delete mode 100644 lab8/.MLproject.kate-swp diff --git a/lab8/.MLproject.kate-swp b/lab8/.MLproject.kate-swp deleted file mode 100644 index 0924345eebbce24eec59792a09267886c574234a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5415 zcmb`L&2C&p5QXQ06WfVnJCi^N1TqBjOK|>x1$!e|?64460~&5e7Cy0s_RtuZ8Monrf=^(+Mn%yz4u`D*1^5~ z+3l^HlgZ@t59hu=_~oxZe*W?9!#{sJ`0n>#&;E2{cPA{_hWp!d9^&P&SNsJ0S@Dzb ze(@4KD?S3>D?Yl=e;NL`>?`o+#jEi9+q>_?zclz*A>CICnQf&IXUm1ea8aX&cKu$KO~=F=QbJ-@Z|2W8jNEj4y6-Km4MbV~zk>8?0f73m6v zRgq2=tctulmSJ*KCAkr%lEg)8O7cp}v2rkD@@Rp128YXUEyv-)T8=9Rj(v}abfYyb z=C8Vag}S2EFwN)$GDZnW5FaIM?xq^s8LJi?0H zg$QfiK67A2{z=W(Bgft&haEZc&<=?TlcM@^oE277t{+%YxxV03V<&#Yipu$e6_xV` zPsgb#3fdu2U{VzR;K98nU`62?f)#~TgcXIOh82aQhLxLUKNaut1B(4L>_Y;63U2d1 z4LAR1U{@LapMk9{d=~a0AT40;Az1#tDJ|vi+XeRg^p}4hYOwsb+UfqWFZzEr-gW=a z!5@{sbFlNz_~+qPi!Z=lKlbx*i*E*BulZht+j=j+{=PE)CAjT{m*E!AMY#F91iMNZ ze;ID^UV&RYSK*fbYjBI_75G-gdmX-6{Ay!WG3@cM>c{%Q($o6Q+3~>A*ZRTI)B3^E z)A@j2fn(a<}D3?nqy{hp!aU{c<6*T`I)c#fnwFTPozc#d4_cmMWE6 zljdqn-&3Vhb7^rD^sNU=Dwhr{sa!g9wm(?XxpZJj=hA^CopTM(;>ps7C7nwLmNwP` zmNw2cto*oi;I0j<{IvL_jY|i+^3(h&Ki>M-uf>yFAJ$fE@n|brAMDzSag?Kd^42J4 zVxA1}Ry=t!z&GF)?oHV8VZQ~paBsst39#RRo4?oKkeB-#_SeTyxa80lzHbCQH|;c7 zxJP84mENb_&C(t>>35l>{J-&^{ijRL)xq}-{H?++1en$4AF(h`XMMvh9JYRwfb|V` zA;9{ETM1a-@KA!x!Y9X-F)eK&WAoR9pV02kyjqGUwTF+u-YW1>*moCr8FsC}%1A^= zM6mD9ER5FlnZF{MgcfnW){Ne~bW@I`5lCbB6AM&`rlqwNQQ7m?EH7bU2c$`nd!=+`_Q9Eed4{;%=DhGmA>G#t zneD__A9U2@E?roYyL4fmOdG$ImiQASoxz)h>QZ>P?;IOLCu_M%;NK*vl@88m-r8Oy zKBw`705}5CYFdbtu|?9_mY{gU>hytnRh^x_97~n0m6all6DF2?(@?@~BlJ@P zGrjZFfcvQdtD^naAKdE#=E=YQ_$}#tYQU1trv{v-1`5Ec=(fYA@ow!#+?fpt^6Kb9JNCZ6F=v2JSqeqh&4t?f8{{{4S)u 1 else 100 + units = int(sys.argv[2]) if len(sys.argv) > 2 else 1 + learning_rate = float(sys.argv[3]) if len(sys.argv) > 3 else 0.1 + + # Konfiguracja serwera i nazwy eksperymentu MLflow + # mlflow.set_tracking_uri("http://172.17.0.1:5000") + mlflow.set_experiment('s449288') # Podpięcie treningu do MLflow with mlflow.start_run() as run: @@ -81,17 +66,12 @@ def train(epochs, units, learning_rate, _run): # Zapis predykcji do pliku results = pd.DataFrame( - {'test_set_piece_count': test_piece_counts.tolist(), 'predicted_price': [round(a[0], 2) for a in y_pred.tolist()]}) + {'test_set_piece_count': test_piece_counts.tolist(), + 'predicted_price': [round(a[0], 2) for a in y_pred.tolist()]}) results.to_csv('lego_reg_results.csv', index=False, header=True) - # Zapis modelu do pliku standardowo poprzez metodę kerasa i poprzez metodę obiektu Experiment z Sacred + # Zapis modelu do pliku model.save('lego_reg_model') - ex.add_artifact('lego_reg_model/saved_model.pb') - - # Przykładowo zwracamy loss ostatniej epoki w charakterze wyników, żeby było widoczne w plikach zapisanych przez obserwator - hist = pd.DataFrame(history.history) - hist['epoch'] = history.epoch - _run.log_scalar('final.training.loss', hist['loss'].iloc[-1]) # Ewaluacja MAE na potrzeby MLflow (kopia z evaluate.py) mae = model.evaluate( @@ -109,11 +89,10 @@ def train(epochs, units, learning_rate, _run): tracking_url_type_store = urlparse(mlflow.get_tracking_uri()).scheme if tracking_url_type_store != 'file': mlflow.keras.log_model(model, 'lego-model', registered_model_name='TFLegoModel', - signature=signature) + signature=signature) else: - mlflow.keras.log_model(model, 'model', signature=signature, input_example=500) + mlflow.keras.log_model(model, 'model', signature=signature, input_example=np.array(500)) -@ex.automain -def main(epochs, units, learning_rate): +if __name__ == '__main__': train() diff --git a/simple_regression_lab8.py b/simple_regression_lab8.py index dc881fe..cbba9fa 100644 --- a/simple_regression_lab8.py +++ b/simple_regression_lab8.py @@ -4,37 +4,22 @@ from keras.models import save_model import pandas as pd import numpy as np import matplotlib.pyplot as plt -from sacred import Experiment -from sacred.observers import FileStorageObserver -from sacred.observers import MongoObserver import mlflow import mlflow.keras from urllib.parse import urlparse - -# Konfiguracja serwera i nazwy eksperymentu MLflow -#mlflow.set_tracking_uri("http://172.17.0.1:5000") -mlflow.set_experiment('s449288') - -# Stworzenie obiektu klasy Experiment do śledzenia przebiegu regresji narzędziem Sacred -ex = Experiment(save_git_info=False) - -# Dodanie obserwatora FileObserver -ex.observers.append(FileStorageObserver('runs')) - -#Dodanie obserwatora Mongo -ex.observers.append(MongoObserver(url='mongodb://admin:IUM_2021@172.17.0.1:27017', db_name='sacred')) - -# Przykładowa modyfikowalna z Sacred konfiguracja wybranych parametrów treningu -@ex.config -def config(): - epochs = 100 - units = 1 - learning_rate = 0.1 +import sys -# Reszta kodu wrzucona do udekorowanej funkcji train do wywołania przez Sacred, żeby coś było capture'owane -@ex.capture -def train(epochs, units, learning_rate, _run): +def train(): + + # Definicja wartości parametrów treningu + epochs = int(sys.argv[1]) if len(sys.argv) > 1 else 100 + units = int(sys.argv[2]) if len(sys.argv) > 2 else 1 + learning_rate = float(sys.argv[3]) if len(sys.argv) > 3 else 0.1 + + # Konfiguracja serwera i nazwy eksperymentu MLflow + # mlflow.set_tracking_uri("http://172.17.0.1:5000") + mlflow.set_experiment('s449288') # Podpięcie treningu do MLflow with mlflow.start_run() as run: @@ -81,17 +66,12 @@ def train(epochs, units, learning_rate, _run): # Zapis predykcji do pliku results = pd.DataFrame( - {'test_set_piece_count': test_piece_counts.tolist(), 'predicted_price': [round(a[0], 2) for a in y_pred.tolist()]}) + {'test_set_piece_count': test_piece_counts.tolist(), + 'predicted_price': [round(a[0], 2) for a in y_pred.tolist()]}) results.to_csv('lego_reg_results.csv', index=False, header=True) - # Zapis modelu do pliku standardowo poprzez metodę kerasa i poprzez metodę obiektu Experiment z Sacred + # Zapis modelu do pliku model.save('lego_reg_model') - ex.add_artifact('lego_reg_model/saved_model.pb') - - # Przykładowo zwracamy loss ostatniej epoki w charakterze wyników, żeby było widoczne w plikach zapisanych przez obserwator - hist = pd.DataFrame(history.history) - hist['epoch'] = history.epoch - _run.log_scalar('final.training.loss', hist['loss'].iloc[-1]) # Ewaluacja MAE na potrzeby MLflow (kopia z evaluate.py) mae = model.evaluate( @@ -109,11 +89,10 @@ def train(epochs, units, learning_rate, _run): tracking_url_type_store = urlparse(mlflow.get_tracking_uri()).scheme if tracking_url_type_store != 'file': mlflow.keras.log_model(model, 'lego-model', registered_model_name='TFLegoModel', - signature=signature) + signature=signature) else: mlflow.keras.log_model(model, 'model', signature=signature, input_example=np.array(500)) -@ex.automain -def main(epochs, units, learning_rate): +if __name__ == '__main__': train()