From bdf10f8606485431063feaf651e7f1f6b987f5fc Mon Sep 17 00:00:00 2001 From: Adrian Charkiewicz Date: Sun, 15 May 2022 19:20:14 +0200 Subject: [PATCH] lab8 --- Jenkinsfile_artifact | 23 +++++++++++++++++++++++ Jenkinsfile_mlflow | 15 +++++++++++++++ MLproject | 13 +++++++++++++ predictArtifact.py | 14 ++++++++++++++ predictMlflow.py | 14 ++++++++++++++ pytorch/pytorch.py | 41 +++++++++++++++++++++++++++++++---------- 6 files changed, 110 insertions(+), 10 deletions(-) create mode 100644 Jenkinsfile_artifact create mode 100644 Jenkinsfile_mlflow create mode 100644 MLproject create mode 100644 predictArtifact.py create mode 100644 predictMlflow.py diff --git a/Jenkinsfile_artifact b/Jenkinsfile_artifact new file mode 100644 index 0000000..52a317e --- /dev/null +++ b/Jenkinsfile_artifact @@ -0,0 +1,23 @@ +pipeline { + agent { + dockerfile { + args '-v /mlruns:/mlruns' + } + } + parameters { + buildSelector( + defaultSelector: lastSuccessful(), + description: 'select build from which to copy artifacts', + name: 'BUILD_SELECTOR' + ) + } + stages { + stage('Copy artifacts') { + steps { + copyArtifacts filter: 'mlruns.tar.gz', projectName: 's449288-training/master', selector: buildParameter('BUILD_SELECTOR') + sh 'mkdir -p mlrunsArtifact && tar xzf mlruns.tar.gz -C mlrunsArtifact --strip-components 1' + sh "python ./predictArtifact.py" + } + } + } +} \ No newline at end of file diff --git a/Jenkinsfile_mlflow b/Jenkinsfile_mlflow new file mode 100644 index 0000000..478b5ec --- /dev/null +++ b/Jenkinsfile_mlflow @@ -0,0 +1,15 @@ +pipeline { + agent { + dockerfile { + args '-v /mlruns:/mlruns' + } + } + stages { + stage('Prediction') { + steps { + sh 'ls -la' + sh "python ./predictMlflow.py" + } + } + } +} \ No newline at end of file diff --git a/MLproject b/MLproject new file mode 100644 index 0000000..66cfaa1 --- /dev/null +++ b/MLproject @@ -0,0 +1,13 @@ +name: tutorial +# conda_env: conda.yaml + +docker_env: + image: ksero/ium:mlflow + +entry_points: + main: + parameters: + epochs: {type: float, default: 1500} + command: "python ./pytorch/pytorch.py {epochs}" + test: + command: "python ./evaluation.py" \ No newline at end of file diff --git a/predictArtifact.py b/predictArtifact.py new file mode 100644 index 0000000..5556299 --- /dev/null +++ b/predictArtifact.py @@ -0,0 +1,14 @@ +import mlflow +import numpy as np +import json + +logged_model = 'mlrunsArtifact/1/6b2323cf51794581bf1e2f6d060d50f6/artifacts/model' +loaded_model = mlflow.pyfunc.load_model(logged_model) + +with open(f'{logged_model}/input_example.json') as f: + input_example_data = json.load(f) + +input_example = np.array(input_example_data['inputs']).reshape(-1,) + +print(f'Input: {input_example}') +print(f'Prediction: {loaded_model.predict(input_example)}') \ No newline at end of file diff --git a/predictMlflow.py b/predictMlflow.py new file mode 100644 index 0000000..aa2061b --- /dev/null +++ b/predictMlflow.py @@ -0,0 +1,14 @@ +import mlflow +import numpy as np +import json + +logged_model = '/mlruns/20/80fe21a0804844088147d15a3cebb3e5/artifacts/lego' +loaded_model = mlflow.pyfunc.load_model(logged_model) + +with open(f'{logged_model}/input_example.json') as f: + input_example_data = json.load(f) + +input_example = np.array(input_example_data['inputs']).reshape(-1,) + +print(f'Input: {input_example}') +print(f'Prediction: {loaded_model.predict(input_example)}') \ No newline at end of file diff --git a/pytorch/pytorch.py b/pytorch/pytorch.py index 491d7f2..b55c000 100644 --- a/pytorch/pytorch.py +++ b/pytorch/pytorch.py @@ -21,6 +21,8 @@ import sys from sacred import Experiment from sacred.observers import FileStorageObserver from sacred.observers import MongoObserver +import mlflow +import mlflow.keras # In[2]: ex = Experiment(save_git_info=False) @@ -30,7 +32,8 @@ ex.observers.append(FileStorageObserver('my_runs')) # ex.observers.append(MongoObserver(url='mongodb://mongo_user:mongo_password_IUM_2021@localhost:27017', db_name='sacred')) ex.observers.append(MongoObserver(url='mongodb://admin:IUM_2021@172.17.0.1:27017', db_name='sacred')) - +mlflow.set_tracking_uri("http://172.17.0.1:5000") +mlflow.set_experiment('s444354') try: numberOfEpochParam = int(sys.argv[1]) @@ -162,21 +165,39 @@ def fit(epochs, lr, model, train_loader, val_loader, _log, _run, opt_func=torch. epochs=epochs history = [] optimizer = opt_func(model.parameters(), lr) - for epoch in range(epochs): - for batch in train_loader: - loss = model.training_step(batch) - loss.backward() - optimizer.step() - optimizer.zero_grad() - result = evaluate(model, val_loader) - model.epoch_end(epoch, result, epochs) - history.append(result) + with mlflow.start_run(): + + for epoch in range(epochs): + for batch in train_loader: + loss = model.training_step(batch) + loss.backward() + optimizer.step() + optimizer.zero_grad() + result = evaluate(model, val_loader) + model.epoch_end(epoch, result, epochs) + history.append(result) + + mlflow.log_param('epochs', epochs) + mlflow.log_param('lr', lr) torch.save(model, 'saved_model.pth') ex.add_artifact("saved_model.pth") _run.info["epochs"] = epochs + signature = mlflow.models.signature.infer_signature(house_price_features, linear_model.predict(house_price_features)) + + tracking_url_type_store = urlparse(mlflow.get_tracking_uri()).scheme + + sampleInp = [0.1, 0.1, 546.0, 555.223, 1., 1., 33.16376, 84.12426] + + if tracking_url_type_store != "file": + mlflow.keras.log_model(model, "model", registered_model_name="red-wine-quality", signature=signature) + else: + mlflow.keras.log_model(model, "model", signature=signature, input_example=np.array(sampleInp)) + + + return history