From b82917a5d90e071de9b67f2b6648be0353c54b62 Mon Sep 17 00:00:00 2001 From: Wojciech Jarmosz Date: Sat, 15 May 2021 18:46:56 +0200 Subject: [PATCH] Adding mlflow --- MLproject | 10 +++ mlflow_exp.py | 65 +++++++++++++++++++ .../meta.yaml | 15 +++++ .../params/epochs | 1 + .../params/verbose | 1 + .../tags/mlflow.gitRepoURL | 1 + .../tags/mlflow.project.entryPoint | 1 + .../tags/mlflow.project.env | 1 + .../tags/mlflow.source.git.commit | 1 + .../tags/mlflow.source.git.repoURL | 1 + .../tags/mlflow.source.name | 1 + .../tags/mlflow.source.type | 1 + .../tags/mlflow.user | 1 + mlruns/0/meta.yaml | 4 ++ 14 files changed, 104 insertions(+) create mode 100644 MLproject create mode 100644 mlflow_exp.py create mode 100644 mlruns/0/023db6388f714cdbb05c20e41895edab/meta.yaml create mode 100644 mlruns/0/023db6388f714cdbb05c20e41895edab/params/epochs create mode 100644 mlruns/0/023db6388f714cdbb05c20e41895edab/params/verbose create mode 100644 mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.gitRepoURL create mode 100644 mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.project.entryPoint create mode 100644 mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.project.env create mode 100644 mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.source.git.commit create mode 100644 mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.source.git.repoURL create mode 100644 mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.source.name create mode 100644 mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.source.type create mode 100644 mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.user create mode 100644 mlruns/0/meta.yaml diff --git a/MLproject b/MLproject new file mode 100644 index 0000000..a040786 --- /dev/null +++ b/MLproject @@ -0,0 +1,10 @@ +name: 434704 + +docker_env: + image: jarmosz/ium +entry_points: + main: + parameters: + verbose: {type: int, default: 100} + epochs: {type: int, default: 0} + command: "python3 mlflow_exp.py {epochs} {verbose}" \ No newline at end of file diff --git a/mlflow_exp.py b/mlflow_exp.py new file mode 100644 index 0000000..f68c0eb --- /dev/null +++ b/mlflow_exp.py @@ -0,0 +1,65 @@ +import sys + +import pandas as pd +import numpy as np +import tensorflow as tf +import os.path + +import mlflow + +from tensorflow import keras +from tensorflow.keras import layers +from tensorflow.keras.layers.experimental import preprocessing + + +with mlflow.start_run(): + + arguments = sys.argv[1:] + + verbose = int(arguments[0]) + epochs = int(arguments[1]) + + mlflow.log_param("verbose", verbose) + mlflow.log_param("epochs", epochs) + pd.set_option("display.max_columns", None) + + # Wczytanie danych + train_data = pd.read_csv("./MoviesOnStreamingPlatforms_updated.train") + + # Stworzenie modelu + columns_to_use = ['Year', 'Runtime', 'Netflix'] + train_X = tf.convert_to_tensor(train_data[columns_to_use]) + train_Y = tf.convert_to_tensor(train_data[["IMDb"]]) + + normalizer = preprocessing.Normalization(input_shape=[3,]) + normalizer.adapt(train_X) + + model = keras.Sequential([ + keras.Input(shape=(len(columns_to_use),)), + normalizer, + layers.Dense(30, activation='relu'), + layers.Dense(10, activation='relu'), + layers.Dense(25, activation='relu'), + layers.Dense(1) + ]) + + model.compile(loss='mean_absolute_error', + optimizer=tf.keras.optimizers.Adam(0.001), + metrics=[tf.keras.metrics.RootMeanSquaredError()]) + + model.fit(train_X, train_Y, verbose=verbose, epochs=epochs) + + model.save('linear_regression.h5') + + # Evaluation + + test_data = pd.read_csv("./MoviesOnStreamingPlatforms_updated.test") + + columns_to_use = ['Year', 'Runtime', 'Netflix'] + test_X = tf.convert_to_tensor(test_data[columns_to_use]) + test_Y = tf.convert_to_tensor(test_data[["IMDb"]]) + + scores = model.evaluate(x=test_X, + y=test_Y) + + mlflow.log_metric("RMSE", scores[1]) \ No newline at end of file diff --git a/mlruns/0/023db6388f714cdbb05c20e41895edab/meta.yaml b/mlruns/0/023db6388f714cdbb05c20e41895edab/meta.yaml new file mode 100644 index 0000000..b186e3f --- /dev/null +++ b/mlruns/0/023db6388f714cdbb05c20e41895edab/meta.yaml @@ -0,0 +1,15 @@ +artifact_uri: file:///Volumes/seagate/ium_434704/mlruns/0/023db6388f714cdbb05c20e41895edab/artifacts +end_time: null +entry_point_name: '' +experiment_id: '0' +lifecycle_stage: active +name: '' +run_id: 023db6388f714cdbb05c20e41895edab +run_uuid: 023db6388f714cdbb05c20e41895edab +source_name: '' +source_type: 4 +source_version: '' +start_time: 1621096782764 +status: 1 +tags: [] +user_id: wj diff --git a/mlruns/0/023db6388f714cdbb05c20e41895edab/params/epochs b/mlruns/0/023db6388f714cdbb05c20e41895edab/params/epochs new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/mlruns/0/023db6388f714cdbb05c20e41895edab/params/epochs @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/mlruns/0/023db6388f714cdbb05c20e41895edab/params/verbose b/mlruns/0/023db6388f714cdbb05c20e41895edab/params/verbose new file mode 100644 index 0000000..105d7d9 --- /dev/null +++ b/mlruns/0/023db6388f714cdbb05c20e41895edab/params/verbose @@ -0,0 +1 @@ +100 \ No newline at end of file diff --git a/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.gitRepoURL b/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.gitRepoURL new file mode 100644 index 0000000..7aa7a81 --- /dev/null +++ b/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.gitRepoURL @@ -0,0 +1 @@ +https://git.wmi.amu.edu.pl/s434704/ium_434704.git \ No newline at end of file diff --git a/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.project.entryPoint b/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.project.entryPoint new file mode 100644 index 0000000..88d050b --- /dev/null +++ b/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.project.entryPoint @@ -0,0 +1 @@ +main \ No newline at end of file diff --git a/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.project.env b/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.project.env new file mode 100644 index 0000000..6d0eac4 --- /dev/null +++ b/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.project.env @@ -0,0 +1 @@ +docker \ No newline at end of file diff --git a/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.source.git.commit b/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.source.git.commit new file mode 100644 index 0000000..567c82f --- /dev/null +++ b/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.source.git.commit @@ -0,0 +1 @@ +340888294d4cc3b500347513e4793df41b1db84d \ No newline at end of file diff --git a/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.source.git.repoURL b/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.source.git.repoURL new file mode 100644 index 0000000..7aa7a81 --- /dev/null +++ b/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.source.git.repoURL @@ -0,0 +1 @@ +https://git.wmi.amu.edu.pl/s434704/ium_434704.git \ No newline at end of file diff --git a/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.source.name b/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.source.name new file mode 100644 index 0000000..51044cf --- /dev/null +++ b/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.source.name @@ -0,0 +1 @@ +file:///Volumes/seagate/ium_434704 \ No newline at end of file diff --git a/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.source.type b/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.source.type new file mode 100644 index 0000000..53f9167 --- /dev/null +++ b/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.source.type @@ -0,0 +1 @@ +PROJECT \ No newline at end of file diff --git a/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.user b/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.user new file mode 100644 index 0000000..1fa4079 --- /dev/null +++ b/mlruns/0/023db6388f714cdbb05c20e41895edab/tags/mlflow.user @@ -0,0 +1 @@ +wj \ No newline at end of file diff --git a/mlruns/0/meta.yaml b/mlruns/0/meta.yaml new file mode 100644 index 0000000..cdbc386 --- /dev/null +++ b/mlruns/0/meta.yaml @@ -0,0 +1,4 @@ +artifact_location: file:///Volumes/seagate/ium_434704/mlruns/0 +experiment_id: '0' +lifecycle_stage: active +name: Default