Generowanie pliku MLmodel.
This commit is contained in:
parent
dcda151944
commit
9b3d61f406
@ -1,144 +1,43 @@
|
|||||||
import torch
|
import torch
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
import torch.optim as optim
|
import torch.optim as optim
|
||||||
from torch.utils.data import Dataset, TensorDataset, DataLoader
|
from torch.utils.data import Dataset, TensorDataset, DataLoader
|
||||||
import argparse
|
|
||||||
import mlflow
|
import mlflow
|
||||||
import mlflow.pytorch
|
import mlflow.pytorch
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from mlflow.models.signature import infer_signature
|
from mlflow.models.signature import infer_signature
|
||||||
|
|
||||||
|
|
||||||
class LayerLinearRegression(nn.Module):
|
class LayerLinearRegression(nn.Module):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
# Instead of our custom parameters, we use a Linear layer with single input and single output
|
# Instead of our custom parameters, we use a Linear layer with single input and single output
|
||||||
self.linear = nn.Linear(1, 1)
|
self.linear = nn.Linear(1, 1)
|
||||||
|
|
||||||
def forward(self, x):
|
def forward(self, x):
|
||||||
# Now it only takes a call to the layer to make predictions
|
# Now it only takes a call to the layer to make predictions
|
||||||
return self.linear(x)
|
return self.linear(x)
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description='Program do uczenia modelu')
|
checkpoint = torch.load('model.pt')
|
||||||
parser.add_argument('-l', '--lr', type=float, default=1e-3, help="Współczynik uczenia (lr)", required=False)
|
|
||||||
parser.add_argument('-e', '--epochs', type=int, default=100, help="Liczba epok", required=False)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
model = LayerLinearRegression()
|
||||||
lr = args.lr
|
#optimizer = optim.SGD(model.parameters(), lr=checkpoint['loss'])
|
||||||
n_epochs = args.epochs
|
|
||||||
with mlflow.start_run(run_name="s426206"):
|
|
||||||
mlflow.log_param("lr", lr)
|
|
||||||
mlflow.log_param("epochs", n_epochs)
|
|
||||||
|
|
||||||
train_dataset = torch.load('train_dataset.pt')
|
model.load_state_dict(checkpoint['model_state_dict'])
|
||||||
#val_dataset = torch.load('val_dataset.pt')
|
|
||||||
|
|
||||||
train_loader = DataLoader(dataset=train_dataset)
|
train_dataset = torch.load('train_dataset.pt')
|
||||||
#val_loader = DataLoader(dataset=val_dataset)
|
x_train = np.array(train_dataset)[:,0] #(Sales Sum row)
|
||||||
|
input_example = np.reshape(x_train, (-1,1))
|
||||||
model = LayerLinearRegression()
|
|
||||||
# Checks model's parameters
|
|
||||||
#print(model.state_dict())
|
|
||||||
|
|
||||||
loss_fn = nn.MSELoss(reduction='mean')
|
with torch.no_grad():
|
||||||
optimizer = optim.SGD(model.parameters(), lr=lr)
|
model.eval()
|
||||||
some_x = 0
|
siganture = infer_signature(x_train, model(torch.tensor(np.reshape(x_train, (-1,1))).float()).numpy())
|
||||||
def make_train_step(model, loss_fn, optimizer):
|
mlflow.set_tracking_uri("http://172.17.0.1:5000")
|
||||||
# Builds function that performs a step in the train loop
|
|
||||||
def train_step(x, y):
|
|
||||||
# Sets model to TRAIN mode
|
|
||||||
model.train()
|
|
||||||
# Makes predictions
|
|
||||||
yhat = model(x)
|
|
||||||
# Computes loss
|
|
||||||
loss = loss_fn(y, yhat)
|
|
||||||
# Computes gradients
|
|
||||||
loss.backward()
|
|
||||||
# Updates parameters and zeroes gradients
|
|
||||||
optimizer.step()
|
|
||||||
optimizer.zero_grad()
|
|
||||||
# Returns the loss
|
|
||||||
return loss.item()
|
|
||||||
|
|
||||||
# Returns the function that will be called inside the train loop
|
|
||||||
return train_step
|
|
||||||
|
|
||||||
# Creates the train_step function for our model, loss function and optimizer
|
|
||||||
train_step = make_train_step(model, loss_fn, optimizer)
|
|
||||||
training_losses = []
|
|
||||||
validation_losses = []
|
|
||||||
#print(model.state_dict())
|
|
||||||
# For each epoch...
|
|
||||||
for epoch in range(n_epochs):
|
|
||||||
|
|
||||||
losses = []
|
|
||||||
# Uses loader to fetch one mini-batch for training
|
|
||||||
for x_batch, y_batch in train_loader:
|
|
||||||
# NOW, sends the mini-batch data to the device
|
|
||||||
# so it matches location of the MODEL
|
|
||||||
# x_batch = x_batch.to(device)
|
|
||||||
# y_batch = y_batch.to(device)
|
|
||||||
# One stpe of training
|
|
||||||
|
|
||||||
loss = train_step(x_batch, y_batch)
|
|
||||||
losses.append(loss)
|
|
||||||
training_loss = np.mean(losses)
|
|
||||||
training_losses.append(training_loss)
|
|
||||||
|
|
||||||
mlflow.log_metric("MSE", training_loss)
|
|
||||||
|
|
||||||
# After finishing training steps for all mini-batches,
|
|
||||||
# it is time for evaluation!
|
|
||||||
# Ewaluacja jest już tutaj nie potrzebna bo odbywa sie w evaluation.py. Można jednak włączyć podgląd ewaluacji dla poszczególnych epok.
|
|
||||||
# # We tell PyTorch to NOT use autograd...
|
|
||||||
# # Do you remember why?
|
|
||||||
# with torch.no_grad():
|
|
||||||
# val_losses = []
|
|
||||||
# # Uses loader to fetch one mini-batch for validation
|
|
||||||
# for x_val, y_val in val_loader:
|
|
||||||
# # Again, sends data to same device as model
|
|
||||||
# # x_val = x_val.to(device)
|
|
||||||
# # y_val = y_val.to(device)
|
|
||||||
|
|
||||||
# model.eval()
|
|
||||||
# # Makes predictions
|
|
||||||
# yhat = model(x_val)
|
|
||||||
# # Computes validation loss
|
|
||||||
# val_loss = loss_fn(y_val, yhat)
|
|
||||||
# val_losses.append(val_loss.item())
|
|
||||||
# validation_loss = np.mean(val_losses)
|
|
||||||
# validation_losses.append(validation_loss)
|
|
||||||
|
|
||||||
# print(f"[{epoch+1}] Training loss: {training_loss:.3f}\t Validation loss: {validation_loss:.3f}")
|
|
||||||
print(f"[{epoch+1}] Training loss: {training_loss:.3f}\t")
|
|
||||||
|
|
||||||
torch.save({
|
|
||||||
'model_state_dict': model.state_dict(),
|
|
||||||
'optimizer_state_dict': optimizer.state_dict(),
|
|
||||||
'loss': lr,
|
|
||||||
}, 'model.pt')
|
|
||||||
|
|
||||||
mlflow.pytorch.log_state_dict(model.state_dict(), artifact_path="model")
|
|
||||||
|
|
||||||
x_train = np.array(train_dataset)[:,0] #(Sales Sum row)
|
|
||||||
input_example = np.reshape(x_train, (-1,1))
|
|
||||||
with torch.no_grad():
|
|
||||||
#print(model(torch.tensor(np.reshape(x_train, (-1,1))).float()))
|
|
||||||
siganture = infer_signature(x_train, model(torch.tensor(np.reshape(x_train, (-1,1))).float()).numpy())
|
|
||||||
#mlflow.set_tracking_uri("http://172.17.0.1:5000")
|
|
||||||
tracking_url_type_store = urlparse(mlflow.get_tracking_uri()).scheme
|
tracking_url_type_store = urlparse(mlflow.get_tracking_uri()).scheme
|
||||||
print(tracking_url_type_store)
|
# print(tracking_url_type_store)
|
||||||
# Model registry does not work with file store
|
# Model registry does not work with file store
|
||||||
if tracking_url_type_store != "file":
|
if tracking_url_type_store != "file":
|
||||||
|
mlflow.sklearn.log_model(model, "model", registered_model_name="s426206", signature=siganture, input_example=input_example)
|
||||||
# Register the model
|
|
||||||
# There are other ways to use the Model Registry, which depends on the use case,
|
|
||||||
# please refer to the doc for more information:
|
|
||||||
# https://mlflow.org/docs/latest/model-registry.html#api-workflow
|
|
||||||
mlflow.sklearn.log_model(model, "model", registered_model_name="s426206")
|
|
||||||
else:
|
else:
|
||||||
mlflow.sklearn.log_model(model, "model", signature=siganture, input_example=input_example)
|
mlflow.sklearn.log_model(model, "model", signature=siganture, input_example=input_example)
|
Loading…
Reference in New Issue
Block a user