Jenkins-2 Zadanie 1.

.gitignore

@@ -0,0 +1,3 @@
+venv
+.ipynb_checkpoints
+.vscode

Dockerfile

@@ -0,0 +1,27 @@
+# Nasz obraz będzie dzidziczył z obrazu Ubuntu w wersji latest
+FROM ubuntu:focal
+
+# Instalujemy niezbędne zależności. Zwróć uwagę na flagę "-y" (assume yes)
+RUN apt update
+RUN apt install -y python3 python3-pip dos2unix git
+RUN pip3 install kaggle
+RUN apt install -y unzip
+RUN mkdir /.kaggle
+RUN chmod -R 777 /.kaggle
+#RUN export KAGGLE_CONFIG_DIR=~/.kaggle
+COPY ./requirments.txt ./
+RUN pip3 install -r requirments.txt
+RUN pip3 install torch==1.8.1+cpu torchvision==0.9.1+cpu torchaudio==0.8.1 -f https://download.pytorch.org/whl/torch_stable.html
+
+# Stwórzmy w kontenerze (jeśli nie istnieje) katalog /app i przejdźmy do niego (wszystkie kolejne polecenia RUN, CMD, ENTRYPOINT, COPY i ADD będą w nim wykonywane)
+WORKDIR /app
+
+# Skopiujmy nasz skrypt do katalogu /app w kontenerze
+# COPY ./skrypt.sh ./
+# RUN chmod +x skrypt.sh
+# RUN dos2unix skrypt.sh
+
+COPY ./dlgssdpytorch.py ./
+RUN chmod +x dlgssdpytorch.py
+COPY ./create_dataset.py ./
+RUN chmod +x create_dataset.py

Jenkins_train

@@ -0,0 +1,29 @@
+pipeline {
+	agent any 
+	stages {
+		stage('checkout') {
+			steps {
+				checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[url: 'https://git.wmi.amu.edu.pl/s426206/ium_426206.git']]])
+			}
+		}
+		
+		stage('docker') {
+			steps {
+                script {
+                    def img = docker.build('rokoch/ium:01')
+                    img.inside {
+                        sh 'chmod +x dlgssdpytorch.py'
+                        sh 'python3 ./dlgssdpytorch.py'
+                    }
+                }
+			}
+		}
+		
+		stage('end') {
+            steps {
+				//Zarchiwizuj wynik
+				archiveArtifacts 'model.pt'
+			}
+		}
+	}
+}

Jenkinsfile

@@ -2,9 +2,9 @@ pipeline {
 	agent any 
 	parameters {
 		string(
-		defaultValue: '0',
-		description: 'Umożliwia zdefiniowanie wielkości odcięcia zbioru danych.',
-		name: 'CUTOFF',
+		defaultValue: '',
+		description: 'Parametry trenowania.',
+		name: 'PARAMETRY',
 		trim: false
 		)
 		string(
@@ -26,12 +26,17 @@ pipeline {
 			}
 		}
 		
-        stage('sh') {
+		stage('docker') {
 			steps {
 				withEnv(["KAGGLE_USERNAME=${params.KAGGLE_USERNAME}",
 					"KAGGLE_KEY=${params.KAGGLE_KEY}"]) {
-					sh "chmod +x skrypt.sh"
-					sh "./skrypt.sh ${params.CUTOFF} | tee output.txt"
+					script {
+						def img = docker.build('rokoch/ium:01')
+						img.inside {
+							sh 'chmod +x create_dataset.py'
+							sh 'python3 ./create_dataset.py $PARAMETRY'
+						}
+					}
 				}
 			}
 		}
@@ -39,7 +44,7 @@ pipeline {
 		stage('end') {
             steps {
 				//Zarchiwizuj wynik
-				archiveArtifacts 'output.txt,Global_Superstore22.csv,Global_Superstore2.csv.dev,Global_Superstore2.csv.test,Global_Superstore2.csv.train'
+				archiveArtifacts 'train_dataset.pt,val_dataset.pt'
 			}
 		}
 	}

Jenkinsfile_stats

@@ -0,0 +1,32 @@
+pipeline {
+    //agent { docker {image 'rokoch/ium:01' }}
+    agent any
+    parameters {
+      buildSelector(
+          defaultSelector: lastSuccessful(),
+          description: 'Which build to use for copying artifacts',
+          name: 'BUILD_SELECTOR')
+    }
+    stages {
+        stage('Copy artifact') {
+                steps {
+                    copyArtifacts filter: 'Global_Superstore22.csv,Global_Superstore2.csv.dev,Global_Superstore2.csv.test,Global_Superstore2.csv.train', fingerprintArtifacts: false, projectName: 's426206-create-dataset', selector: buildParameter('BUILD_SELECTOR')
+                }
+            }
+        stage('Clone repo') {
+            steps {
+                script {
+                    //docker.withRegistry("https://hub.docker.com/r/rokoch/ium"){
+                        docker.image("rokoch/ium:01").inside {
+                            sh 'rm -rf ium_426206'
+                            sh 'git clone https://git.wmi.amu.edu.pl/s426206/ium_426206.git'
+                            sh "chmod +x ium_426206/stats.sh"
+                            sh "ium_426206/stats.sh | tee output.txt"
+                            archiveArtifacts 'output.txt'
+                        }
+                    //}
+                }
+            }
+        }
+    }
+}

create_dataset.py

@@ -0,0 +1,59 @@
+import zipfile
+import torch
+import pandas as pd
+import datetime
+import numpy as np
+
+from kaggle.api.kaggle_api_extended import KaggleApi
+import torch.nn as nn
+from torch.utils.data.dataset import random_split
+from torch.utils.data import Dataset, TensorDataset
+from sklearn import preprocessing
+
+
+api = KaggleApi()
+api.authenticate()
+api.dataset_download_file('apoorvaappz/global-super-store-dataset',
+                              file_name='Global_Superstore2.csv', path='./')
+
+with zipfile.ZipFile('Global_Superstore2.csv.zip', 'r') as zipref:
+    zipref.extractall('.')
+
+data = pd.read_csv("Global_Superstore2.csv", header=0, sep=',')
+
+data["Order Date"] = pd.to_datetime(data["Order Date"])
+data = data.sort_values(by="Order Date")
+
+#print(data)
+
+byMonthsYears = {}
+for index, row in data.iterrows():
+    #datee = datetime.datetime.strptime(row['Order Date'], "%d-%m-%Y")
+    #byMonthsYears.setdefault(datee.strftime("%m-%Y"), 0)
+    #byMonthsYears[datee.strftime("%m-%Y")] += row['Sales']
+    byMonthsYears.setdefault(row['Order Date'].strftime("%d-%m-%Y"), 0)
+    byMonthsYears[row['Order Date'].strftime("%d-%m-%Y")] += row['Sales']
+df = data.groupby('Order Date').agg({'Customer Name':'count', 'Sales': 'sum'}).reset_index().rename(columns={'Sales':'Sales sum', 'Customer Name':'Sales count'})
+
+#normalizacja danych
+flcols = df[['Sales count', 'Sales sum']].columns
+x = df[['Sales count', 'Sales sum']].values
+# min_max_scaler = preprocessing.MinMaxScaler()
+max_abs_scaler = preprocessing.MaxAbsScaler()
+# x_scaled = min_max_scaler.fit_transform(x)
+x_scaled = max_abs_scaler.fit_transform(x)
+normcols = pd.DataFrame(x_scaled, columns=flcols)
+for col in flcols:
+    df[col] = normcols[col]
+#df.to_csv('mms_norm.csv')
+
+x_tensor = torch.tensor(df['Sales sum'].values).float()
+y_tensor = torch.tensor(df['Sales count'].values).float()
+
+dataset = TensorDataset(x_tensor, y_tensor)
+
+lengths = [int(len(dataset)*0.8), int(len(dataset)*0.2)]
+train_dataset, val_dataset = random_split(dataset, lengths)
+
+torch.save(train_dataset, 'train_dataset.pt')
+torch.save(val_dataset, 'val_dataset.pt')

dlgssdpytorch copy.py

@@ -0,0 +1,180 @@
+import zipfile
+import torch
+import pandas as pd
+import matplotlib
+import matplotlib.pyplot as plt
+import datetime
+import numpy as np
+
+from kaggle.api.kaggle_api_extended import KaggleApi
+import torch.nn as nn
+import torch.optim as optim
+from torch.utils.data.dataset import random_split
+from torch.utils.data import Dataset, TensorDataset, DataLoader
+from torchviz import make_dot
+from sklearn import preprocessing
+
+
+# api = KaggleApi()
+# api.authenticate()
+# api.dataset_download_file('apoorvaappz/global-super-store-dataset',
+#                               file_name='Global_Superstore2.csv', path='./')
+
+# with zipfile.ZipFile('Global_Superstore2.csv.zip', 'r') as zipref:
+#     zipref.extractall('.')
+
+data = pd.read_csv("Global_Superstore2.csv", header=0, sep=',')
+
+data["Order Date"] = pd.to_datetime(data["Order Date"])
+data = data.sort_values(by="Order Date")
+
+#print(data)
+
+byMonthsYears = {}
+for index, row in data.iterrows():
+    #datee = datetime.datetime.strptime(row['Order Date'], "%d-%m-%Y")
+    #byMonthsYears.setdefault(datee.strftime("%m-%Y"), 0)
+    #byMonthsYears[datee.strftime("%m-%Y")] += row['Sales']
+    byMonthsYears.setdefault(row['Order Date'].strftime("%d-%m-%Y"), 0)
+    byMonthsYears[row['Order Date'].strftime("%d-%m-%Y")] += row['Sales']
+df = data.groupby('Order Date').agg({'Customer Name':'count', 'Sales': 'sum'}).reset_index().rename(columns={'Sales':'Sales sum', 'Customer Name':'Sales count'})
+#normalizacja danych
+flcols = df[['Sales count', 'Sales sum']].columns
+x = df[['Sales count', 'Sales sum']].values
+# min_max_scaler = preprocessing.MinMaxScaler()
+max_abs_scaler = preprocessing.MaxAbsScaler()
+# x_scaled = min_max_scaler.fit_transform(x)
+x_scaled = max_abs_scaler.fit_transform(x)
+normcols = pd.DataFrame(x_scaled, columns=flcols)
+for col in flcols:
+    df[col] = normcols[col]
+df.to_csv('mms_norm.csv')
+exit()
+# fig, ax = plt.subplots()
+# fig.set_figheight(15)
+# fig.set_figwidth(20)
+# ax.scatter(df['Month and Year'], df['Sum of sales'])
+#plt.show()
+# # Data Generation
+# np.random.seed(42)
+# x = np.random.rand(100, 1)
+# y = 1 + 2 * x + .1 * np.random.randn(100, 1)
+
+# # Shuffles the indices
+# idx = np.arange(100)
+# np.random.shuffle(idx)
+
+# # Uses first 80 random indices for train
+# train_idx = idx[:80]
+# # Uses the remaining indices for validation
+# val_idx = idx[80:]
+
+# # Generates train and validation sets
+# x_train, y_train = x[train_idx], y[train_idx]
+# x_val, y_val = x[val_idx], y[val_idx]
+# x_tensor = torch.from_numpy(x_train).float()
+# y_tensor = torch.from_numpy(y_train).float()
+
+x_tensor = torch.tensor(df['Sales sum'].values).float()
+y_tensor = torch.tensor(df['Sales count'].values).float()
+
+dataset = TensorDataset(x_tensor, y_tensor)
+
+
+#torch.manual_seed(42)
+lengths = [int(len(dataset)*0.8), int(len(dataset)*0.2)]
+train_dataset, val_dataset = random_split(dataset, lengths)
+
+train_loader = DataLoader(dataset=train_dataset)
+val_loader = DataLoader(dataset=val_dataset)
+
+
+class LayerLinearRegression(nn.Module):
+    def __init__(self):
+        super().__init__()
+        # Instead of our custom parameters, we use a Linear layer with single input and single output
+        self.linear = nn.Linear(1, 1)
+                
+    def forward(self, x):
+        # Now it only takes a call to the layer to make predictions
+        return self.linear(x)
+    
+model = LayerLinearRegression()
+# Checks model's parameters
+#print(model.state_dict())   
+
+lr = 1e-3
+n_epochs = 100
+
+loss_fn = nn.MSELoss(reduction='mean')
+optimizer = optim.SGD(model.parameters(), lr=lr)
+
+def make_train_step(model, loss_fn, optimizer):
+    # Builds function that performs a step in the train loop
+    def train_step(x, y):
+        # Sets model to TRAIN mode
+        model.train()
+        # Makes predictions
+        yhat = model(x)
+        # Computes loss
+        loss = loss_fn(y, yhat)
+        # Computes gradients
+        loss.backward()
+        # Updates parameters and zeroes gradients
+        optimizer.step()
+        optimizer.zero_grad()
+        # Returns the loss
+        return loss.item()
+    
+    # Returns the function that will be called inside the train loop
+    return train_step
+
+# Creates the train_step function for our model, loss function and optimizer
+train_step = make_train_step(model, loss_fn, optimizer)
+training_losses = []
+validation_losses = []
+print(model.state_dict())   
+# For each epoch...
+for epoch in range(n_epochs):
+    losses = []
+    # Uses loader to fetch one mini-batch for training
+    for x_batch, y_batch in train_loader:
+        # NOW, sends the mini-batch data to the device
+        # so it matches location of the MODEL
+        # x_batch = x_batch.to(device)
+        # y_batch = y_batch.to(device)
+        # One stpe of training
+        loss = train_step(x_batch, y_batch)
+        losses.append(loss)
+    training_loss = np.mean(losses)
+    training_losses.append(training_loss)
+        
+    # After finishing training steps for all mini-batches,
+    # it is time for evaluation!
+        
+    # We tell PyTorch to NOT use autograd...
+    # Do you remember why?
+    with torch.no_grad():
+        val_losses = []
+        # Uses loader to fetch one mini-batch for validation
+        for x_val, y_val in val_loader:
+            # Again, sends data to same device as model
+            # x_val = x_val.to(device)
+            # y_val = y_val.to(device)
+            
+            # What is that?!
+            model.eval()
+            # Makes predictions
+            yhat = model(x_val)
+            # Computes validation loss
+            val_loss = loss_fn(y_val, yhat)
+            val_losses.append(val_loss.item())
+        validation_loss = np.mean(val_losses)
+        validation_losses.append(validation_loss)
+
+    print(f"[{epoch+1}] Training loss: {training_loss:.3f}\t Validation loss: {validation_loss:.3f}")
+
+# Checks model's parameters
+print(model.state_dict())
+print(np.mean(losses))
+print(np.mean(val_losses))   

dlgssdpytorch.py

@@ -0,0 +1,115 @@
+import torch
+import numpy as np
+
+import torch.nn as nn
+import torch.optim as optim
+from torch.utils.data import Dataset, TensorDataset, DataLoader
+import argparse
+
+parser = argparse.ArgumentParser(description='Program do uczenia modelu')
+parser.add_argument('-l', '--lr', type=float, default=1e-3, help="Współczynik uczenia (lr)", required=False)
+parser.add_argument('-e', '--epochs', type=int, default=100, help="Liczba epok", required=False)
+args = parser.parse_args()
+
+lr = args.lr
+n_epochs = args.epochs
+
+train_dataset = torch.load('train_dataset.pt')
+val_dataset = torch.load('val_dataset.pt')
+
+train_loader = DataLoader(dataset=train_dataset)
+val_loader = DataLoader(dataset=val_dataset)
+
+class LayerLinearRegression(nn.Module):
+    def __init__(self):
+        super().__init__()
+        # Instead of our custom parameters, we use a Linear layer with single input and single output
+        self.linear = nn.Linear(1, 1)
+                
+    def forward(self, x):
+        # Now it only takes a call to the layer to make predictions
+        return self.linear(x)
+    
+model = LayerLinearRegression()
+# Checks model's parameters
+#print(model.state_dict())   
+
+loss_fn = nn.MSELoss(reduction='mean')
+optimizer = optim.SGD(model.parameters(), lr=lr)
+
+def make_train_step(model, loss_fn, optimizer):
+    # Builds function that performs a step in the train loop
+    def train_step(x, y):
+        # Sets model to TRAIN mode
+        model.train()
+        # Makes predictions
+        yhat = model(x)
+        # Computes loss
+        loss = loss_fn(y, yhat)
+        # Computes gradients
+        loss.backward()
+        # Updates parameters and zeroes gradients
+        optimizer.step()
+        optimizer.zero_grad()
+        # Returns the loss
+        return loss.item()
+    
+    # Returns the function that will be called inside the train loop
+    return train_step
+
+# Creates the train_step function for our model, loss function and optimizer
+train_step = make_train_step(model, loss_fn, optimizer)
+training_losses = []
+validation_losses = []
+#print(model.state_dict())   
+# For each epoch...
+for epoch in range(n_epochs):
+    losses = []
+    # Uses loader to fetch one mini-batch for training
+    for x_batch, y_batch in train_loader:
+        # NOW, sends the mini-batch data to the device
+        # so it matches location of the MODEL
+        # x_batch = x_batch.to(device)
+        # y_batch = y_batch.to(device)
+        # One stpe of training
+        loss = train_step(x_batch, y_batch)
+        losses.append(loss)
+    training_loss = np.mean(losses)
+    training_losses.append(training_loss)
+        
+    # After finishing training steps for all mini-batches,
+    # it is time for evaluation!
+        
+    # We tell PyTorch to NOT use autograd...
+    # Do you remember why?
+    with torch.no_grad():
+        val_losses = []
+        # Uses loader to fetch one mini-batch for validation
+        for x_val, y_val in val_loader:
+            # Again, sends data to same device as model
+            # x_val = x_val.to(device)
+            # y_val = y_val.to(device)
+            
+            model.eval()
+            # Makes predictions
+            yhat = model(x_val)
+            # Computes validation loss
+            val_loss = loss_fn(y_val, yhat)
+            val_losses.append(val_loss.item())
+        validation_loss = np.mean(val_losses)
+        validation_losses.append(validation_loss)
+
+    print(f"[{epoch+1}] Training loss: {training_loss:.3f}\t Validation loss: {validation_loss:.3f}")
+
+# Checks model's parameters
+print("Model's state_dict:")
+for param_tensor in model.state_dict():
+    print(param_tensor, "\t", model.state_dict()[param_tensor])
+
+# Print optimizer's state_dict
+print("Optimizer's state_dict:")
+for var_name in optimizer.state_dict():
+    print(var_name, "\t", optimizer.state_dict()[var_name])
+print("Mean squared error for training: ", np.mean(losses))
+print("Mean squared error for validating: ", np.mean(val_losses))   
+torch.save(model, 'model.pt')

requirments.txt

@@ -0,0 +1,27 @@
+certifi==2020.12.5
+chardet==4.0.0
+cycler==0.10.0
+graphviz==0.16
+idna==2.10
+joblib==1.0.1
+kaggle==1.5.12
+kiwisolver==1.3.1
+matplotlib==3.4.1
+numpy==1.20.2
+pandas==1.2.4
+Pillow==8.2.0
+pyparsing==2.4.7
+python-dateutil==2.8.1
+python-slugify==4.0.1
+pytz==2021.1
+requests==2.25.1
+scikit-learn==0.24.1
+scipy==1.6.2
+six==1.15.0
+sklearn==0.0
+text-unidecode==1.3
+threadpoolctl==2.1.0
+torchviz==0.0.2
+tqdm==4.60.0
+typing-extensions==3.7.4.3
+urllib3==1.26.4

skrypt.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 kaggle datasets download -d apoorvaappz/global-super-store-dataset
 unzip -o global-super-store-dataset.zip
-if [ $1 = "0" ]; then
+if [[ $1 = "0" ]]; then
 	CUTOFF=51291
 	cp Global_Superstore2.csv Global_Superstore22.csv
 else

zadanie1.ipynb

@@ -3,7 +3,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "eligible-business",
+   "id": "strange-teens",
    "metadata": {
     "tags": []
    },
@@ -18,7 +18,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "limited-memorial",
+   "id": "another-accessory",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -28,7 +28,7 @@
   {
    "cell_type": "code",
    "execution_count": 158,
-   "id": "aware-allah",
+   "id": "valid-malta",
    "metadata": {},
    "outputs": [
     {
@@ -48,7 +48,7 @@
   {
    "cell_type": "code",
    "execution_count": 159,
-   "id": "drawn-financing",
+   "id": "noble-compilation",
    "metadata": {},
    "outputs": [
     {
@@ -448,7 +448,7 @@
   {
    "cell_type": "code",
    "execution_count": 160,
-   "id": "boring-consumption",
+   "id": "multiple-council",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -458,7 +458,7 @@
   {
    "cell_type": "code",
    "execution_count": 161,
-   "id": "cathedral-frank",
+   "id": "green-trunk",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -468,7 +468,7 @@
   {
    "cell_type": "code",
    "execution_count": 162,
-   "id": "satisfactory-venice",
+   "id": "operating-catalyst",
    "metadata": {},
    "outputs": [
     {
@@ -489,7 +489,7 @@
   {
    "cell_type": "code",
    "execution_count": 163,
-   "id": "united-climate",
+   "id": "female-landscape",
    "metadata": {},
    "outputs": [
     {
@@ -510,7 +510,7 @@
   {
    "cell_type": "code",
    "execution_count": 164,
-   "id": "institutional-corpus",
+   "id": "thirty-auckland",
    "metadata": {},
    "outputs": [
     {
@@ -531,7 +531,7 @@
   {
    "cell_type": "code",
    "execution_count": 165,
-   "id": "caroline-shannon",
+   "id": "mysterious-alignment",
    "metadata": {},
    "outputs": [
     {
@@ -552,7 +552,7 @@
   {
    "cell_type": "code",
    "execution_count": 166,
-   "id": "sublime-quarter",
+   "id": "stone-combining",
    "metadata": {},
    "outputs": [
     {
@@ -950,7 +950,7 @@
   {
    "cell_type": "code",
    "execution_count": 167,
-   "id": "committed-disease",
+   "id": "demanding-milwaukee",
    "metadata": {
     "tags": []
    },
@@ -990,7 +990,7 @@
   {
    "cell_type": "code",
    "execution_count": 168,
-   "id": "corporate-fisher",
+   "id": "above-script",
    "metadata": {
     "tags": []
    },
@@ -1026,7 +1026,7 @@
   {
    "cell_type": "code",
    "execution_count": 169,
-   "id": "revised-study",
+   "id": "abroad-durham",
    "metadata": {},
    "outputs": [
     {
@@ -1047,7 +1047,7 @@
   {
    "cell_type": "code",
    "execution_count": 170,
-   "id": "checked-thought",
+   "id": "centered-realtor",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1058,7 +1058,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "assigned-mobility",
+   "id": "relevant-receptor",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1076,7 +1076,7 @@
   {
    "cell_type": "code",
    "execution_count": 172,
-   "id": "stunning-metallic",
+   "id": "informal-unemployment",
    "metadata": {},
    "outputs": [
     {
@@ -1474,7 +1474,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "boolean-calgary",
+   "id": "reserved-cookie",
    "metadata": {},
    "outputs": [],
    "source": []
@@ -1496,7 +1496,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.9"
+   "version": "3.8.5"
   }
  },
  "nbformat": 4,