jenkinsfile2

Dockerfile

@@ -1,34 +1,15 @@
 FROM ubuntu:latest
-# Install required dependencies
+FROM tensorflow/tensorflow:latest
-RUN apt update
+RUN apt update && apt install -y
-RUN apt-get update
+RUN apt-get install -y python3
-RUN apt install -y figlet
+RUN apt-get install -y unzip
-RUN export PATH=”$PATH:/usr/local/bin/python”
+RUN apt-get install -y python3-pip
-RUN apt install python3-pip -y
+RUN apt-get install unzip
-RUN apt install unzip -y
+RUN apt-get install wget
-RUN pip3 install --upgrade pip
+# RUN python3 -m pip install kaggle
-RUN pip3 install --upgrade Pillow
+RUN python3 -m pip install pandas
-RUN pip3 install pandas
-RUN pip3 install scikit-learn
 RUN pip3 install matplotlib
-RUN pip3 install jovian
+# RUN ln -s ~/.local/bin/kaggle /usr/bin/kaggle
-RUN pip3 install torch
-RUN pip3 install seaborn
-RUN pip3 install torchvision
-
-
-# COPY ./script.sh ./
-COPY ./runTrain.sh ./runTrain.sh 
-
-# RUN chmod a+x ./script.sh
-# Create app directory in image
 WORKDIR /app
-COPY ./winequality-red.csv ./
+COPY . .
-# Copy init dataset script to /app directory in image
+RUN ls -la
-COPY ./pytorch/pytorch.py ./
-
-# Script executed after docker run
-# CMD python3 ./pytorch.py
-COPY ./runTrain.sh ./
-RUN chmod a+x  ./runTrain.sh
-RUN ./runTrain.sh

Dockerfile_old

@@ -0,0 +1,34 @@
+FROM ubuntu:latest
+# Install required dependencies
+RUN apt update
+RUN apt-get update
+RUN apt install -y figlet
+RUN export PATH=”$PATH:/usr/local/bin/python”
+RUN apt install python3-pip -y
+RUN apt install unzip -y
+RUN pip3 install --upgrade pip
+RUN pip3 install --upgrade Pillow
+RUN pip3 install pandas
+RUN pip3 install scikit-learn
+RUN pip3 install matplotlib
+RUN pip3 install jovian
+RUN pip3 install torch
+RUN pip3 install seaborn
+RUN pip3 install torchvision
+
+
+# COPY ./script.sh ./
+COPY ./runTrain.sh ./runTrain.sh 
+
+# RUN chmod a+x ./script.sh
+# Create app directory in image
+WORKDIR /app
+COPY ./winequality-red.csv ./
+# Copy init dataset script to /app directory in image
+COPY ./pytorch/pytorch.py ./
+
+# Script executed after docker run
+# CMD python3 ./pytorch.py
+COPY ./runTrain.sh ./
+RUN chmod a+x  ./runTrain.sh
+RUN ./runTrain.sh

Jenkinsfile

@@ -21,11 +21,13 @@ pipeline {
         KAGGLE_KEY="$params.KAGGLE_KEY"
         CUTOFF="$params.CUTOFF"
     }
-    agent 
+    agent {
-	 { 
+        docker { image 'ksero/ium:pytorch' }
-	 dockerfile true 
+    }
-	 }
+	options {
-
+        copyArtifactPermission('s444354-training');
+    }
+	
     stages {
         stage("Check out from version control") {
             steps {
@@ -36,11 +38,15 @@ pipeline {
             steps {
 				sh "chmod u+x ./script.sh"
                 sh "KAGGLE_USERNAME=${KAGGLE_USERNAME} KAGGLE_KEY=${KAGGLE_KEY} CUTOFF=${CUTOFF} ./script.sh"
-                archiveArtifacts 'TEST_winequality-red.csv'
+                //archiveArtifacts 'TEST_winequality-red.csv'
-				archiveArtifacts 'TRAIN_winequality-red.csv'
+				//archiveArtifacts 'TRAIN_winequality-red.csv'
-				archiveArtifacts 'VAL_winequality-red.csv'
+				//archiveArtifacts 'VAL_winequality-red.csv'
 				archiveArtifacts 'pytorch/result.txt'
             }
         }
+		 stage ('Starting train job') {
+            steps {
+                build job: 's444354-training/master', wait: false
+            }
     }
 }

Jenkinsfile2

@@ -1,25 +1,32 @@
 pipeline {
-     agent { 
+     agent {
-	 dockerfile {image 'ksero/ium:pytorch'}
+        docker { image 'ubuntu:latest' }
-	 }
+    }
 	 
- /*   parameters{
+  parameters{
         buildSelector(
             defaultSelector: lastSuccessful(),
-            name: 's444354-create-dataset',
+            name: 'BUILD_SELECTOR',
-            description: 'Which build to use  for copying artifacts'
+            description: 'Build selector for Copy artifact'
         )
-    }*/
+    }
-
 
    
-    stages {
+     stages {
         stage("Check out from version control") {
-           steps {
+            steps {
-				checkout([$class: 'GitSCM', branches: [[name: '*/master']], extensions: [], userRemoteConfigs: [[credentialsId: 's444354', url: 'https://git.wmi.amu.edu.pl/s444354/ium_444354.git']]])
+                checkout scm
-			}
+            }
         }
+		
+		
         stage("Shell Script") {
+            steps {
+                copyArtifacts(
+                    fingerprintArtifacts: true, 
+                    projectName: 's444354-create-dataset',
+                    selector: buildParameter('BUILD_SELECTOR')
+                )
             steps {
                 sh "chmod u+x ./script2.sh"
                 sh " ./script2.sh"

Jenkinsfile3

@@ -0,0 +1,45 @@
+pipeline {
+     agent {
+        dockerfile true
+    }
+	 
+   parameters{
+        string(
+            defaultValue: '5', 
+            description: 'Epoch number', 
+            name: 'EPOCH_NUMBER'
+        )
+    }
+
+   
+stages {
+        stage('Copy') {
+            steps {
+                copyArtifacts projectName: 's44354-create-dataset'
+                sh 'ls -la'
+                sh 'echo $EPOCH_NUMBER'
+                sh 'python3 ./pytorch/pytorch.py $EPOCH_NUMBER'
+            }
+        }
+        stage('Archive') {
+            steps {
+                dir('saved_model') {
+                    archiveArtifacts artifacts: '**/**'
+                }
+            }
+        }
+        stage('Starting eval job') {
+            steps {
+                build job: 's444354-evaluation/master', wait: false
+            }
+        }
+    }
+    options {
+        copyArtifactPermission('s444354-evaluation');
+    }
+    post {
+        always {
+            emailext body: "${currentBuild.currentResult}", subject: 's444354-testing build status', to: 'e19191c5.uam.onmicrosoft.com@emea.teams.ms'
+        }
+    }
+}

Jenkinsfile_evaluation

@@ -1,32 +1,50 @@
 pipeline {
-    agent {
+    agent { 
-        docker { image 's444507_create_dataset_image:latest' }
+        dockerfile true
     }
-    parameters {
+    parameters{
-        buildSelector(defaultSelector: lastSuccessful(), description: 'Which build to use for copying artifacts.', name: 'BUILD_SELECTOR')
+        string(
-        gitParameter branchFilter: 'origin/(.*)', defaultValue: 'master', name: 'BRANCH', type: 'PT_BRANCH'
+            defaultValue: 'master', 
-
+            description: 'training branch name', 
+            name: 'TRAINING_BRANCH'
+        )
+        password(
+            defaultValue: '',
+            description: 'JENKINS password',
+            name: 'JENKINS_KEY'
+        )
     }
     stages {
-        stage('Get arifacts') {
+        stage('Copy prev build artifact') {
             steps {
-                copyArtifacts fingerprintArtifacts: true, projectName: 's444507-create-dataset', selector: buildParameter('BUILD_SELECTOR')
+                script {
-                copyArtifacts fingerprintArtifacts: true, projectName: '444507-training/${BRANCH}', selector: buildParameter('BUILD_SELECTOR')
+                    if (currentBuild.previousBuild) {
-                copyArtifacts fingerprintArtifacts: true, projectName: 's444507-evaluation/master', selector: buildParameter('BUILD_SELECTOR'), optional: true
+                        try {
+                            copyArtifacts(projectName: currentBuild.projectName,
+                                          selector: specific("${currentBuild.previousBuild.number}"))
+                            def previousFile = readFile(file: "TRAIN_winequality-red.csv")
+                            echo("The current build is ${currentBuild.number}")
+                            echo("The previous build artifact was: ${previousFile}")
+                        } catch(err) {
+                            // ignore error
+                        }
+                    }
+                }
             }
         }
-        stage('Run evaluation') {
+        stage('Copy') {
             steps {
-                sh "python3 ./lab06_evaluation.py ${currentBuild.number}"
+                copyArtifacts projectName: 's444354-create-dataset'
+                copyArtifacts projectName: 's444354-training/$TRAINING_BRANCH'
+                sh 'python3 ./evaluation.py'
+                archiveArtifacts 'TRAIN_winequality-red.csv'
+                sh "ls -la"
             }
         }
     }
     post {
-        success {
-            archiveArtifacts artifacts: 'metrics.*', followSymlinks: false
-        }
         always {
-            emailext body: "${currentBuild.currentResult}", subject: 's444507-evaluation', to: 'e19191c5.uam.onmicrosoft.com@emea.teams.ms'
+            emailext body: "${currentBuild.currentResult}", subject: '444354-evaluation build status', to: 'e19191c5.uam.onmicrosoft.com@emea.teams.ms'
         }
     }
 }