From b238956acd0b7b30236012f40d59fb574e736a65 Mon Sep 17 00:00:00 2001 From: Adrian Charkiewicz Date: Fri, 6 May 2022 17:08:30 +0200 Subject: [PATCH] jenkinsfile2 --- Dockerfile | 43 ++++++++++-------------------------- Dockerfile_old | 34 ++++++++++++++++++++++++++++ Jenkinsfile | 22 ++++++++++++------- Jenkinsfile2 | 31 ++++++++++++++++---------- Jenkinsfile3 | 45 +++++++++++++++++++++++++++++++++++++ Jenkinsfile_evaluation | 50 ++++++++++++++++++++++++++++-------------- 6 files changed, 158 insertions(+), 67 deletions(-) create mode 100644 Dockerfile_old create mode 100644 Jenkinsfile3 diff --git a/Dockerfile b/Dockerfile index c2f4c89..55d694a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,34 +1,15 @@ FROM ubuntu:latest -# Install required dependencies -RUN apt update -RUN apt-get update -RUN apt install -y figlet -RUN export PATH=”$PATH:/usr/local/bin/python” -RUN apt install python3-pip -y -RUN apt install unzip -y -RUN pip3 install --upgrade pip -RUN pip3 install --upgrade Pillow -RUN pip3 install pandas -RUN pip3 install scikit-learn +FROM tensorflow/tensorflow:latest +RUN apt update && apt install -y +RUN apt-get install -y python3 +RUN apt-get install -y unzip +RUN apt-get install -y python3-pip +RUN apt-get install unzip +RUN apt-get install wget +# RUN python3 -m pip install kaggle +RUN python3 -m pip install pandas RUN pip3 install matplotlib -RUN pip3 install jovian -RUN pip3 install torch -RUN pip3 install seaborn -RUN pip3 install torchvision - - -# COPY ./script.sh ./ -COPY ./runTrain.sh ./runTrain.sh - -# RUN chmod a+x ./script.sh -# Create app directory in image +# RUN ln -s ~/.local/bin/kaggle /usr/bin/kaggle WORKDIR /app -COPY ./winequality-red.csv ./ -# Copy init dataset script to /app directory in image -COPY ./pytorch/pytorch.py ./ - -# Script executed after docker run -# CMD python3 ./pytorch.py -COPY ./runTrain.sh ./ -RUN chmod a+x ./runTrain.sh -RUN ./runTrain.sh +COPY . . +RUN ls -la diff --git a/Dockerfile_old b/Dockerfile_old new file mode 100644 index 0000000..c2f4c89 --- /dev/null +++ b/Dockerfile_old @@ -0,0 +1,34 @@ +FROM ubuntu:latest +# Install required dependencies +RUN apt update +RUN apt-get update +RUN apt install -y figlet +RUN export PATH=”$PATH:/usr/local/bin/python” +RUN apt install python3-pip -y +RUN apt install unzip -y +RUN pip3 install --upgrade pip +RUN pip3 install --upgrade Pillow +RUN pip3 install pandas +RUN pip3 install scikit-learn +RUN pip3 install matplotlib +RUN pip3 install jovian +RUN pip3 install torch +RUN pip3 install seaborn +RUN pip3 install torchvision + + +# COPY ./script.sh ./ +COPY ./runTrain.sh ./runTrain.sh + +# RUN chmod a+x ./script.sh +# Create app directory in image +WORKDIR /app +COPY ./winequality-red.csv ./ +# Copy init dataset script to /app directory in image +COPY ./pytorch/pytorch.py ./ + +# Script executed after docker run +# CMD python3 ./pytorch.py +COPY ./runTrain.sh ./ +RUN chmod a+x ./runTrain.sh +RUN ./runTrain.sh diff --git a/Jenkinsfile b/Jenkinsfile index 4b4783a..5576e8e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,11 +21,13 @@ pipeline { KAGGLE_KEY="$params.KAGGLE_KEY" CUTOFF="$params.CUTOFF" } - agent - { - dockerfile true - } - + agent { + docker { image 'ksero/ium:pytorch' } + } + options { + copyArtifactPermission('s444354-training'); + } + stages { stage("Check out from version control") { steps { @@ -36,11 +38,15 @@ pipeline { steps { sh "chmod u+x ./script.sh" sh "KAGGLE_USERNAME=${KAGGLE_USERNAME} KAGGLE_KEY=${KAGGLE_KEY} CUTOFF=${CUTOFF} ./script.sh" - archiveArtifacts 'TEST_winequality-red.csv' - archiveArtifacts 'TRAIN_winequality-red.csv' - archiveArtifacts 'VAL_winequality-red.csv' + //archiveArtifacts 'TEST_winequality-red.csv' + //archiveArtifacts 'TRAIN_winequality-red.csv' + //archiveArtifacts 'VAL_winequality-red.csv' archiveArtifacts 'pytorch/result.txt' } } + stage ('Starting train job') { + steps { + build job: 's444354-training/master', wait: false + } } } \ No newline at end of file diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 1b1e6d9..23678dc 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -1,25 +1,32 @@ pipeline { - agent { - dockerfile {image 'ksero/ium:pytorch'} - } + agent { + docker { image 'ubuntu:latest' } + } - /* parameters{ + parameters{ buildSelector( defaultSelector: lastSuccessful(), - name: 's444354-create-dataset', - description: 'Which build to use for copying artifacts' + name: 'BUILD_SELECTOR', + description: 'Build selector for Copy artifact' ) - }*/ - + } - stages { + stages { stage("Check out from version control") { - steps { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], extensions: [], userRemoteConfigs: [[credentialsId: 's444354', url: 'https://git.wmi.amu.edu.pl/s444354/ium_444354.git']]]) - } + steps { + checkout scm + } } + + stage("Shell Script") { + steps { + copyArtifacts( + fingerprintArtifacts: true, + projectName: 's444354-create-dataset', + selector: buildParameter('BUILD_SELECTOR') + ) steps { sh "chmod u+x ./script2.sh" sh " ./script2.sh" diff --git a/Jenkinsfile3 b/Jenkinsfile3 new file mode 100644 index 0000000..3588551 --- /dev/null +++ b/Jenkinsfile3 @@ -0,0 +1,45 @@ +pipeline { + agent { + dockerfile true + } + + parameters{ + string( + defaultValue: '5', + description: 'Epoch number', + name: 'EPOCH_NUMBER' + ) + } + + +stages { + stage('Copy') { + steps { + copyArtifacts projectName: 's44354-create-dataset' + sh 'ls -la' + sh 'echo $EPOCH_NUMBER' + sh 'python3 ./pytorch/pytorch.py $EPOCH_NUMBER' + } + } + stage('Archive') { + steps { + dir('saved_model') { + archiveArtifacts artifacts: '**/**' + } + } + } + stage('Starting eval job') { + steps { + build job: 's444354-evaluation/master', wait: false + } + } + } + options { + copyArtifactPermission('s444354-evaluation'); + } + post { + always { + emailext body: "${currentBuild.currentResult}", subject: 's444354-testing build status', to: 'e19191c5.uam.onmicrosoft.com@emea.teams.ms' + } + } +} \ No newline at end of file diff --git a/Jenkinsfile_evaluation b/Jenkinsfile_evaluation index 9698f44..85ef402 100644 --- a/Jenkinsfile_evaluation +++ b/Jenkinsfile_evaluation @@ -1,32 +1,50 @@ pipeline { - agent { - docker { image 's444507_create_dataset_image:latest' } + agent { + dockerfile true } - parameters { - buildSelector(defaultSelector: lastSuccessful(), description: 'Which build to use for copying artifacts.', name: 'BUILD_SELECTOR') - gitParameter branchFilter: 'origin/(.*)', defaultValue: 'master', name: 'BRANCH', type: 'PT_BRANCH' - + parameters{ + string( + defaultValue: 'master', + description: 'training branch name', + name: 'TRAINING_BRANCH' + ) + password( + defaultValue: '', + description: 'JENKINS password', + name: 'JENKINS_KEY' + ) } stages { - stage('Get arifacts') { + stage('Copy prev build artifact') { steps { - copyArtifacts fingerprintArtifacts: true, projectName: 's444507-create-dataset', selector: buildParameter('BUILD_SELECTOR') - copyArtifacts fingerprintArtifacts: true, projectName: '444507-training/${BRANCH}', selector: buildParameter('BUILD_SELECTOR') - copyArtifacts fingerprintArtifacts: true, projectName: 's444507-evaluation/master', selector: buildParameter('BUILD_SELECTOR'), optional: true + script { + if (currentBuild.previousBuild) { + try { + copyArtifacts(projectName: currentBuild.projectName, + selector: specific("${currentBuild.previousBuild.number}")) + def previousFile = readFile(file: "TRAIN_winequality-red.csv") + echo("The current build is ${currentBuild.number}") + echo("The previous build artifact was: ${previousFile}") + } catch(err) { + // ignore error + } + } + } } } - stage('Run evaluation') { + stage('Copy') { steps { - sh "python3 ./lab06_evaluation.py ${currentBuild.number}" + copyArtifacts projectName: 's444354-create-dataset' + copyArtifacts projectName: 's444354-training/$TRAINING_BRANCH' + sh 'python3 ./evaluation.py' + archiveArtifacts 'TRAIN_winequality-red.csv' + sh "ls -la" } } } post { - success { - archiveArtifacts artifacts: 'metrics.*', followSymlinks: false - } always { - emailext body: "${currentBuild.currentResult}", subject: 's444507-evaluation', to: 'e19191c5.uam.onmicrosoft.com@emea.teams.ms' + emailext body: "${currentBuild.currentResult}", subject: '444354-evaluation build status', to: 'e19191c5.uam.onmicrosoft.com@emea.teams.ms' } } } \ No newline at end of file