From c7ff03a1c917ed7d79ff3131b817350cee9bde18 Mon Sep 17 00:00:00 2001 From: Mateusz Kuc Date: Fri, 30 Jun 2023 14:08:58 +0200 Subject: [PATCH] updated to combine docker with jenkins --- Jenkinsfile1 | 68 ---------------------------------------- JenkinsfileCreateDataset | 63 +++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 68 deletions(-) delete mode 100644 Jenkinsfile1 create mode 100644 JenkinsfileCreateDataset diff --git a/Jenkinsfile1 b/Jenkinsfile1 deleted file mode 100644 index 6cb4e76..0000000 --- a/Jenkinsfile1 +++ /dev/null @@ -1,68 +0,0 @@ -pipeline { - agent any - - parameters{ - string( - defaultValue: 'mattkrawl', - description: 'Kaggle username', - name: 'KAGGLE_USERNAME', - trim: false - ) - password( - defaultValue: '8e6132e627c5c176f7e19d880270d22e', - description: 'Kaggle token taken from kaggle.json file, as described in https://github.com/Kaggle/kaggle-api#api-credentials', - name: 'KAGGLE_KEY' - ) - string( - defaultValue: '100', - description: 'CUTOFF', - name: 'CUTOFF', - trim: false - ) - } - stages { - stage('clear_all') { - steps { - sh 'rm -rf *' - } - } - - stage('Build'){ - steps{ - //cloning github - sh 'git clone https://git.wmi.amu.edu.pl/s458023/ium_458023' - - //running Build - withEnv(["KAGGLE_USERNAME=${params.KAGGLE_USERNAME}", - "KAGGLE_KEY=${params.KAGGLE_KEY}" ]) { - // downloading and unzipping kaggle dataset - sh 'kaggle datasets download -d arnabchaki/data-science-salaries-2023' - sh 'unzip data-science-salaries-2023.zip -d ./ium_458023' - - // removing zip file - sh 'rm data-science-salaries-2023.zip' - sh 'ls -a' - sh 'ls -a ./ium_458023' - - } - } - } - stage('Docker'){ - agent{ - dockerfile{ - filename 'Dockerfile' - dir 'ium_458023' - reuseNode true - } - } - steps{ - sh 'ls -a' - sh 'python ium_458023/create-dataset.py' - archiveArtifacts 'salaries_test.csv' - archiveArtifacts 'salaries_dev.csv' - archiveArtifacts 'salaries_train.csv' - } - } - - } -} \ No newline at end of file diff --git a/JenkinsfileCreateDataset b/JenkinsfileCreateDataset new file mode 100644 index 0000000..b7a5f82 --- /dev/null +++ b/JenkinsfileCreateDataset @@ -0,0 +1,63 @@ +pipeline { + agent { + dockerfile { + filename 'lab4.dockerfile' + reuseNode true + } + } + + + parameters{ + string( + defaultValue: 'mattkrawl', + description: 'Kaggle username', + name: 'KAGGLE_USERNAME', + trim: false + ) + password( + defaultValue: 'c21878a7463faa44361330ffbcea68a6', + description: 'Kaggle token taken from kaggle.json file, as described in https://github.com/Kaggle/kaggle-api#api-credentials', + name: 'KAGGLE_KEY' + ) + string( + defaultValue: '1000', + description: 'CUTOFF', + name: 'CUTOFF', + trim: false + ) + } + stages { + stage('Checkout') { + steps { + checkout scmGit( + branches: [[name: '*/master']], + extensions: [cleanBeforeChceckout()], + userRemoteConfigs: [[url: 'https://git.wmi.amu.edu.pl/s458023/ium_458023.git']] + ) + } + } + + stage('Prepare data'){ + steps{ + //running Build + withEnv(["KAGGLE_USERNAME=${params.KAGGLE_USERNAME}", + "KAGGLE_KEY=${params.KAGGLE_KEY}" ]) { + // downloading and unzipping kaggle dataset + sh 'echo "{\"username\":\"$KAGGLE_USERNAME\",\"key\":\"$KAGGLE_KEY\"}" > /.kaggle/kaggle.json' + sh './get-data.sh' + sh 'python3 prepare_dataset.py' + + } + } + } + stage('Archive artifacts'){ + steps{ + withEnv(["CUTOFF=${params.CUTOFF}"]){ + archiveArtifacts 'wines_test.csv' + archiveArtifacts 'wines_dev.csv' + archiveArtifacts 'wines_train.csv' + } + } + } + } +} \ No newline at end of file