From 28581651daecc01f4afbdec3d0203474758b0eba Mon Sep 17 00:00:00 2001 From: Jan Nowak Date: Sat, 10 Apr 2021 18:00:26 +0200 Subject: [PATCH] Dockerfile --- Dockerfile | 18 +++++++++++ Jenkinsfile | 86 +++++++++++++++++++++++++---------------------------- README.md | 4 +-- skrypt.sh | 4 +-- 4 files changed, 62 insertions(+), 50 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a693aef --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +# Nasz obraz będzie dzidziczył z obrazu Ubuntu w wersji latest +FROM ubuntu:latest + +# Instalujemy niezbędne zależności. Zwróć uwagę na flagę "-y" (assume yes) +RUN apt update +RUN apt install -y python3 python3-pip dos2unix +RUN pip3 install kaggle +RUN apt install -y unzip + +# Stwórzmy w kontenerze (jeśli nie istnieje) katalog /app i przejdźmy do niego (wszystkie kolejne polecenia RUN, CMD, ENTRYPOINT, COPY i ADD będą w nim wykonywane) +WORKDIR /app + +# Skopiujmy nasz skrypt do katalogu /app w kontenerze +COPY ./skrypt.sh ./ +RUN chmod +x skrypt.sh +RUN dos2unix skrypt.sh + +CMD ./skrypt.sh 0 diff --git a/Jenkinsfile b/Jenkinsfile index 2bab0aa..6ba4e6b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,46 +1,40 @@ -pipeline { - agent any - parameters { - string( - defaultValue: '0', - description: 'Umożliwia zdefiniowanie wielkości odcięcia zbioru danych.', - name: 'CUTOFF', - trim: false - ) - string( - defaultValue: 'rokoch', - description: 'Kaggle username', - name: 'KAGGLE_USERNAME', - trim: false - ) - password( - defaultValue: '', - description: 'Kaggle token taken from kaggle.json file, as described in https://github.com/Kaggle/kaggle-api#api-credentials', - name: 'KAGGLE_KEY' - ) - } - stages { - stage('checkout') { - steps { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[url: 'https://git.wmi.amu.edu.pl/s426206/ium_426206.git']]]) - } - } - - stage('sh') { - steps { - withEnv(["KAGGLE_USERNAME=${params.KAGGLE_USERNAME}", - "KAGGLE_KEY=${params.KAGGLE_KEY}"]) { - sh "chmod +x skrypt.sh" - sh "./skrypt.sh ${params.CUTOFF} | tee output.txt" - } - } - } - - stage('end') { - steps { - //Zarchiwizuj wynik - archiveArtifacts 'output.txt,Global_Superstore22.csv,Global_Superstore2.csv.dev,Global_Superstore2.csv.test,Global_Superstore2.csv.train' - } - } - } -} \ No newline at end of file +pipeline { + agent any + parameters { + string( + defaultValue: '0', + description: 'Umożliwia zdefiniowanie wielkości odcięcia zbioru danych.', + name: 'CUTOFF', + trim: false + ) + string( + defaultValue: 'rokoch', + description: 'Kaggle username', + name: 'KAGGLE_USERNAME', + trim: false + ) + password( + defaultValue: '', + description: 'Kaggle token taken from kaggle.json file, as described in https://github.com/Kaggle/kaggle-api#api-credentials', + name: 'KAGGLE_KEY' + ) + } + stages { + stage('checkout') { + steps { + checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[url: 'https://git.wmi.amu.edu.pl/s426206/ium_426206.git']]]) + } + } + + stage('docker') { + agent { dockerfile true } + steps { + withEnv(["KAGGLE_USERNAME=${params.KAGGLE_USERNAME}", + "KAGGLE_KEY=${params.KAGGLE_KEY}"]) { + sh "chmod +x skrypt.sh" + sh "./skrypt.sh ${params.CUTOFF} | tee output.txt" + } + } + } + } +} diff --git a/README.md b/README.md index 85fc543..5c8b3a9 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ -# ium_426206 - +# ium_426206 + Inżynieria Uczenia Maszynowego \ No newline at end of file diff --git a/skrypt.sh b/skrypt.sh index 7b0b3c0..25e4973 100644 --- a/skrypt.sh +++ b/skrypt.sh @@ -1,7 +1,7 @@ #!/bin/bash kaggle datasets download -d apoorvaappz/global-super-store-dataset unzip -o global-super-store-dataset.zip -if [ $1 = "0" ]; then +if [[ $1 = "0" ]]; then CUTOFF=51291 cp Global_Superstore2.csv Global_Superstore22.csv else @@ -16,4 +16,4 @@ head -n $((prop1*2)) Global_Superstore2.csv.shuf | tail -n $prop1 > Global_Super tail -n +$((prop1*2+1)) Global_Superstore2.csv.shuf > Global_Superstore2.csv.train rm Global_Superstore2.csv.shuf #Sprawdźmy, czy wielkości się zgadzają: -wc -l Global_Superstore2* \ No newline at end of file +wc -l Global_Superstore2*