diff --git a/Dockerfile b/Dockerfile index df944ef..6a23d5c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ COPY . . ARG KAGGLE_USERNAME ARG KAGGLE_KEY -RUN chmod u+x ./script-download.py +RUN chmod u+x ./stats-docker.sh RUN chmod u+x ./script-stats.py # RUN ./download.sh 117928 diff --git a/Jenkinsfile-docker b/Jenkinsfile-docker index 51bc47a..e685d49 100644 --- a/Jenkinsfile-docker +++ b/Jenkinsfile-docker @@ -20,13 +20,13 @@ pipeline { } agent { dockerfile{ - additionalBuildArgs '--build-arg KAGGLE_USERNAME="$KAGGLE_USERNAME" --build-arg KAGGLE_KEY="$KAGGLE_KEY" --no-cache=true' + additionalBuildArgs '--build-arg KAGGLE_USERNAME="$KAGGLE_USERNAME" --build-arg KAGGLE_KEY="$KAGGLE_KEY" -t s444507_create_dataset_image' } } stages { stage('Prepare dataset') { steps { - sh 'python3 ./script-stats.py' + sh './stats-docker.sh' } } } diff --git a/Jenkinsfile-stats b/Jenkinsfile-stats new file mode 100644 index 0000000..1bf1bd7 --- /dev/null +++ b/Jenkinsfile-stats @@ -0,0 +1,38 @@ +pipeline { + parameters { + string( + defaultValue: 'heatedboss2', + description: 'Kaggle username', + name: 'KAGGLE_USERNAME', + trim: false + ) + password( + defaultValue: '', + description: 'Kaggle token', + name: 'KAGGLE_KEY' + ) + string( + defaultValue: '117928', + description: 'Cutoff', + name: 'CUTOFF', + trim: false + ) + } + agent { + dockerfile{ + additionalBuildArgs '--build-arg KAGGLE_USERNAME="$KAGGLE_USERNAME" --build-arg KAGGLE_KEY="$KAGGLE_KEY" -t s444507_create_dataset_image' + } + } + stages { + stage('Prepare dataset') { + steps { + sh 'python3 ./script-stats.py' + } + } + } + post { + success { + archiveArtifacts artifacts: 'Car_Prices_Poland_Kaggle_*', followSymlinks: false + } + } +} \ No newline at end of file diff --git a/script-download.py b/script-download.py index c6286ae..8f0145d 100644 --- a/script-download.py +++ b/script-download.py @@ -9,7 +9,7 @@ import numpy as np def unzip_package(): """Unzip dataset""" print('Unzipping dataset...') - os.system('unzip -o car-prices-poland.zip') + os.system('unzip -o ./car-prices-poland.zip') print('Dataset unzipped') print('Removing .zip file...') os.system('rm ./car-prices-poland.zip') diff --git a/stats-docker.sh b/stats-docker.sh new file mode 100644 index 0000000..ceb586e --- /dev/null +++ b/stats-docker.sh @@ -0,0 +1,12 @@ + +echo 'Total elements in Car Prices Poland dataset:' +wc -l ./Car_Prices_Poland_Kaggle.csv + +echo 'Total elements in train dataset:' +wc -l ./Car_Prices_Poland_Kaggle_train.csv + +echo 'Total elements in test dataset:' +wc -l ./Car_Prices_Poland_Kaggle_test.csv + +echo 'Total elements in dev dataset:' +wc -l ./Car_Prices_Poland_Kaggle_dev.csv