From 912599ede470e7b40a7ce76c3734b19caff7ddd5 Mon Sep 17 00:00:00 2001 From: Kacper Dudzic Date: Sat, 2 Apr 2022 15:42:02 +0200 Subject: [PATCH 01/19] Update 'Dockerfile' --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8d205c1..f770302 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,6 @@ RUN pip3 install --user numpy RUN pip3 install --user sklearn COPY ./download_dataset.sh ./ -COPY ./process_data.py ./ +COPY ./process_dataset.py ./ RUN chmod u+x ./download_dataset.sh ./process_data.py \ No newline at end of file From cdb9c846186797d62d73cc0678a306a2028ab8e8 Mon Sep 17 00:00:00 2001 From: Kacper Dudzic Date: Sat, 2 Apr 2022 15:42:56 +0200 Subject: [PATCH 02/19] Update 'Dockerfile' --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f770302..124244d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,4 +14,4 @@ RUN pip3 install --user sklearn COPY ./download_dataset.sh ./ COPY ./process_dataset.py ./ -RUN chmod u+x ./download_dataset.sh ./process_data.py \ No newline at end of file +RUN chmod u+x ./download_dataset.sh ./process_dataset.py \ No newline at end of file From f59d74671477321389f25ea8d02295397a0ea4b4 Mon Sep 17 00:00:00 2001 From: Kacper Dudzic Date: Sat, 2 Apr 2022 15:44:40 +0200 Subject: [PATCH 03/19] Update 'Jenkinsfile' --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 82f4c17..15807bd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -20,6 +20,7 @@ pipeline { stages { stage('Stage 1') { steps { + sh 'chmod u+x ./download_dataset.sh ./process_dataset.py' echo 'Downloading dataset...' sh './download_dataset.sh' echo 'Dataset downloaded' From bf20f27daf8b489331870d906f6cd11916f28ddf Mon Sep 17 00:00:00 2001 From: Kacper Dudzic Date: Sat, 2 Apr 2022 15:52:30 +0200 Subject: [PATCH 04/19] Update 'Dockerfile' --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 124244d..16227cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,12 +4,12 @@ WORKDIR /ium RUN apt update && apt install -y python3-pip unzip -RUN pip3 install --user kaggle +RUN pip3 install kaggle RUN echo "export PATH=\"\$HOME/.local/bin:\$PATH\"" >> ~/.bashrc -RUN pip3 install --user pandas -RUN pip3 install --user numpy -RUN pip3 install --user sklearn +RUN pip3 install pandas +RUN pip3 install numpy +RUN pip3 install sklearn COPY ./download_dataset.sh ./ COPY ./process_dataset.py ./ From f148a3f24f04af7e7c0bf1f0f89985e783eb5211 Mon Sep 17 00:00:00 2001 From: Kacper Dudzic Date: Sat, 2 Apr 2022 15:56:52 +0200 Subject: [PATCH 05/19] Update 'process_dataset.py' --- process_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process_dataset.py b/process_dataset.py index 2384cc9..9c2857f 100644 --- a/process_dataset.py +++ b/process_dataset.py @@ -15,7 +15,7 @@ lego['prod_id'] = lego['prod_id'].apply(np.int64) # wglad, statystyki print(lego) -print(lego.describe(include='all')) +print(lego.describe(include='all')).encode('utf-8') # pierwszy podzial, wydzielamy zbior treningowy lego_train, lego_rem = train_test_split(lego, train_size=0.8, random_state=1) From a2d5b214a786dd10871c55c29d4b35a2dcdaab21 Mon Sep 17 00:00:00 2001 From: Kacper Dudzic Date: Sat, 2 Apr 2022 15:57:11 +0200 Subject: [PATCH 06/19] Update 'process_dataset.py' --- process_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process_dataset.py b/process_dataset.py index 9c2857f..0a45d62 100644 --- a/process_dataset.py +++ b/process_dataset.py @@ -15,7 +15,7 @@ lego['prod_id'] = lego['prod_id'].apply(np.int64) # wglad, statystyki print(lego) -print(lego.describe(include='all')).encode('utf-8') +print(lego.describe(include='all').encode('utf-8')) # pierwszy podzial, wydzielamy zbior treningowy lego_train, lego_rem = train_test_split(lego, train_size=0.8, random_state=1) From d45c1c00b0c87f71834ae3f58e7973b73dfc5deb Mon Sep 17 00:00:00 2001 From: Kacper Dudzic Date: Sat, 2 Apr 2022 16:02:15 +0200 Subject: [PATCH 07/19] Update 'Jenkinsfile' --- Jenkinsfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 15807bd..28af142 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -20,6 +20,8 @@ pipeline { stages { stage('Stage 1') { steps { + withEnv(["KAGGLE_USERNAME=${params.KAGGLE_USERNAME}", + "KAGGLE_KEY=${params.KAGGLE_KEY}" ]) { sh 'chmod u+x ./download_dataset.sh ./process_dataset.py' echo 'Downloading dataset...' sh './download_dataset.sh' @@ -27,6 +29,7 @@ pipeline { echo 'Processing dataset...' sh 'python3 process_dataset.py' echo 'Dataset processed' + } } } } From 00b3572ff4a819ee3f116c8dd47424c2a8d8ab29 Mon Sep 17 00:00:00 2001 From: Kacper Dudzic Date: Sat, 2 Apr 2022 16:06:56 +0200 Subject: [PATCH 08/19] Update 'Jenkinsfile' --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 28af142..dc68d69 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,6 +27,7 @@ pipeline { sh './download_dataset.sh' echo 'Dataset downloaded' echo 'Processing dataset...' + sh 'export PYTHONIOENCODING=UTF-8' sh 'python3 process_dataset.py' echo 'Dataset processed' } From 882024034191ba7bef427c7580af2a82644b02de Mon Sep 17 00:00:00 2001 From: Kacper Dudzic Date: Sat, 2 Apr 2022 16:07:21 +0200 Subject: [PATCH 09/19] Update 'process_dataset.py' --- process_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process_dataset.py b/process_dataset.py index 0a45d62..2384cc9 100644 --- a/process_dataset.py +++ b/process_dataset.py @@ -15,7 +15,7 @@ lego['prod_id'] = lego['prod_id'].apply(np.int64) # wglad, statystyki print(lego) -print(lego.describe(include='all').encode('utf-8')) +print(lego.describe(include='all')) # pierwszy podzial, wydzielamy zbior treningowy lego_train, lego_rem = train_test_split(lego, train_size=0.8, random_state=1) From dcbfbd957a902bec9f914b5772e8a388d5be8bf1 Mon Sep 17 00:00:00 2001 From: Kacper Dudzic Date: Sat, 2 Apr 2022 16:10:53 +0200 Subject: [PATCH 10/19] Update 'Jenkinsfile' --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index dc68d69..00b0fd5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,6 +23,7 @@ pipeline { withEnv(["KAGGLE_USERNAME=${params.KAGGLE_USERNAME}", "KAGGLE_KEY=${params.KAGGLE_KEY}" ]) { sh 'chmod u+x ./download_dataset.sh ./process_dataset.py' + chmod 'ugo+rwx ~/.kaggle/' echo 'Downloading dataset...' sh './download_dataset.sh' echo 'Dataset downloaded' From a10b61dcc13c1e7f3385571d054735706dc15893 Mon Sep 17 00:00:00 2001 From: Kacper Dudzic Date: Sat, 2 Apr 2022 16:12:02 +0200 Subject: [PATCH 11/19] Update 'Jenkinsfile' --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 00b0fd5..b51fa8b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,7 +23,7 @@ pipeline { withEnv(["KAGGLE_USERNAME=${params.KAGGLE_USERNAME}", "KAGGLE_KEY=${params.KAGGLE_KEY}" ]) { sh 'chmod u+x ./download_dataset.sh ./process_dataset.py' - chmod 'ugo+rwx ~/.kaggle/' + sh 'chmod ugo+rwx ~/.kaggle/' echo 'Downloading dataset...' sh './download_dataset.sh' echo 'Dataset downloaded' From 8ff5416fadb0173c954185f33ae736bf65409dca Mon Sep 17 00:00:00 2001 From: Kacper Dudzic Date: Sat, 2 Apr 2022 16:13:11 +0200 Subject: [PATCH 12/19] Update 'Jenkinsfile' --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b51fa8b..ca699e2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,7 +23,7 @@ pipeline { withEnv(["KAGGLE_USERNAME=${params.KAGGLE_USERNAME}", "KAGGLE_KEY=${params.KAGGLE_KEY}" ]) { sh 'chmod u+x ./download_dataset.sh ./process_dataset.py' - sh 'chmod ugo+rwx ~/.kaggle/' + sh 'chmod ugo+rwx /root/.kaggle/' echo 'Downloading dataset...' sh './download_dataset.sh' echo 'Dataset downloaded' From e744f600aa4a6ff95e2fb0e190226424000088ed Mon Sep 17 00:00:00 2001 From: Kacper Dudzic Date: Sat, 2 Apr 2022 16:14:23 +0200 Subject: [PATCH 13/19] Update 'Dockerfile' --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 16227cd..bd76e36 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,4 @@ RUN pip3 install numpy RUN pip3 install sklearn COPY ./download_dataset.sh ./ -COPY ./process_dataset.py ./ - -RUN chmod u+x ./download_dataset.sh ./process_dataset.py \ No newline at end of file +COPY ./process_dataset.py ./ \ No newline at end of file From 802e49a3dd0ad7e2c705bfe1fff3e7edb7b4bc62 Mon Sep 17 00:00:00 2001 From: Kacper Dudzic Date: Sat, 2 Apr 2022 16:15:47 +0200 Subject: [PATCH 14/19] Update 'download_dataset.sh' --- download_dataset.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/download_dataset.sh b/download_dataset.sh index bae3c01..587a653 100644 --- a/download_dataset.sh +++ b/download_dataset.sh @@ -1,3 +1,3 @@ #!/bin/sh -kaggle datasets download -d mterzolo/lego-sets +KAGGLE_USERNAME=${params.KAGGLE_USERNAME} KAGGLE_KEY=${params.KAGGLE_KEY} kaggle datasets download -d mterzolo/lego-sets unzip -o lego-sets.zip From e795acb7b6186ac179480b0ff06078fea0ec999f Mon Sep 17 00:00:00 2001 From: Kacper Dudzic Date: Sat, 2 Apr 2022 16:16:47 +0200 Subject: [PATCH 15/19] Update 'Jenkinsfile' --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ca699e2..dc68d69 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,7 +23,6 @@ pipeline { withEnv(["KAGGLE_USERNAME=${params.KAGGLE_USERNAME}", "KAGGLE_KEY=${params.KAGGLE_KEY}" ]) { sh 'chmod u+x ./download_dataset.sh ./process_dataset.py' - sh 'chmod ugo+rwx /root/.kaggle/' echo 'Downloading dataset...' sh './download_dataset.sh' echo 'Dataset downloaded' From 2a95847872094b3f9e83b06c92b94a5724d26c21 Mon Sep 17 00:00:00 2001 From: Kacper Dudzic Date: Sat, 2 Apr 2022 16:18:39 +0200 Subject: [PATCH 16/19] Update 'download_dataset.sh' --- download_dataset.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/download_dataset.sh b/download_dataset.sh index 587a653..a55d8f1 100644 --- a/download_dataset.sh +++ b/download_dataset.sh @@ -1,3 +1,3 @@ -#!/bin/sh +#!/bin/bash KAGGLE_USERNAME=${params.KAGGLE_USERNAME} KAGGLE_KEY=${params.KAGGLE_KEY} kaggle datasets download -d mterzolo/lego-sets unzip -o lego-sets.zip From 41bcce7ddabbb14dbec91d2f14a1a8d56899ef24 Mon Sep 17 00:00:00 2001 From: Kacper Dudzic Date: Sat, 2 Apr 2022 16:38:45 +0200 Subject: [PATCH 17/19] Update 'download_dataset.sh' --- download_dataset.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/download_dataset.sh b/download_dataset.sh index a55d8f1..bae3c01 100644 --- a/download_dataset.sh +++ b/download_dataset.sh @@ -1,3 +1,3 @@ -#!/bin/bash -KAGGLE_USERNAME=${params.KAGGLE_USERNAME} KAGGLE_KEY=${params.KAGGLE_KEY} kaggle datasets download -d mterzolo/lego-sets +#!/bin/sh +kaggle datasets download -d mterzolo/lego-sets unzip -o lego-sets.zip From be10e1759b15029c73c0fd801177b158089557af Mon Sep 17 00:00:00 2001 From: Kacper Dudzic Date: Sat, 2 Apr 2022 16:40:30 +0200 Subject: [PATCH 18/19] Update 'Jenkinsfile' --- Jenkinsfile | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index dc68d69..7897f1b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,24 +13,17 @@ pipeline { ) } agent { - dockerfile { - additionalBuildArgs "--build-arg KAGGLE_USERNAME=${params.KAGGLE_USERNAME} --build-arg KAGGLE_KEY=${params.KAGGLE_KEY}" + dockerfile true } } stages { stage('Stage 1') { steps { - withEnv(["KAGGLE_USERNAME=${params.KAGGLE_USERNAME}", - "KAGGLE_KEY=${params.KAGGLE_KEY}" ]) { - sh 'chmod u+x ./download_dataset.sh ./process_dataset.py' - echo 'Downloading dataset...' - sh './download_dataset.sh' + sh 'chmod u+x ./process_dataset.py' echo 'Dataset downloaded' echo 'Processing dataset...' - sh 'export PYTHONIOENCODING=UTF-8' sh 'python3 process_dataset.py' echo 'Dataset processed' - } } } } From c6d482c70fa4722f88869b9b3a56490511e57266 Mon Sep 17 00:00:00 2001 From: Kacper Dudzic Date: Sat, 2 Apr 2022 16:41:37 +0200 Subject: [PATCH 19/19] Update 'Dockerfile' --- Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index bd76e36..4928c9a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,14 +2,11 @@ FROM ubuntu:latest WORKDIR /ium -RUN apt update && apt install -y python3-pip unzip - -RUN pip3 install kaggle -RUN echo "export PATH=\"\$HOME/.local/bin:\$PATH\"" >> ~/.bashrc +RUN apt update && apt install -y python3-pip RUN pip3 install pandas RUN pip3 install numpy RUN pip3 install sklearn -COPY ./download_dataset.sh ./ +COPY ./lego_sets.csv ./ COPY ./process_dataset.py ./ \ No newline at end of file