From fe61b195fac79ff313f3b9c1ec112d8605af4352 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cezary=20Ga=C5=82=C4=85zkiewicz?= <cgalazkiewicz@gmail.com>
Date: Mon, 28 Mar 2022 00:52:40 +0200
Subject: [PATCH] Zad 03.Jenkins - zapisanie zbioru danych

---
 download.sh | 2 +-
 jenkinsfile | 7 ++++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/download.sh b/download.sh
index 83007a3..497211a 100644
--- a/download.sh
+++ b/download.sh
@@ -1,7 +1,7 @@
 kaggle datasets download -d csafrit2/steel-industry-energy-consumption --force
 unzip -o steel-industry-energy-consumption.zip
 
-tail -n +2 Steel_industry_data.csv | cut -d, -f8 --complement | shuf > steel_industry_data_shuffled.csv
+head -n $CUTOFF Steel_industry_data.csv | tail -n +2 | cut -d, -f8 --complement | shuf > steel_industry_data_shuffled.csv
 number_of_lines=$(wc -l steel_industry_data_shuffled.csv | awk '{print $1}')
 test_len=$((number_of_lines/10))
 dev_len=$((number_of_lines/10))
diff --git a/jenkinsfile b/jenkinsfile
index e1032d6..097b4b1 100644
--- a/jenkinsfile
+++ b/jenkinsfile
@@ -12,6 +12,10 @@ pipeline {
                  description: 'Kaggle token taken from kaggle.json file, as described in https://github.com/Kaggle/kaggle-api#api-credentials',
                  name: 'KAGGLE_KEY'
              )
+        string(
+                defaultValue: '35041',
+                description: 'CUTOFF parameter',
+                name: 'CUTOFF')
     }
     environment {
         KAGGLE_USERNAME="$params.KAGGLE_USERNAME"
@@ -32,7 +36,8 @@ pipeline {
             steps {
                     sh 'chmod u+x ./download.sh'
                 withEnv(["KAGGLE_USERNAME=${params.KAGGLE_USERNAME}",
-                         "KAGGLE_KEY=${params.KAGGLE_KEY}"]) {
+                         "KAGGLE_KEY=${params.KAGGLE_KEY}",
+                         "CUTOFF=${params.CUTOFF}"]) {
                     sh "echo KAGGLE_USERNAME: $KAGGLE_USERNAME"
                     sh "./download.sh"
                     archiveArtifacts artifacts: "steel_industry_data_test.csv, steel_industry_data_dev.csv, steel_industry_data_train.csv"