From 00e3446c93fe7cafb85a30255745a9780edd408a Mon Sep 17 00:00:00 2001
From: s434765 <karolinaoparczyk@gmail.com>
Date: Sat, 27 Mar 2021 21:39:10 +0100
Subject: [PATCH] script fix

---
 Jenkinsfile | 9 ++++++---
 get_data.sh | 7 +------
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 631a49e..baed021 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -9,8 +9,11 @@ node {
                             trim: false),
                         password(defaultValue: '',
                             description: 'Kaggle token',
-                            name: 'KAGGLE_KEY')
-
+                            name: 'KAGGLE_KEY'),
+string(defaultV         string(defaultValue: '5000',
+                            description: 'Data cutoff',
+                            name: 'CUTOFF',
+                            trim: false),
                             ])
                 ]
                 )
@@ -22,7 +25,7 @@ node {
                     sh '''
                         #!/usr/bin/env bash
                         chmod 777 get_data_simple.sh
-                        ./get_data_simple.sh
+                        ./get_data_simple.sh ${params.CUTOFF}"
                     '''
                     archiveArtifacts "data_dev"
                     archiveArtifacts "data_shuf"
diff --git a/get_data.sh b/get_data.sh
index 8ab956d..2145aae 100755
--- a/get_data.sh
+++ b/get_data.sh
@@ -5,15 +5,10 @@ if kaggle datasets download -d sgonkaggle/youtube-trend-with-subscriber && unzip
     COUNT=$(wc -l "USvideos_modified.csv")
     echo "${COUNT}"
     head -n -1 "USvideos_modified.csv" | shuf > "data_shuf"
-    head -n 544 "data_shuf" > "data_test"
-    head -n 1088 "data_shuf" | tail -n 544 > "data_dev"
-    head -n +1089 "data_shuf" > "data_train"
+    head -n "$1" "data_shuf" > "data_train"
     echo "Shuffled dataset"
     wc -l "data_shuf"
     echo "Test dataset"
-    wc -l "data_test"
-    echo "Dev dataset"
-    wc -l "data_dev"
     echo "Train dataset"
     wc -l "data_train"
     python main.py USvideos_modified.csv