From 278daac14c718af0e81964aaf5b653a694a2a8dd Mon Sep 17 00:00:00 2001
From: Wojciech Jarmosz <wojciech@jarmosz.com>
Date: Sat, 27 Mar 2021 22:37:36 +0100
Subject: [PATCH] Dodanie skryptu + konfiguracji Jenkins - zadanie 5,6,7

---
 Jenkinsfile   | 16 ++++++++++++----
 data_stats.sh | 12 ++++++++++++
 2 files changed, 24 insertions(+), 4 deletions(-)
 create mode 100755 data_stats.sh

diff --git a/Jenkinsfile b/Jenkinsfile
index ffd6cc0..bc47fa3 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,10 +1,18 @@
 pipeline {
     agent any 
     stages {
-        stage('Stage 1') {
-            steps {
-                echo 'Hello world!' 
+        node {
+            stage('Clone repo + run bash script') { 
+                withEnv(["KAGGLE_USERNAME=${params.KAGGLE_USERNAME}",
+                "KAGGLE_KEY=${params.KAGGLE_KEY}", "CUTOFF=${params.CUTOFF}" ]) {
+                    checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'f746c8e0-e974-4758-838d-1920a7b0e3dc', url: 'https://git.wmi.amu.edu.pl/s434704/ium_434704']]])
+                    sh "chmod 777 ./data_sets.sh"
+                    sh "./data_sets.sh"
+                    archiveArtifacts "MoviesOnStreamingPlatforms_updated.dev"
+                    archiveArtifacts "MoviesOnStreamingPlatforms_updated.test"
+                    archiveArtifacts "MoviesOnStreamingPlatforms_updated.train"
+                }
             }
-        }
+        }       
     }
 }
\ No newline at end of file
diff --git a/data_stats.sh b/data_stats.sh
new file mode 100755
index 0000000..b1f5360
--- /dev/null
+++ b/data_stats.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+pip install kaggle
+kaggle datasets download ruchi798/movies-on-netflix-prime-video-hulu-and-disney --unzip
+sed -i '/^$/d' ./MoviesOnStreamingPlatforms_updated.csv
+
+shuf ./MoviesOnStreamingPlatforms_updated.csv | tail -n +"${CUTOFF}" > ./MoviesOnStreamingPlatforms_updated.shuf
+head -n 1674 ./MoviesOnStreamingPlatforms_updated.shuf > ./MoviesOnStreamingPlatforms_updated.test
+head -n 3348 ./MoviesOnStreamingPlatforms_updated.shuf | tail -n 1674 > ./MoviesOnStreamingPlatforms_updated.dev
+tail -n +3349 ./MoviesOnStreamingPlatforms_updated.shuf > ./MoviesOnStreamingPlatforms_updated.train
+rm ./MoviesOnStreamingPlatforms_updated.shuf
+rm ./MoviesOnStreamingPlatforms_updated.csv
+wc -l ./MoviesOnStreamingPlatforms_updated*
\ No newline at end of file