Added new jenkinsfile and kuggle download script

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/ium_464979.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/ium_464979.iml" filepath="$PROJECT_DIR$/.idea/ium_464979.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

Jenkinsfile

@@ -1,28 +1,32 @@
 pipeline {
     agent any
-   //Definijuemy parametry, kt<6B>re b<>dzie mo<6D>na poda<64> podczas wywo<77>ywania zadania
+
     parameters {
-     string (
-         defaultValue: 'Hello World!',
-         description: 'Tekst, kt<6B>rym chcesz przywita<74> <20>wiat',
-         name: 'INPUT_TEXT',
-         trim: false
-        )
+	properties([
+    		parameters([
+        		string(name: 'KAGGLE_DATASET_ID', defaultValue: '', description: 'Kaggle dataset'),
+        		string(name: 'REPO_URL', defaultValue: '', description: 'Git Url'),
+        		string(name: 'CUTOFF', defaultValue: '10000', description: 'Liczba wierszy do obcięcia ze zbioru danych')
+    		])
+	])
     }
+
     stages {
-      stage('Hello') {
+        stage('Clone Repository') {
             steps {
-            //Wypisz warto<74><6F> parametru w konsoli (To nie jest polecenie bash, tylko groovy!)
-            echo "INPUT_TEXT: promoscan"
-            //Wywo<77>aj w konsoli komend<6E> "figlet", kt<6B>ra generuje ASCI-art
-            sh "figlet \"promoscan\" | tee output.txt"
+                git url: "${params.REPO_URL}"
             }
         }
-      stage('Goodbye!') {
+
+        stage('Download, Process, and Split Dataset') {
             steps {
-            echo 'Goodbye!'
-            //Zarchiwizuj wynik
-            archiveArtifacts 'output.txt'
+                withEnv([
+                    "KAGGLE_USERNAME=${env.KAGGLE_USERNAME}", 
+                    "KAGGLE_KEY=${env.KAGGLE_KEY}"
+                ]) {
+                    sh "bash ./kuggle_download.sh ${params.KAGGLE_DATASET_ID} ${params.CUTOFF}"
+                    sh './kuggle_download.sh' 
+                }
             }
         }
     }

kuggle_download.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+pip install kaggle
+
+kaggle datasets download -d $DATASET_ID
+
+unzip -o $DATASET_FILE
+
+shuf $DATASET_FILE > shuffled_dataset.csv
+
+split -l 80000 shuffled_dataset.csv train.csv
+split -l 10000 train.csv dev.csv
+mv shuffled_dataset.csv test.csv
+
+head -n 1000 train.csv > train_head.csv
+tail -n 1000 train.csv > train_tail.csv
+
+if [ -n "$CUTOFF" ]; then
+  head -n "$CUTOFF" train.csv > train_cutoff.csv
+fi
+
+tar -czf artifacts.tar.gz train.csv dev.csv test.csv train_head.csv train_tail.csv train_cutoff.csv
+
+rm $DATASET_FILE shuffled_dataset.csv
+
+echo "artifacts.tar.gz"
+