Added sh script and done to 7-th task.

This commit is contained in:
Jan Nowak 2021-03-28 22:00:27 +02:00
parent d9518ea09c
commit bf3280423d
2 changed files with 57 additions and 2 deletions

43
Jenkinsfile vendored
View File

@ -1,10 +1,49 @@
pipeline {
agent any
stages {
stage('Stage 1') {
stage('Preparation') {
properties([
parameters([
string(
defaultValue: '0',
description: 'Umożliwia zdefiniowanie wielkości odcięcia zbioru danych.',
name: 'CUTOFF',
trim: false
),
string(
defaultValue: 'rokoch',
description: 'Kaggle username',
name: 'KAGGLE_USERNAME',
trim: false
),
password(
defaultValue: '',
description: 'Kaggle token taken from kaggle.json file, as described in https://github.com/Kaggle/kaggle-api#api-credentials',
name: 'KAGGLE_KEY'
)
])
])
}
stage('checkout') {
steps {
echo 'Hello world!'
checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[url: 'https://git.wmi.amu.edu.pl/s426206/ium_426206.git']]])
}
}
stage('sh') {
steps {
withEnv(["KAGGLE_USERNAME=${params.KAGGLE_USERNAME}",
"KAGGLE_KEY=${params.KAGGLE_KEY}") {
sh 'echo KAGGLE_USERNAME: $KAGGLE_USERNAME'
sh 'kaggle datasets list'
sh './skrypt.sh params.CUTOFF | tee output.txt'
}
}
}
stage('end') {
//Zarchiwizuj wynik
archiveArtifacts 'output.txt'
}
}
}

16
skrypt.sh Normal file
View File

@ -0,0 +1,16 @@
#!/bin/bash
kaggle datasets download -d apoorvaappz/global-super-store-dataset
unzip global-super-store-dataset.zip
cat Global_Superstore2.csv | shuf > Global_Superstore2.csv.shuf
if [ $1="0" ]; then
CUTOFF=51291
else
CUTOFF=$1
fi
prop1=$((CUTOFF/5))
head -n $prop1 Global_Superstore2.csv.shuf > Global_Superstore2.csv.test
head -n $((prop1*2)) Global_Superstore2.csv.shuf | tail -n $prop1 > Global_Superstore2.csv.dev
tail -n +$((prop1*2+1)) Global_Superstore2.csv.shuf > Global_Superstore2.csv.train
rm Global_Superstore2.csv.shuf
#Sprawdźmy, czy wielkości się zgadzają:
wc -l Global_Superstore2.csv*