Loading dataset script update

This commit is contained in:
s478841 2022-03-27 23:34:18 +02:00
parent d84f6f6f43
commit 58f3c22b6e
2 changed files with 24 additions and 24 deletions

34
Jenkinsfile vendored
View File

@ -1,34 +1,22 @@
node { node {
// stage('Preparation') { stage('checkout: Check out from version control') {
// properties ([ git 'https://git.wmi.amu.edu.pl/s478841/ium_478841.git'
// parameters([ }
// string(
// defaultValue:'mateuszogrodowczyk',
// description: 'Kaggle username',
// name: 'KAGGLE_USERNAME',
// trim: false
// ),
// password(
// description: 'Kaggle access token retrieved from kaggle.json file - https://github.com/Kaggle/kaggle-api#api-credentials',
// name: 'KAGGLE_KEY'
// )
// ])
// ])
// }
stage('Install depends.') { stage('Install depends.') {
sh 'pip install --user -r requirements.txt' sh 'pip install --user -r requirements.txt'
} }
stage('Prepare dataset') { stage('sh: Shell script') {
withEnv([ withEnv([
"KAGGLE_USERNAME=${params.KAGGLE_USERNAME}", "KAGGLE_USERNAME=${params.KAGGLE_USERNAME}",
"KAGGLE_KEY=${params.KAGGLE_KEY}" "KAGGLE_KEY=${params.KAGGLE_KEY}"
]) { ]) {
sh 'echo Welcome: $KAGGLE_USERNAME' sh 'chmod u+x ./load_data.sh'
sh 'kaggle datasets list' sh './load_data.sh'
}
}
post {
always {
archiveArtifacts artifacts: 'avocado.data*', onlyIfSuccessful: true
} }
sh 'chmod u+x ./load_data.sh'
sh './load_data.sh'
} }
} }

View File

@ -1 +1,13 @@
figlet "Load data" figlet "Welcome $KAGGLE_USERNAME"
rm -r avocado.data*
kaggle datasets download -d neuromusic/avocado-prices
unzip -o avocado-prices.zip
tail -n +2 avocado.csv | shuf > avocado_shuf.csv
head -n 14000 avocado_shuf.csv > avocado.data.train
tail -n +14001 avocado_shuf.csv | head -n 2249 > avocado.data.valid
tail -n 2000 avocado_shuf.csv > avocado.data.test
wc -l avocado.data* > results.txt