x1/Jenkinsfile
wojciechbatruszewicz 2749956f9a Create dataset
2023-06-26 19:08:30 +02:00

69 lines
2.1 KiB
Groovy

pipeline {
agent any
parameters {
string(
defaultValue: 'wojciechbatruszewicz',
description: 'Kaggle username',
name: 'KAGGLE_USERNAME',
trim: false
)
password(
defaultValue: '',
description: 'Kaggle token taken from kaggle.json file, as described in https://github.com/Kaggle/kaggle-api#api-credentials',
name: 'KAGGLE_KEY'
)
string(
defaultValue: '30',
description: 'dataset cutoff',
name: 'CUTOFF',
trim: false
)
}
stages {
stage('Download dataset') {
steps {
checkout scm
dir ('./createDataset') {
sh 'ls -l'
withEnv(["KAGGLE_USERNAME=${params.KAGGLE_USERNAME}",
"KAGGLE_KEY=${params.KAGGLE_KEY}" ]) {
sh 'kaggle datasets download -d elakiricoder/gender-classification-dataset'
sh 'unzip -o gender-classification-dataset.zip'
}
}
}
}
stage('Docker') {
steps {
script {
def dockerImage = docker.build("docker-image", "./")
dockerImage.inside {
sh 'ls -l'
dir ('./createDataset') {
sh 'ls -l'
sh 'python3 createDataset.py'
archiveArtifacts 'gender_classification_train.csv'
archiveArtifacts 'gender_classification_test.csv'
archiveArtifacts 'gender_classification_val.csv'
}
sh 'ls -l'
}
}
}
}
// stage('Archive file') {
// steps {
// dir ('./createDataset') {
// archiveArtifacts artifacts: 'loan_sanction_shuffled.csv', fingerprint: true\
// }
// }
// }
}
// post {
// success {
// build job: 'z-s487179-training/main', wait: false
// }
// }
}