pipeline { agent { dockerfile true } parameters{ buildSelector( defaultSelector: lastSuccessful(), description: 'Which build to use for copying artifacts', name: 'WHICH_BUILD' ) } stages { stage('One') { properties([ parameters([ string(defaultValue: '', description: 'Kaggle username', name: 'KAGGLE_USERNAME', trim: false), password(defaultValue: '', description: 'Kaggle token', name: 'KAGGLE_KEY'), string(defaultValue: '1000', description: 'Data cutoff', name: 'CUTOFF', trim: false) ]) ] ) stage('Two') { withEnv(["KAGGLE_USERNAME=${params.KAGGLE_USERNAME}", "KAGGLE_KEY=${params.KAGGLE_KEY}", "CUTOFF=${params.CUTOFF}" ]) { checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[url: 'https://git.wmi.amu.edu.pl/s440058/ium_440058']]]) checkout scm def image = docker.build("s440058/ium") image.inside { sh 'python3 ./pytorch-example.py > model.txt' sh "chmod 777 ./bash.sh" sh "./bash.sh" archiveArtifacts "courses.data.dev" archiveArtifacts "courses.data.test" archiveArtifacts "courses.data.train" archiveArtifacts 'model.txt' } } } stage('DVC') { steps { withCredentials([sshUserPrivateKey(credentialsId: '48ac7004-216e-4260-abba-1fe5db753e18', keyFileVariable: 'IUM_SFTP_KEY')]) { copyArtifacts fingerprintArtifacts: true, projectName: 's440058-create-dataset', selector: buildParameter('WHICH_BUILD') sh "dvc init -f" sh "dvc remote add -d ium_ssh_remote ssh://ium-sftp@tzietkiewicz.vm.wmi.amu.edu.pl/ium-sftp" sh "dvc remote modify --local ium_ssh_remote keyfile $IUM_SFTP_KEY" sh "dvc pull" sh "dvc reproduce" } } } } post { success { build job: 's440058-dvc' mail body: 'SUCCESS TRAINING', subject: 's440058', to: '26ab8f35.uam.onmicrosoft.com@emea.teams.ms' } failure { mail body: 'FAILURE TRAINING', subject: 's440058', to: '26ab8f35.uam.onmicrosoft.com@emea.teams.ms' } } }