Jenkins dvc job

This commit is contained in:
MatOgr 2022-06-06 00:33:36 +02:00
parent 274baa507d
commit 50a662240b
9 changed files with 79 additions and 1 deletions

3
.dvc/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
/config.local
/tmp
/cache

View File

@ -0,0 +1,4 @@
[core]
remote = ium_ssh_remote
['remote "ium_ssh_remote"']
url = ssh://ium-sftp@tzietkiewicz.vm.wmi.amu.edu.pl

16
dvc.yml Normal file
View File

@ -0,0 +1,16 @@
stages:
download:
cmd: scripts/load_data.sh
prepare:
cmd: python3 scripts/grab_avocado.py
train:
cmd: python3 scripts/model.py
deps:
- scripts/data/avocado.data.train
outs:
- scripts/data/predictions.csv
params:
- step
- epochs

41
jenkins/dvc.Jenkinsfile Normal file
View File

@ -0,0 +1,41 @@
pipeline {
parameters {
string(
defaultValue: 'mateuszogrodowczyk',
description: 'Kaggle username',
name: 'KAGGLE_USERNAME',
trim: false
)
password(
defaultValue: '',
description: 'Kaggle token taken from kaggle.json file, as described in https://github.com/Kaggle/kaggle-api#api-credentials',
name: 'KAGGLE_KEY'
)
}
agent {
dockerfile {
additionalBuildArgs "--build-arg KAGGLE_USERNAME=${params.KAGGLE_USERNAME} --build-arg KAGGLE_KEY=${params.KAGGLE_KEY} -t s478841-create-dataset"
}
}
stages {
stage("Run DVC") {
steps{
withCredentials([
sshUserPrivateKey(
credentialsId: '48ac7004-216e-4260-abba-1fe5db753e18',
keyFileVariable: 'IUM_SFTP_KEY',
passphraseVariable: '',
usernameVariable: 'USER'
)
]) {
sh 'dvc remote modify --local ium_ssh_remote keyfile $IUM_SFTP_KEY'
sh 'dvc remote modify --local ium_ssh_remote password IUM@2021'
sh 'dvc pull'
sh 'dvc repro'
}
}
}
}
}

2
model_params.yml Normal file
View File

@ -0,0 +1,2 @@
step: 10
epochs: 15

View File

@ -7,3 +7,6 @@ matplotlib
sacred sacred
pymongo pymongo
mlflow mlflow
dvc
dvc-ssh
paramiko

1
scripts/data/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
/plots.png

View File

@ -0,0 +1,4 @@
outs:
- md5: 860c8fe454e7e4683620393359c90e58
size: 25040
path: plots.png

View File

@ -0,0 +1,4 @@
outs:
- md5: 4482cb25938b104e670ad748014354dc
size: 56401
path: predictions.csv