Jenkins dvc job
This commit is contained in:
parent
274baa507d
commit
50a662240b
3
.dvc/.gitignore
vendored
Normal file
3
.dvc/.gitignore
vendored
Normal file
@ -0,0 +1,3 @@
|
||||
/config.local
|
||||
/tmp
|
||||
/cache
|
@ -0,0 +1,4 @@
|
||||
[core]
|
||||
remote = ium_ssh_remote
|
||||
['remote "ium_ssh_remote"']
|
||||
url = ssh://ium-sftp@tzietkiewicz.vm.wmi.amu.edu.pl
|
16
dvc.yml
Normal file
16
dvc.yml
Normal file
@ -0,0 +1,16 @@
|
||||
stages:
|
||||
download:
|
||||
cmd: scripts/load_data.sh
|
||||
|
||||
prepare:
|
||||
cmd: python3 scripts/grab_avocado.py
|
||||
|
||||
train:
|
||||
cmd: python3 scripts/model.py
|
||||
deps:
|
||||
- scripts/data/avocado.data.train
|
||||
outs:
|
||||
- scripts/data/predictions.csv
|
||||
params:
|
||||
- step
|
||||
- epochs
|
41
jenkins/dvc.Jenkinsfile
Normal file
41
jenkins/dvc.Jenkinsfile
Normal file
@ -0,0 +1,41 @@
|
||||
pipeline {
|
||||
parameters {
|
||||
string(
|
||||
defaultValue: 'mateuszogrodowczyk',
|
||||
description: 'Kaggle username',
|
||||
name: 'KAGGLE_USERNAME',
|
||||
trim: false
|
||||
)
|
||||
password(
|
||||
defaultValue: '',
|
||||
description: 'Kaggle token taken from kaggle.json file, as described in https://github.com/Kaggle/kaggle-api#api-credentials',
|
||||
name: 'KAGGLE_KEY'
|
||||
)
|
||||
}
|
||||
|
||||
agent {
|
||||
dockerfile {
|
||||
additionalBuildArgs "--build-arg KAGGLE_USERNAME=${params.KAGGLE_USERNAME} --build-arg KAGGLE_KEY=${params.KAGGLE_KEY} -t s478841-create-dataset"
|
||||
}
|
||||
}
|
||||
|
||||
stages {
|
||||
stage("Run DVC") {
|
||||
steps{
|
||||
withCredentials([
|
||||
sshUserPrivateKey(
|
||||
credentialsId: '48ac7004-216e-4260-abba-1fe5db753e18',
|
||||
keyFileVariable: 'IUM_SFTP_KEY',
|
||||
passphraseVariable: '',
|
||||
usernameVariable: 'USER'
|
||||
)
|
||||
]) {
|
||||
sh 'dvc remote modify --local ium_ssh_remote keyfile $IUM_SFTP_KEY'
|
||||
sh 'dvc remote modify --local ium_ssh_remote password IUM@2021'
|
||||
sh 'dvc pull'
|
||||
sh 'dvc repro'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
2
model_params.yml
Normal file
2
model_params.yml
Normal file
@ -0,0 +1,2 @@
|
||||
step: 10
|
||||
epochs: 15
|
@ -7,3 +7,6 @@ matplotlib
|
||||
sacred
|
||||
pymongo
|
||||
mlflow
|
||||
dvc
|
||||
dvc-ssh
|
||||
paramiko
|
1
scripts/data/.gitignore
vendored
Normal file
1
scripts/data/.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
/plots.png
|
4
scripts/data/plots.png.dvc
Normal file
4
scripts/data/plots.png.dvc
Normal file
@ -0,0 +1,4 @@
|
||||
outs:
|
||||
- md5: 860c8fe454e7e4683620393359c90e58
|
||||
size: 25040
|
||||
path: plots.png
|
4
scripts/data/predictions.csv.dvc
Normal file
4
scripts/data/predictions.csv.dvc
Normal file
@ -0,0 +1,4 @@
|
||||
outs:
|
||||
- md5: 4482cb25938b104e670ad748014354dc
|
||||
size: 56401
|
||||
path: predictions.csv
|
Loading…
Reference in New Issue
Block a user