Compare commits

...

48 Commits

Author SHA1 Message Date
0204ef10e9 :(((( 2021-05-24 01:26:02 +02:00
c579d63845 fix 2021-05-24 00:55:40 +02:00
93a992d53d fix 2021-05-24 00:51:33 +02:00
84641091ce fix 2021-05-24 00:42:36 +02:00
5a6d4f832d fix 2021-05-24 00:36:16 +02:00
c212ffdb9c fix 2021-05-24 00:30:08 +02:00
a024e32143 fix 2021-05-24 00:22:18 +02:00
27743b754f fix 2021-05-24 00:22:01 +02:00
5cc3fd58b1 fix 2021-05-23 23:29:58 +02:00
069a5860d6 fix 2021-05-23 23:25:50 +02:00
30c8343c6c fix 2021-05-23 23:21:33 +02:00
9050bcb8e7 fix 2021-05-23 23:14:15 +02:00
b5774c67b7 some changes 2021-05-23 23:02:17 +02:00
eee8d50045 **/* 2021-05-23 22:30:23 +02:00
f8addf09d4 change for Domino2
All checks were successful
s434695-training/pipeline/head This commit looks good
2021-05-23 19:56:42 +02:00
caffb7a796 change for Domino
All checks were successful
s434695-training/pipeline/head This commit looks good
2021-05-23 19:47:59 +02:00
01eea15055 elo
Some checks failed
s434695-training/pipeline/head There was a failure building this commit
2021-05-23 19:36:03 +02:00
c01ba17840 elo
Some checks failed
s434695-training/pipeline/head There was a failure building this commit
2021-05-23 19:31:11 +02:00
db58b28106 asd 2021-05-23 18:20:21 +02:00
49d70902b1 :(
Some checks failed
s434695-training/pipeline/head There was a failure building this commit
2021-05-23 14:46:00 +02:00
982b1a1263 :)
All checks were successful
s434695-training/pipeline/head This commit looks good
2021-05-23 14:41:37 +02:00
9947cf587f fix
Some checks failed
s434695-training/pipeline/head There was a failure building this commit
2021-05-17 12:32:14 +02:00
07e0eb04eb fix
Some checks failed
s434695-training/pipeline/head There was a failure building this commit
2021-05-17 12:31:07 +02:00
3d068ac7d1 fix
Some checks failed
s434695-training/pipeline/head There was a failure building this commit
2021-05-16 22:45:28 +02:00
8fad47a012 fix 2021-05-16 22:32:44 +02:00
e754b04eb2 test worked 2021-05-16 22:29:16 +02:00
9d5a047ec7 test
All checks were successful
s434695-training/pipeline/head This commit looks good
2021-05-16 22:27:29 +02:00
41d3a9cb9a :(
Some checks failed
s434695-training/pipeline/head There was a failure building this commit
2021-05-16 22:10:50 +02:00
f40f237a02 fix2
Some checks failed
s434695-training/pipeline/head There was a failure building this commit
2021-05-16 22:00:03 +02:00
26ddaa3a0c fix train
Some checks failed
s434695-training/pipeline/head There was a failure building this commit
2021-05-16 21:50:54 +02:00
ef245b69da no flag user in dockerfile
Some checks failed
s434695-training/pipeline/head There was a failure building this commit
2021-05-16 21:37:25 +02:00
b1bb478c17 a
Some checks failed
s434695-training/pipeline/head There was a failure building this commit
2021-05-16 21:18:33 +02:00
01373b20e6 top 10 anime bterayals
Some checks failed
s434695-training/pipeline/head There was a failure building this commit
2021-05-16 21:10:40 +02:00
da73d5dca4 only sad reactions
Some checks failed
s434695-training/pipeline/head There was a failure building this commit
2021-05-16 21:04:29 +02:00
101850ba06 test
Some checks failed
s434695-training/pipeline/head There was a failure building this commit
2021-05-16 20:58:24 +02:00
d30489e817 only sad reactions
Some checks failed
s434695-training/pipeline/head There was a failure building this commit
2021-05-16 20:33:23 +02:00
93516ec2ef :(
Some checks failed
s434695-training/pipeline/head There was a failure building this commit
2021-05-16 20:22:27 +02:00
2f78b3d264 elo
Some checks failed
s434695-training/pipeline/head There was a failure building this commit
2021-05-16 20:01:42 +02:00
835f99ece5 fix
Some checks failed
s434695-training/pipeline/head There was a failure building this commit
2021-05-16 19:50:58 +02:00
8172699654 train
Some checks failed
s434695-training/pipeline/head There was a failure building this commit
2021-05-16 19:48:11 +02:00
c6146b9dfc sadge
Some checks failed
s434695-training/pipeline/head There was a failure building this commit
2021-05-16 18:41:24 +02:00
9a0711c0d3 :(
Some checks failed
s434695-training/pipeline/head There was a failure building this commit
2021-05-15 19:19:56 +02:00
18172eadf9 training changes 2021-05-15 19:14:37 +02:00
2d0e53d1eb sadge3 2021-05-15 18:24:30 +02:00
f5158a272c saving artifacts fix 2021-05-15 18:06:33 +02:00
72b0e06eb3 fixfixfix 2021-05-15 17:38:43 +02:00
391d4309df fix 2021-05-15 17:35:29 +02:00
e92325d5f5 new branch 2021-05-15 17:01:54 +02:00
159 changed files with 750 additions and 93 deletions

View File

@ -5,20 +5,21 @@ FROM ubuntu:latest
RUN apt update && apt install -y figlet
RUN apt install -y git
RUN apt install -y python3-pip
RUN pip3 install --user kaggle
RUN pip3 install --user pandas
RUN pip3 install --user numpy
RUN pip3 install --user seaborn
RUN pip3 install --user sklearn
RUN pip3 install --user matplotlib
RUN pip3 install --user tensorflow
RUN pip3 install --user wget
RUN pip3 install setuptools
RUN pip3 install kaggle
RUN pip3 install pandas
RUN pip3 install numpy
RUN pip3 install seaborn
RUN pip3 install sklearn
RUN pip3 install matplotlib
RUN pip3 install tensorflow
RUN pip3 install sacred
RUN pip3 install wget
RUN pip3 install keras
RUN pip3 install GitPython
RUN pip3 install pymongo
RUN pip3 install mlflow
WORKDIR /app
COPY ./skrypt.sh ./
COPY ./zadanie2.py ./
COPY ./zadanie5.py ./
CMD ./zadanie2.py
CMD ./zadanie5.py
COPY ./train.py ./

91
Jenkinsfile vendored
View File

@ -1,40 +1,61 @@
pipeline {
agent {
dockerfile true
}
//Definijuemy parametry, które będzie można podać podczas wywoływania zadania
parameters {
string (
defaultValue: '10000',
description: 'Obcięcie zbioru',
name: 'CUTOFF',
trim: false
agent { dockerfile true }
parameters{
buildSelector(
defaultSelector: lastSuccessful(),
description: 'Which build to use for copying artifacts',
name: 'WHICH_BUILD'
)
string(
defaultValue: '16',
description: 'batch size',
name: 'BATCH_SIZE'
)
string(
defaultValue: '15',
description: 'epochs',
name: 'EPOCHS'
)
}
stages {
stage('Test') {
steps {
sh 'cat /etc/issue'
}
}
stage('checkout: Check out from version control') {
steps {
git credentialsId: 'f746c8e0-e974-4758-838d-1920a7b0e3dc', url: 'https://git.wmi.amu.edu.pl/s434695/ium_434695.git'
}
}
stage('sh: Shell Script') {
steps {
sh 'chmod +x skrypt.sh'
sh './skrypt.sh ${CUTOFF}'
}
}
stage('Artifacts') {
steps {
archiveArtifacts 'test.csv'
archiveArtifacts 'dev.csv'
archiveArtifacts 'train.csv'
archiveArtifacts 'obcietedane.csv'
}
}
stage('copy artifacts')
{
steps
{
copyArtifacts(fingerprintArtifacts: true, projectName: 's434695-create-dataset', selector: buildParameter('WHICH_BUILD'))
}
}
stage('train')
{
steps
{
sh 'python3 train.py ${BATCH_SIZE} ${EPOCHS}'
sh 'rm -r my_model'
sh 'python3 vgsales-mlflow.py'
}
}
stage('Archive artifacts') {
steps{
archiveArtifacts 'vgsales_model.h5'
archiveArtifacts 'my_runs/**/*'
archiveArtifacts 'my_model/**/*'
}
}
}
post {
success {
build job: 's434695-evaluation/evaluation'
mail body: 'SUCCESS TRAINING', subject: 's434695', to: '26ab8f35.uam.onmicrosoft.com@emea.teams.ms'
}
failure {
mail body: 'FAILURE TRAINING', subject: 's434695', to: '26ab8f35.uam.onmicrosoft.com@emea.teams.ms'
}
}
}
}

11
MLproject Normal file
View File

@ -0,0 +1,11 @@
name: 434695-mlflow
docker_env:
image: shroomy/ium:7
entry_points:
main:
parameters:
epochs: {type: int, default: 15}
batch_size: {type: int, default: 16}
command: "python3 vgsales-mlflow.py {epochs} {batch_size}"

View File

@ -0,0 +1,15 @@
artifact_uri: file:///home/tomasz/IUM/ium_434695/mlruns/0/13e3e1a2ce674a07b82227babfec5f00/artifacts
end_time: null
entry_point_name: ''
experiment_id: '0'
lifecycle_stage: active
name: ''
run_id: 13e3e1a2ce674a07b82227babfec5f00
run_uuid: 13e3e1a2ce674a07b82227babfec5f00
source_name: ''
source_type: 4
source_version: ''
start_time: 1621790521589
status: 1
tags: []
user_id: tomasz

View File

@ -0,0 +1 @@
db58b281061f170a706135380f1e7c86c7162acc

View File

@ -0,0 +1 @@
/home/tomasz/IUM/ium_434695/vgsales-mlflow.py

View File

@ -0,0 +1 @@
LOCAL

View File

@ -0,0 +1 @@
tomasz

View File

@ -0,0 +1,15 @@
artifact_uri: file:///home/tomasz/IUM/ium_434695/mlruns/0/156c624fa6ff468887bf9543a37ce8b3/artifacts
end_time: 1621804324404
entry_point_name: ''
experiment_id: '0'
lifecycle_stage: active
name: ''
run_id: 156c624fa6ff468887bf9543a37ce8b3
run_uuid: 156c624fa6ff468887bf9543a37ce8b3
source_name: ''
source_type: 4
source_version: ''
start_time: 1621804324311
status: 4
tags: []
user_id: tomasz

View File

@ -0,0 +1 @@
b5774c67b74e143c2e0d3a990180a7305140b234

View File

@ -0,0 +1 @@
/home/tomasz/IUM/ium_434695/vgsales-mlflow.py

View File

@ -0,0 +1 @@
LOCAL

View File

@ -0,0 +1 @@
tomasz

View File

@ -0,0 +1,15 @@
artifact_uri: file:///home/tomasz/IUM/ium_434695/mlruns/0/1797ad0fc8144ad48230f8965a689293/artifacts
end_time: 1621790489715
entry_point_name: ''
experiment_id: '0'
lifecycle_stage: active
name: ''
run_id: 1797ad0fc8144ad48230f8965a689293
run_uuid: 1797ad0fc8144ad48230f8965a689293
source_name: ''
source_type: 4
source_version: ''
start_time: 1621790450170
status: 4
tags: []
user_id: tomasz

View File

@ -0,0 +1 @@
1621790489709 0.032775074938251283 0

View File

@ -0,0 +1 @@
16

View File

@ -0,0 +1 @@
15

View File

@ -0,0 +1 @@
db58b281061f170a706135380f1e7c86c7162acc

View File

@ -0,0 +1 @@
/home/tomasz/IUM/ium_434695/vgsales-mlflow.py

View File

@ -0,0 +1 @@
LOCAL

View File

@ -0,0 +1 @@
tomasz

View File

@ -0,0 +1,14 @@
artifact_path: vgsales_model.h5
flavors:
keras:
data: data
keras_module: tensorflow.keras
keras_version: 2.4.0
save_format: tf
python_function:
data: data
env: conda.yaml
loader_module: mlflow.keras
python_version: 3.8.5
run_id: 1d5601fad0dc496b8e16c284d2fcd1c9
utc_time_created: '2021-05-23 17:16:38.081474'

View File

@ -0,0 +1,10 @@
channels:
- defaults
- conda-forge
dependencies:
- python=3.8.5
- pip
- pip:
- mlflow
- tensorflow==2.4.1
name: mlflow-env

View File

@ -0,0 +1,15 @@
artifact_uri: file:///home/tomasz/IUM/ium_434695/mlruns/0/1d5601fad0dc496b8e16c284d2fcd1c9/artifacts
end_time: 1621790199119
entry_point_name: ''
experiment_id: '0'
lifecycle_stage: active
name: ''
run_id: 1d5601fad0dc496b8e16c284d2fcd1c9
run_uuid: 1d5601fad0dc496b8e16c284d2fcd1c9
source_name: ''
source_type: 4
source_version: ''
start_time: 1621790167199
status: 4
tags: []
user_id: tomasz

View File

@ -0,0 +1 @@
1621790198079 0.031762412384138385 0

View File

@ -0,0 +1 @@
16

View File

@ -0,0 +1 @@
15

View File

@ -0,0 +1 @@
[{"run_id": "1d5601fad0dc496b8e16c284d2fcd1c9", "artifact_path": "vgsales_model.h5", "utc_time_created": "2021-05-23 17:16:38.081474", "flavors": {"keras": {"keras_module": "tensorflow.keras", "keras_version": "2.4.0", "save_format": "tf", "data": "data"}, "python_function": {"loader_module": "mlflow.keras", "python_version": "3.8.5", "data": "data", "env": "conda.yaml"}}}]

View File

@ -0,0 +1 @@
db58b281061f170a706135380f1e7c86c7162acc

View File

@ -0,0 +1 @@
/home/tomasz/IUM/ium_434695/vgsales-mlflow.py

View File

@ -0,0 +1 @@
LOCAL

View File

@ -0,0 +1 @@
tomasz

View File

@ -0,0 +1,15 @@
artifact_uri: file:///home/tomasz/IUM/ium_434695/mlruns/0/4be4d7d988dc477e8ec483ca0ec5bb30/artifacts
end_time: null
entry_point_name: ''
experiment_id: '0'
lifecycle_stage: active
name: ''
run_id: 4be4d7d988dc477e8ec483ca0ec5bb30
run_uuid: 4be4d7d988dc477e8ec483ca0ec5bb30
source_name: ''
source_type: 4
source_version: ''
start_time: 1621804433172
status: 1
tags: []
user_id: tomasz

View File

@ -0,0 +1 @@
b5774c67b74e143c2e0d3a990180a7305140b234

View File

@ -0,0 +1 @@
/home/tomasz/IUM/ium_434695/vgsales-mlflow.py

View File

@ -0,0 +1 @@
LOCAL

View File

@ -0,0 +1 @@
tomasz

View File

@ -0,0 +1,15 @@
artifact_uri: file:///home/tomasz/IUM/ium_434695/mlruns/0/5d3630baec6a424da8572f7ef9ef85b3/artifacts
end_time: 1621790904083
entry_point_name: ''
experiment_id: '0'
lifecycle_stage: active
name: ''
run_id: 5d3630baec6a424da8572f7ef9ef85b3
run_uuid: 5d3630baec6a424da8572f7ef9ef85b3
source_name: ''
source_type: 4
source_version: ''
start_time: 1621790867133
status: 4
tags: []
user_id: tomasz

View File

@ -0,0 +1 @@
1621790904078 0.03182406217035666 0

View File

@ -0,0 +1 @@
16

View File

@ -0,0 +1 @@
15

View File

@ -0,0 +1 @@
db58b281061f170a706135380f1e7c86c7162acc

View File

@ -0,0 +1 @@
/home/tomasz/IUM/ium_434695/vgsales-mlflow.py

View File

@ -0,0 +1 @@
LOCAL

View File

@ -0,0 +1 @@
tomasz

View File

@ -0,0 +1,15 @@
artifact_uri: file:///home/tomasz/IUM/ium_434695/mlruns/0/63c390abb797412baf1d85029f15c493/artifacts
end_time: 1621805381189
entry_point_name: ''
experiment_id: '0'
lifecycle_stage: active
name: ''
run_id: 63c390abb797412baf1d85029f15c493
run_uuid: 63c390abb797412baf1d85029f15c493
source_name: ''
source_type: 4
source_version: ''
start_time: 1621805345446
status: 3
tags: []
user_id: tomasz

View File

@ -0,0 +1 @@
1621805379411 0.03060625342038236 0

View File

@ -0,0 +1 @@
16

View File

@ -0,0 +1 @@
15

View File

@ -0,0 +1 @@
069a5860d67ce9de0724996a8ab92c6a204fcb23

View File

@ -0,0 +1 @@
/home/tomasz/IUM/ium_434695/vgsales-mlflow.py

View File

@ -0,0 +1 @@
LOCAL

View File

@ -0,0 +1 @@
tomasz

View File

@ -0,0 +1,15 @@
artifact_uri: file:///home/tomasz/IUM/ium_434695/mlruns/0/794e3e4e628e416289e6237f26ef142d/artifacts
end_time: 1621790754185
entry_point_name: ''
experiment_id: '0'
lifecycle_stage: active
name: ''
run_id: 794e3e4e628e416289e6237f26ef142d
run_uuid: 794e3e4e628e416289e6237f26ef142d
source_name: ''
source_type: 4
source_version: ''
start_time: 1621790714349
status: 3
tags: []
user_id: tomasz

View File

@ -0,0 +1 @@
1621790752218 0.031056313979876536 0

View File

@ -0,0 +1 @@
16

View File

@ -0,0 +1 @@
15

View File

@ -0,0 +1 @@
db58b281061f170a706135380f1e7c86c7162acc

View File

@ -0,0 +1 @@
/home/tomasz/IUM/ium_434695/vgsales-mlflow.py

View File

@ -0,0 +1 @@
LOCAL

View File

@ -0,0 +1 @@
tomasz

View File

@ -0,0 +1,14 @@
artifact_path: vgsales_model.h5
flavors:
keras:
data: data
keras_module: tensorflow.keras
keras_version: 2.4.0
save_format: tf
python_function:
data: data
env: conda.yaml
loader_module: mlflow.keras
python_version: 3.8.5
run_id: 883a0165bd624848bff8037bbad39446
utc_time_created: '2021-05-23 17:12:20.066683'

View File

@ -0,0 +1,10 @@
channels:
- defaults
- conda-forge
dependencies:
- python=3.8.5
- pip
- pip:
- mlflow
- tensorflow==2.4.1
name: mlflow-env

View File

@ -0,0 +1,15 @@
artifact_uri: file:///home/tomasz/IUM/ium_434695/mlruns/0/883a0165bd624848bff8037bbad39446/artifacts
end_time: 1621789941879
entry_point_name: ''
experiment_id: '0'
lifecycle_stage: active
name: ''
run_id: 883a0165bd624848bff8037bbad39446
run_uuid: 883a0165bd624848bff8037bbad39446
source_name: ''
source_type: 4
source_version: ''
start_time: 1621789909395
status: 3
tags: []
user_id: tomasz

View File

@ -0,0 +1 @@
1621789940064 0.03283662485627263 0

View File

@ -0,0 +1 @@
16

View File

@ -0,0 +1 @@
15

View File

@ -0,0 +1 @@
[{"run_id": "883a0165bd624848bff8037bbad39446", "artifact_path": "vgsales_model.h5", "utc_time_created": "2021-05-23 17:12:20.066683", "flavors": {"keras": {"keras_module": "tensorflow.keras", "keras_version": "2.4.0", "save_format": "tf", "data": "data"}, "python_function": {"loader_module": "mlflow.keras", "python_version": "3.8.5", "data": "data", "env": "conda.yaml"}}}]

View File

@ -0,0 +1 @@
db58b281061f170a706135380f1e7c86c7162acc

View File

@ -0,0 +1 @@
/home/tomasz/IUM/ium_434695/vgsales-mlflow.py

View File

@ -0,0 +1 @@
LOCAL

View File

@ -0,0 +1 @@
tomasz

View File

@ -0,0 +1,15 @@
artifact_uri: file:///home/tomasz/IUM/ium_434695/mlruns/0/b19dc8b40ada4fdea65402cf2f8a4c39/artifacts
end_time: 1621804368194
entry_point_name: ''
experiment_id: '0'
lifecycle_stage: active
name: ''
run_id: b19dc8b40ada4fdea65402cf2f8a4c39
run_uuid: b19dc8b40ada4fdea65402cf2f8a4c39
source_name: ''
source_type: 4
source_version: ''
start_time: 1621804367588
status: 4
tags: []
user_id: tomasz

View File

@ -0,0 +1 @@
b5774c67b74e143c2e0d3a990180a7305140b234

View File

@ -0,0 +1 @@
/home/tomasz/IUM/ium_434695/vgsales-mlflow.py

View File

@ -0,0 +1 @@
LOCAL

View File

@ -0,0 +1 @@
tomasz

View File

@ -0,0 +1,15 @@
artifact_uri: file:///home/tomasz/IUM/ium_434695/mlruns/0/c292a5394ecd4efb891579d87f350572/artifacts
end_time: 1621791326975
entry_point_name: ''
experiment_id: '0'
lifecycle_stage: active
name: ''
run_id: c292a5394ecd4efb891579d87f350572
run_uuid: c292a5394ecd4efb891579d87f350572
source_name: ''
source_type: 4
source_version: ''
start_time: 1621791287578
status: 3
tags: []
user_id: tomasz

View File

@ -0,0 +1 @@
1621791324872 0.03170582289731189 0

View File

@ -0,0 +1 @@
16

View File

@ -0,0 +1 @@
15

View File

@ -0,0 +1 @@
c01ba178401a83dc1b9d48a1c8eb9552894c9db4

View File

@ -0,0 +1 @@
/home/tomasz/IUM/ium_434695/vgsales-mlflow.py

View File

@ -0,0 +1 @@
LOCAL

View File

@ -0,0 +1 @@
tomasz

View File

@ -0,0 +1,14 @@
artifact_path: vgsales_model.h5
flavors:
keras:
data: data
keras_module: tensorflow.keras
keras_version: 2.4.0
save_format: tf
python_function:
data: data
env: conda.yaml
loader_module: mlflow.keras
python_version: 3.8.5
run_id: c982ec1561894780b45c04392e54dede
utc_time_created: '2021-05-23 17:19:52.104292'

View File

@ -0,0 +1,10 @@
channels:
- defaults
- conda-forge
dependencies:
- python=3.8.5
- pip
- pip:
- mlflow
- tensorflow==2.4.1
name: mlflow-env

Some files were not shown because too many files have changed in this diff Show More