From eeffaa00b866655ecc6f95731f9c95273baaade6 Mon Sep 17 00:00:00 2001
From: AdamOsiowy123 <adaosi@st.amu.edu.pl>
Date: Sun, 8 May 2022 13:35:38 +0200
Subject: [PATCH] sacred

---
 Jenkins/Jenkinsfile.evaluation | 36 ++++++++++++++++++++++------
 Jenkins/Jenkinsfile.training   | 43 +++++++++++++++++++++++++++-------
 requirements.txt               |  1 +
 3 files changed, 65 insertions(+), 15 deletions(-)

diff --git a/Jenkins/Jenkinsfile.evaluation b/Jenkins/Jenkinsfile.evaluation
index 7a2091b..d89f62d 100644
--- a/Jenkins/Jenkinsfile.evaluation
+++ b/Jenkins/Jenkinsfile.evaluation
@@ -17,15 +17,35 @@ node {
                     description: 'Which build to use for copying artifacts',
                     name: 'BUILD_SELECTOR'
                 ),
-                string(
-                    defaultValue: ".,14000,1,50,100",
-                    description: 'Test params: data_path,num_words,epochs,batch_size,pad_length',
-                    name: 'TEST_PARAMS'
-                ),
                 string(
                     defaultValue: "0",
                     description: 'Build number',
                     name: 'BUILD_NR'
+                ),
+                string(
+                    defaultValue: ".",
+                    description: 'Data path',
+                    name: 'DATA_PATH'
+                ),
+                string(
+                    defaultValue: "1",
+                    description: 'EPOCHS',
+                    name: 'EPOCHS'
+                ),
+                string(
+                    defaultValue: "20000",
+                    description: 'Num words',
+                    name: 'NUM_WORDS'
+                ),
+                string(
+                    defaultValue: "1000",
+                    description: 'Batch size',
+                    name: 'BATCH_SIZE'
+                ),
+                string(
+                    defaultValue: "2000",
+                    description: 'Pad length',
+                    name: 'PAD_LENGTH'
                 )
                 ])
             ])
@@ -37,12 +57,14 @@ node {
             copyArtifacts filter: 'model/neural_net', projectName: "s444452-training/${params.BRANCH}", selector: buildParameter('BUILD_SELECTOR')
         }
         stage('Run script') {
-            withEnv(["TEST_PARAMS=${params.TEST_PARAMS}", "BUILD_NR=${params.BUILD_NR}"]) {
-                sh "python3 Scripts/evaluate_neural_network.py $BUILD_NR $TEST_PARAMS"
+            withEnv(["BUILD_NR=${params.BUILD_NR}","DATA_PATH=${params.DATA_PATH}","EPOCHS=${params.EPOCHS}",
+                "NUM_WORDS=${params.NUM_WORDS}", "BATCH_SIZE=${params.BATCH_SIZE}","PAD_LENGTH=${params.PAD_LENGTH}"]) {
+                sh "python3 Scripts/evaluate_neural_network.py $BUILD_NR $DATA_PATH $EPOCHS $NUM_WORDS $BATCH_SIZE $PAD_LENGTH"
             }
         }
         stage('Archive artifacts') {
             archiveArtifacts "neural_network_evaluation.csv, evaluation.png"
+            archiveArtifacts "my_runs/**"
         }
         }
     } catch (e) {
diff --git a/Jenkins/Jenkinsfile.training b/Jenkins/Jenkinsfile.training
index bd204c0..7b9ee3a 100644
--- a/Jenkins/Jenkinsfile.training
+++ b/Jenkins/Jenkinsfile.training
@@ -7,9 +7,29 @@ node {
                 pipelineTriggers([upstream(threshold: hudson.model.Result.SUCCESS, upstreamProjects: "s444452-create-dataset")]),
                 parameters([
                 string(
-                 defaultValue: ".,14000,1,50,100",
-                 description: 'Train params: data_path,num_words,epochs,batch_size,pad_length',
-                 name: 'TRAIN_PARAMS'
+                 defaultValue: ".",
+                 description: 'Data path',
+                 name: 'DATA_PATH'
+                ),
+                string(
+                 defaultValue: "1",
+                 description: 'EPOCHS',
+                 name: 'EPOCHS'
+                ),
+                string(
+                 defaultValue: "20000",
+                 description: 'Num words',
+                 name: 'NUM_WORDS'
+                ),
+                string(
+                 defaultValue: "1000",
+                 description: 'Batch size',
+                 name: 'BATCH_SIZE'
+                ),
+                string(
+                 defaultValue: "2000",
+                 description: 'Pad length',
+                 name: 'PAD_LENGTH'
                 )
                 ])
             ])
@@ -19,12 +39,14 @@ node {
             copyArtifacts filter: 'test_data.csv', fingerprintArtifacts: true, projectName: 's444452-create-dataset'
         }
         stage('Run script') {
-            withEnv(["TRAIN_PARAMS=${params.TRAIN_PARAMS}"]) {
-                sh "python3 Scripts/train_neural_network.py $TRAIN_PARAMS"
+            withEnv(["DATA_PATH=${params.DATA_PATH}","EPOCHS=${params.EPOCHS}","NUM_WORDS=${params.NUM_WORDS}",
+                "BATCH_SIZE=${params.BATCH_SIZE}","PAD_LENGTH=${params.PAD_LENGTH}"]) {
+                sh "python3 Scripts/train_neural_network.py $DATA_PATH $EPOCHS $NUM_WORDS $BATCH_SIZE $PAD_LENGTH"
             }
         }
         stage('Archive artifacts') {
             archiveArtifacts "model/neural_net"
+            archiveArtifacts "my_runs/**"
         }
         }
     } catch (e) {
@@ -38,15 +60,20 @@ def notifyBuild(String buildStatus = 'STARTED') {
     buildStatus = buildStatus ?: 'SUCCESS'
 
     def subject = "Job: ${env.JOB_NAME}"
-    def details = "Build nr: ${env.BUILD_NUMBER}, status: ${buildStatus} \n url: ${env.BUILD_URL} \n build params: ${params.TRAIN_PARAMS}"
+    def build_params = "Path: ${params.DATA_PATH}, Epochs: ${params.EPOCHS}, Num_words: ${params.NUM_WORDS}, Batch_size: ${params.BATCH_SIZE}, Pad_length: ${params.PAD_LENGTH}"
+    def details = "Build nr: ${env.BUILD_NUMBER}, status: ${buildStatus} \n url: ${env.BUILD_URL} \n build params: ${build_params}"
 
     if (buildStatus == 'SUCCESS') {
         build (
             job: "s444452-evaluation/${env.BRANCH_NAME}",
             parameters: [
                 gitParameter(name: "BRANCH", value: "${env.BRANCH_NAME}"),
-                string(name: "TEST_PARAMS",  value: "${params.TRAIN_PARAMS}"),
-                string(name: "BUILD_NR", value: "${env.BUILD_NUMBER}")
+                string(name: "BUILD_NR", value: "${env.BUILD_NUMBER}"),
+                string(name: "DATA_PATH",  value: "${params.DATA_PATH}"),
+                string(name: "EPOCHS",  value: "${params.EPOCHS}"),
+                string(name: "NUM_WORDS",  value: "${params.NUM_WORDS}"),
+                string(name: "BATCH_SIZE",  value: "${params.BATCH_SIZE}"),
+                string(name: "PAD_LENGTH",  value: "${params.PAD_LENGTH}")
             ],
             wait: false
         )
diff --git a/requirements.txt b/requirements.txt
index 04a3068..86e7cdc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -40,6 +40,7 @@ protobuf==3.20.1
 py-cpuinfo==8.0.0
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
+pymongo==4.1.1
 pyparsing==3.0.8
 python-dateutil==2.8.2
 python-slugify==6.1.1