added stats/Jenkinsfile

Jenkinsfile

@@ -26,7 +26,7 @@ pipeline {
                         "KAGGLE_KEY=${KAGGLE_KEY}" ]) {
                   sh 'chmod 777 ./data_download.sh'
                   sh './data_download.sh --cutoff ${CUTOFF}'
-                  archiveArtifacts artifacts: 'test_without_id_column.csv,cutoff_train.csv', followSymlinks: false
+                  archiveArtifacts artifacts: 'test_without_id_column.csv,cutoff_train.csv,test.csv,train.csv', followSymlinks: false
               }
         }
       }

stats/Jenkinsfile

@@ -0,0 +1,20 @@
+pipeline {
+  agent any
+  parameters {
+      buildSelector(
+        defaultSelector: lastSuccessful(),
+        description: 'Which build to use for copying artifacts'
+        name: 'BUILD_SELECTOR'
+      )
+    }
+  stages {
+    stage('Dataset statistics') {
+        steps {
+          copyArtifacts filter:'cutoff_train.csv',fingerprintArtifacts: true, projectName: 'z-s495716-create-dataset', selector: buildParameter('BUILD_SELECTOR')
+          sh 'chmod 777 stats/data_stats.sh'
+          sh './data_stats.sh'
+          archiveArtifacts artifacts: 'train_lines.txt', followSymlinks: false
+        }
+      }
+   }
+}

stats/data_stats.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+figlet "Counting dataset lines"
+wc -l cutoff_train.csv > train_lines.txt