update
This commit is contained in:
parent
9857b4339b
commit
e3e995ea3f
5
Dockerfile_sec
Normal file
5
Dockerfile_sec
Normal file
@ -0,0 +1,5 @@
|
||||
FROM python:latest
|
||||
|
||||
RUN apt-get update && apt-get install -y
|
||||
|
||||
RUN pip install pandas
|
4
JenkinsFile → Jenkinsfile
vendored
4
JenkinsFile → Jenkinsfile
vendored
@ -1,7 +1,6 @@
|
||||
pipeline {
|
||||
agent any
|
||||
|
||||
//Definijuemy parametry, które będzie można podać podczas wywoływania zadania
|
||||
parameters{
|
||||
string(
|
||||
defaultValue: '500',
|
||||
@ -13,7 +12,6 @@ pipeline {
|
||||
stages {
|
||||
stage('clear_all') {
|
||||
steps {
|
||||
//Wypisz wartość parametru w konsoli (To nie jest polecenie bash, tylko groovy!)
|
||||
sh 'rm -rf *'
|
||||
}
|
||||
}
|
||||
@ -21,7 +19,6 @@ pipeline {
|
||||
stage('Build') {
|
||||
steps {
|
||||
sh 'git clone https://git.wmi.amu.edu.pl/s444439/ium_z444439'
|
||||
// Run the maven build
|
||||
sh 'curl -O https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data'
|
||||
sh 'sed -i "1i\\age,workclass,fnlwgt,education,education-num,marital-status,occupation,relationship,race,sex,capital-gain,capital-loss,hours-per-week,native-country,income" adult.data'
|
||||
sh 'mv adult.data adult.csv'
|
||||
@ -41,6 +38,7 @@ agent {
|
||||
steps {
|
||||
sh 'ls -a'
|
||||
sh 'python ./ium_z444439/create-dataset.py'
|
||||
echo 'process finish'
|
||||
archiveArtifacts 'X_test.csv'
|
||||
archiveArtifacts 'X_dev.csv'
|
||||
archiveArtifacts 'X_train.csv'
|
50
Jenkinsfile_stats
Normal file
50
Jenkinsfile_stats
Normal file
@ -0,0 +1,50 @@
|
||||
pipeline {
|
||||
agent any
|
||||
parameters{
|
||||
choice(
|
||||
choices: ['lastSuccessful()', 'lastCompleted()', 'latestSavedBuild()'],
|
||||
description: 'Which build to use for copying artifacts',
|
||||
name: 'BUILD_SELECTOR'
|
||||
)}
|
||||
stages {
|
||||
stage('clear_all') {
|
||||
steps {
|
||||
sh 'rm -rf ium_z444439'
|
||||
}
|
||||
}
|
||||
stage('checkout') {
|
||||
steps {
|
||||
sh 'git clone https://git.wmi.amu.edu.pl/s444439/ium_z444439'
|
||||
}
|
||||
}
|
||||
stage('copy_artifacts') {
|
||||
steps {
|
||||
copyArtifacts filter: 'X_test.csv,X_dev.csv,X_train.csv', fingerprintArtifacts: true, projectName: 'z-s444439-create-dataset', selector: workspace()
|
||||
}
|
||||
}
|
||||
stage('Docker') {
|
||||
agent {
|
||||
dockerfile {
|
||||
filename 'Dockerfile_sec'
|
||||
dir 'ium_z444439'
|
||||
reuseNode true
|
||||
}
|
||||
}
|
||||
steps {
|
||||
sh 'ls -a'
|
||||
sh 'python ./ium_z444439/stats.py'
|
||||
echo 'process finish'
|
||||
archiveArtifacts 'X_test.csv'
|
||||
archiveArtifacts 'X_dev.csv'
|
||||
archiveArtifacts 'X_train.csv'
|
||||
echo 'finish'
|
||||
}
|
||||
}
|
||||
stage('Goodbye!') {
|
||||
steps {
|
||||
archiveArtifacts 'dataset.csv'
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
18
create-dataset.py
Normal file
18
create-dataset.py
Normal file
@ -0,0 +1,18 @@
|
||||
import pandas
|
||||
import os
|
||||
from sklearn.model_selection import train_test_split
|
||||
|
||||
CUTOFF = int(os.environ['CUTOFF'])
|
||||
adults = pandas.read_csv('./ium_z444439/adult.csv', engine='python', encoding='ISO-8859-1', sep=',')
|
||||
adults = adults.dropna()
|
||||
|
||||
adults = adults.sample(CUTOFF)
|
||||
|
||||
X, Y = adults, adults
|
||||
|
||||
X_train, X_temp, Y_train, Y_temp = train_test_split(X, Y, test_size=0.3, random_state=1)
|
||||
X_dev, X_test, Y_dev, Y_test = train_test_split(X_temp, Y_temp, test_size=0.3, random_state=1)
|
||||
|
||||
X_train.to_csv('X_train.csv', index=False)
|
||||
X_dev.to_csv('X_dev.csv', index=False)
|
||||
X_test.to_csv('X_test.csv', index=False)
|
10
stats.py
Normal file
10
stats.py
Normal file
@ -0,0 +1,10 @@
|
||||
import pandas
|
||||
|
||||
X_dev = pandas.read_csv('X_dev.csv', engine='python', encoding='ISO-8859-1', sep=',')
|
||||
X_train = pandas.read_csv('X_train.csv', engine='python', encoding='ISO-8859-1', sep=',')
|
||||
|
||||
X_test = pandas.read_csv('X_test.csv', engine='python', encoding='ISO-8859-1', sep=',')
|
||||
|
||||
X_dev.describe(include='all').to_csv('X_dev_stats.csv', index=True)
|
||||
X_train.describe(include='all').to_csv('X_train_stats.csv', index=True)
|
||||
X_test.describe(include='all').to_csv('X_test_stats.csv', index=True)
|
Loading…
Reference in New Issue
Block a user