s444439/ium_z444439
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
s444439 2023-04-19 19:46:29 +02:00
parent e3e995ea3f
commit c891fcf53f
4 changed files with 43 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Jenkinsfile vendored
View File

@ -1,6 +1,5 @@
pipeline {
agent any
parameters{
string(
defaultValue: '500',
@ -10,7 +9,7 @@ pipeline {
)
}
stages {
stage('clear_all') {
stage('clear') {
steps {
sh 'rm -rf *'
}
@ -39,9 +38,9 @@ agent {
sh 'ls -a'
sh 'python ./ium_z444439/create-dataset.py'
echo 'process finish'
archiveArtifacts 'X_test.csv'
archiveArtifacts 'X_dev.csv'
archiveArtifacts 'X_train.csv'
archiveArtifacts 'adult_test.csv'
archiveArtifacts 'adult_dev.csv'
archiveArtifacts 'adult_train.csv'
}
}
}

18
Jenkinsfile_stats
View File

@ -7,9 +7,9 @@ pipeline {
name: 'BUILD_SELECTOR'
)}
stages {
stage('clear_all') {
stage('clear') {
steps {
sh 'rm -rf ium_z444439'
sh 'rm -rf *'
}
}
stage('checkout') {
@ -19,13 +19,13 @@ pipeline {
}
stage('copy_artifacts') {
steps {
copyArtifacts filter: 'X_test.csv,X_dev.csv,X_train.csv', fingerprintArtifacts: true, projectName: 'z-s444439-create-dataset', selector: workspace()
copyArtifacts filter: 'adult_test.csv,adult_dev.csv,adult_train.csv', fingerprintArtifacts: true, projectName: 'z-s444439-create-dataset', selector: workspace()
}
}
stage('Docker') {
agent {
dockerfile {
filename 'Dockerfile_sec'
filename 'Dockerfile'
dir 'ium_z444439'
reuseNode true
}
@ -34,16 +34,14 @@ pipeline {
sh 'ls -a'
sh 'python ./ium_z444439/stats.py'
echo 'process finish'
archiveArtifacts 'X_test.csv'
archiveArtifacts 'X_dev.csv'
archiveArtifacts 'X_train.csv'
echo 'finish'
archiveArtifacts 'adult_test_stats.csv'
archiveArtifacts 'adult_dev_stats.csv'
archiveArtifacts 'adult_train_stats.csv'
}
}
stage('Goodbye!') {
steps {
archiveArtifacts 'dataset.csv'
sh 'rm -rf *'
}
}
}

15
create-dataset.py
View File

@ -8,11 +8,12 @@ adults = adults.dropna()
adults = adults.sample(CUTOFF)
X, Y = adults, adults
adult_X, adult_Y = adults, adults
adult_X_train, adult_X_temp, adult_Y_train, adult_Y_temp = train_test_split(adult_X, adult_Y, test_size=0.3,
random_state=1)
adult_X_dev, adult_X_test, adult_Y_dev, adult_Y_test = train_test_split(adult_X_temp, adult_Y_temp, test_size=0.3,
random_state=1)
X_train, X_temp, Y_train, Y_temp = train_test_split(X, Y, test_size=0.3, random_state=1)
X_dev, X_test, Y_dev, Y_test = train_test_split(X_temp, Y_temp, test_size=0.3, random_state=1)
X_train.to_csv('X_train.csv', index=False)
X_dev.to_csv('X_dev.csv', index=False)
X_test.to_csv('X_test.csv', index=False)
adult_X_train.to_csv('adult_train.csv', index=False)
adult_X_dev.to_csv('adult_dev.csv', index=False)
adult_X_test.to_csv('adult_test.csv', index=False)

12
stats.py
View File

@ -1,10 +1,10 @@
import pandas
X_dev = pandas.read_csv('X_dev.csv', engine='python', encoding='ISO-8859-1', sep=',')
X_train = pandas.read_csv('X_train.csv', engine='python', encoding='ISO-8859-1', sep=',')
adult_dev = pandas.read_csv('adult_dev.csv', engine='python', encoding='ISO-8859-1', sep=',')
adult_train = pandas.read_csv('adult_train.csv', engine='python', encoding='ISO-8859-1', sep=',')
X_test = pandas.read_csv('X_test.csv', engine='python', encoding='ISO-8859-1', sep=',')
adult_test = pandas.read_csv('adult_test.csv', engine='python', encoding='ISO-8859-1', sep=',')
X_dev.describe(include='all').to_csv('X_dev_stats.csv', index=True)
X_train.describe(include='all').to_csv('X_train_stats.csv', index=True)
X_test.describe(include='all').to_csv('X_test_stats.csv', index=True)
adult_dev.describe(include='all').to_csv('adult_dev_stats.csv', index=True)
adult_train.describe(include='all').to_csv('adult_train_stats.csv', index=True)
adult_test.describe(include='all').to_csv('adult_test_stats.csv', index=True)