s444439/ium_z444439
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
s444439 2023-04-19 19:46:29 +02:00
parent e3e995ea3f
commit c891fcf53f
4 changed files with 43 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Jenkinsfile vendored
View File

@ -1,6 +1,5 @@
pipeline { pipeline {
agent any agent any
parameters{ parameters{
string( string(
defaultValue: '500', defaultValue: '500',
@ -10,7 +9,7 @@ pipeline {
) )
} }
stages { stages {
stage('clear_all') { stage('clear') {
steps { steps {
sh 'rm -rf *' sh 'rm -rf *'
} }
@ -39,9 +38,9 @@ agent {
sh 'ls -a' sh 'ls -a'
sh 'python ./ium_z444439/create-dataset.py' sh 'python ./ium_z444439/create-dataset.py'
echo 'process finish' echo 'process finish'
archiveArtifacts 'X_test.csv' archiveArtifacts 'adult_test.csv'
archiveArtifacts 'X_dev.csv' archiveArtifacts 'adult_dev.csv'
archiveArtifacts 'X_train.csv' archiveArtifacts 'adult_train.csv'
} }
} }
} }

18
Jenkinsfile_stats
View File

@ -7,9 +7,9 @@ pipeline {
name: 'BUILD_SELECTOR' name: 'BUILD_SELECTOR'
)} )}
stages { stages {
stage('clear_all') { stage('clear') {
steps { steps {
sh 'rm -rf ium_z444439' sh 'rm -rf *'
} }
} }
stage('checkout') { stage('checkout') {
@ -19,13 +19,13 @@ pipeline {
} }
stage('copy_artifacts') { stage('copy_artifacts') {
steps { steps {
copyArtifacts filter: 'X_test.csv,X_dev.csv,X_train.csv', fingerprintArtifacts: true, projectName: 'z-s444439-create-dataset', selector: workspace() copyArtifacts filter: 'adult_test.csv,adult_dev.csv,adult_train.csv', fingerprintArtifacts: true, projectName: 'z-s444439-create-dataset', selector: workspace()
} }
} }
stage('Docker') { stage('Docker') {
agent { agent {
dockerfile { dockerfile {
filename 'Dockerfile_sec' filename 'Dockerfile'
dir 'ium_z444439' dir 'ium_z444439'
reuseNode true reuseNode true
} }
@ -34,16 +34,14 @@ pipeline {
sh 'ls -a' sh 'ls -a'
sh 'python ./ium_z444439/stats.py' sh 'python ./ium_z444439/stats.py'
echo 'process finish' echo 'process finish'
archiveArtifacts 'X_test.csv' archiveArtifacts 'adult_test_stats.csv'
archiveArtifacts 'X_dev.csv' archiveArtifacts 'adult_dev_stats.csv'
archiveArtifacts 'X_train.csv' archiveArtifacts 'adult_train_stats.csv'
echo 'finish'
} }
} }
stage('Goodbye!') { stage('Goodbye!') {
steps { steps {
archiveArtifacts 'dataset.csv' sh 'rm -rf *'
} }
} }
} }

15
create-dataset.py
View File

@ -8,11 +8,12 @@ adults = adults.dropna()
adults = adults.sample(CUTOFF) adults = adults.sample(CUTOFF)
X, Y = adults, adults adult_X, adult_Y = adults, adults
adult_X_train, adult_X_temp, adult_Y_train, adult_Y_temp = train_test_split(adult_X, adult_Y, test_size=0.3,
random_state=1)
adult_X_dev, adult_X_test, adult_Y_dev, adult_Y_test = train_test_split(adult_X_temp, adult_Y_temp, test_size=0.3,
random_state=1)
X_train, X_temp, Y_train, Y_temp = train_test_split(X, Y, test_size=0.3, random_state=1) adult_X_train.to_csv('adult_train.csv', index=False)
X_dev, X_test, Y_dev, Y_test = train_test_split(X_temp, Y_temp, test_size=0.3, random_state=1) adult_X_dev.to_csv('adult_dev.csv', index=False)
adult_X_test.to_csv('adult_test.csv', index=False)
X_train.to_csv('X_train.csv', index=False)
X_dev.to_csv('X_dev.csv', index=False)
X_test.to_csv('X_test.csv', index=False)

12
stats.py
View File

@ -1,10 +1,10 @@
import pandas import pandas
X_dev = pandas.read_csv('X_dev.csv', engine='python', encoding='ISO-8859-1', sep=',') adult_dev = pandas.read_csv('adult_dev.csv', engine='python', encoding='ISO-8859-1', sep=',')
X_train = pandas.read_csv('X_train.csv', engine='python', encoding='ISO-8859-1', sep=',') adult_train = pandas.read_csv('adult_train.csv', engine='python', encoding='ISO-8859-1', sep=',')
X_test = pandas.read_csv('X_test.csv', engine='python', encoding='ISO-8859-1', sep=',') adult_test = pandas.read_csv('adult_test.csv', engine='python', encoding='ISO-8859-1', sep=',')
X_dev.describe(include='all').to_csv('X_dev_stats.csv', index=True) adult_dev.describe(include='all').to_csv('adult_dev_stats.csv', index=True)
X_train.describe(include='all').to_csv('X_train_stats.csv', index=True) adult_train.describe(include='all').to_csv('adult_train_stats.csv', index=True)
X_test.describe(include='all').to_csv('X_test_stats.csv', index=True) adult_test.describe(include='all').to_csv('adult_test_stats.csv', index=True)