s495719/ium_495719
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

change script

Browse Source
This commit is contained in:
PawelDopierala 2024-04-02 20:40:42 +02:00
parent f0730957eb
commit cbc64f8115
2 changed files with 25 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Jenkinsfile vendored
View File

@ -35,14 +35,17 @@ pipeline {
steps { steps {
withEnv(["KAGGLE_USERNAME=${params.KAGGLE_USERNAME}", withEnv(["KAGGLE_USERNAME=${params.KAGGLE_USERNAME}",
"KAGGLE_KEY=${params.KAGGLE_KEY}" ]) { "KAGGLE_KEY=${params.KAGGLE_KEY}" ]) {
sh 'chmod 777 ./data_processing.sh' sh 'chmod 777 ./data_processing.py'
sh "./data_processing.sh ${params.CUTOFF}" sh "./data_processing.py ${params.CUTOFF}"
} }
} }
} }
stage('Artifacts') { stage('Artifacts') {
steps { steps {
archiveArtifacts artifacts: 'processed_data.txt' def artifactsList = ['hp_train.csv', 'hp_dev.csv', 'hp_test.csv'']
artifactsList.each { artifact ->
archiveArtifacts artifacts: artifact
}
} }
} }
} }

19
data_processing.py Normal file
View File

@ -0,0 +1,19 @@
import sklearn
from sklearn.preprocessing import OneHotEncoder
from sklearn.model_selection import train_test_split
import pandas as pd
import subprocess
subprocess.run(["kaggle", "datasets", "download", "muhammadbinimran/housing-price-prediction-data", "--unzip"])
housing_price_dataset = pd.read_csv('housing_price_dataset.csv')
hp_train_test, hp_dev = sklearn.model_selection.train_test_split(housing_price_dataset, test_size=0.1)
hp_train, hp_test = sklearn.model_selection.train_test_split(hp_train_test, test_size=1000)
hp_train = pd.get_dummies(hp_train, columns=['Neighborhood'])
hp_dev = pd.get_dummies(hp_dev, columns=['Neighborhood'])
hp_test = pd.get_dummies(hp_test, columns=['Neighborhood'])
hp_train.to_csv('hp_train.csv', index=False)
hp_dev.to_csv('hp_dev.csv', index=False)
hp_test.to_csv('hp_test.csv', index=False)