s444463/ium_444463
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

a

Browse Source
This commit is contained in:
Mikołaj Pokrywka 2022-04-03 16:00:01 +02:00
parent ace853de66
commit c20abbb368
3 changed files with 8 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Jenkinsfile vendored
View File

@ -37,7 +37,7 @@ pipeline {
// sh './process_data.sh' // sh './process_data.sh'
sh 'ls' sh 'ls'
sh 'python3 ./download_data_and_process.py' sh 'python3 ./download_data_and_process.py'
archiveArtifacts artifacts: "data_test.csv, data_dev.csv, data_train.csv, column_titles.csv, fake_job_postings.csv" archiveArtifacts artifacts: "data_test.csv, data_dev.csv, data_train.csv, column_titles.csv, data.csv"
} }
} }
} }

8
download_data_and_process.py
View File

@ -5,7 +5,13 @@ import numpy as np
# kaggle.api.authenticate() # kaggle.api.authenticate()
# kaggle.api.dataset_download_files('shivamb/real-or-fake-fake-jobposting-prediction', path='fake_job_postings.csv', unzip=True) # kaggle.api.dataset_download_files('shivamb/real-or-fake-fake-jobposting-prediction', path='fake_job_postings.csv', unzip=True)
data=pd.read_csv('fake_job_postings.csv')
data=pd.read_csv('data.csv')
data = data.replace(np.nan, '', regex=True) data = data.replace(np.nan, '', regex=True)
print("="*20) print("="*20)

1
process_data.sh
View File

@ -4,7 +4,6 @@ echo $KAGGLE_USERNAME
kaggle datasets download -d shivamb/real-or-fake-fake-jobposting-prediction kaggle datasets download -d shivamb/real-or-fake-fake-jobposting-prediction
unzip -o real-or-fake-fake-jobposting-prediction.zip unzip -o real-or-fake-fake-jobposting-prediction.zip
ls ls
cat fake_job_postings.csv > data.csv
echo "Save column titles" echo "Save column titles"
head -n 1 fake_job_postings.csv > column_titles.csv head -n 1 fake_job_postings.csv > column_titles.csv
tail -n +2 fake_job_postings.csv > data_not_shuf.csv tail -n +2 fake_job_postings.csv > data_not_shuf.csv