s464863/ium_464863
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

IUM_04 - back to old Jenkinsfile, old download_dataset script

Browse Source
This commit is contained in:
Paweł Łączkowski 2024-04-03 23:29:46 +02:00
parent 29ed2c18ca
commit 68c257e7fb
2 changed files with 25 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
Jenkinsfile vendored
View File

@ -1,15 +1,5 @@
pipeline {
agent {
dockerfile {
filename 'Dockerfile'
reuseNode true
}
}
environment {
KAGGLE_USERNAME = credentials('KAGGLE_USERNAME')
KAGGLE_KEY = credentials('KAGGLE_KEY')
}
agent any
parameters {
password (
@ -36,18 +26,30 @@ pipeline {
}
}
stage('Download dataset and preprocess data') {
stage('Download dataset') {
steps {
script {
sh "echo ${env.KAGGLE_USERNAME}"
sh "echo ${KAGGLE_USERNAME}"
sh "echo ${CUTOFF}"
sh "echo ${params.CUTOFF}"
sh "chmod +x ./download_dataset.py"
sh "python3 ./download_dataset.py ${params.CUTOFF}"
archiveArtifacts artifacts: 'datasets/*', onlyIfSuccessful: true
withEnv(["KAGGLE_USERNAME=${params.KAGGLE_USERNAME}", "KAGGLE_KEY=${params.KAGGLE_KEY}"]) {
sh "kaggle datasets download -d uciml/breast-cancer-wisconsin-data"
sh "unzip -o breast-cancer-wisconsin-data.zip"
sh "mkdir -p datasets"
sh "mv data.csv datasets/data.csv"
}
}
}
stage('Preprocess data') {
agent {
dockerfile {
filename 'Dockerfile'
reuseNode true
}
}
steps {
sh "chmod +x ./download_dataset.py"
sh "python3 ./download_dataset.py ${params.CUTOFF}"
archiveArtifacts artifacts: 'datasets/*', onlyIfSuccessful: true
}
}
}
}

6
download_dataset.py
View File

@ -1,13 +1,13 @@
# Necessary imports
import pandas as pd
import kaggle
# import kaggle
import sys
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
# Download the dataset
kaggle.api.authenticate()
kaggle.api.dataset_download_files('uciml/breast-cancer-wisconsin-data', path='./datasets', unzip=True)
# kaggle.api.authenticate()
# kaggle.api.dataset_download_files('uciml/breast-cancer-wisconsin-data', path='./datasets', unzip=True)
# Load the dataset
df = pd.read_csv('./datasets/data.csv', index_col='id')