Compare commits

...

2 Commits

Author SHA1 Message Date
4d70607765 Merge remote-tracking branch 'origin/main'
# Conflicts:
#	Jenkinsfile
2024-03-26 19:06:32 +01:00
66dd18de06 jenkinsfile 2024-03-26 18:56:58 +01:00
2 changed files with 51 additions and 2 deletions

22
Jenkinsfile vendored
View File

@ -1,10 +1,28 @@
pipeline {
agent any
parameters {
string(name: 'KAGGLE_USERNAME', defaultValue: '', description: 'Kaggle username')
password(name: 'KAGGLE_KEY', defaultValue:'', description: 'Kaggle Key')
}
stages {
stage('Stage 1') {
stage('Git Checkout') {
steps {
echo 'Test!'
git "https://git.wmi.amu.edu.pl/s464914/ium_464914.git"
}
}
stage('Cleanup') {
steps {
sh 'rm -rf artifacts'
}
}
}
stage('Build') {
withEnv(["KAGGLE_USERNAME=${params.KAGGLE_USERNAME}",
"KAGGLE_KEY=${params.KAGGLE_KEY}" ]) {
sh 'bash ./get_dataset.sh'
}
}
}

31
get_dataset.sh Normal file
View File

@ -0,0 +1,31 @@
#!/bin/bash
pip install kaggle --upgrade
kaggle datasets download -d nasa/meteorite-landings
unzip -o meteorite-landings.zip
###Zmienne###
train_ratio=0.8
test_val_ratio=0.5
##Przetwrazanie pliku##
shuf meteorite-landings.csv -o shuffled-meteorite-landings.csv
total_lines=$(wc -l < shuffled-meteorite-landings.csv)
train_lines=$(echo $total_lines*$train_ratio| bc)
train_lines=$(echo "($train_lines+0.5)/1" | bc )
test_lines=$(echo "($total_lines-$train_lines)*$test_val_ratio" | bc)
test_lines=$(echo "($test_lines+0.5)/1" | bc )
validation_lines=$(echo $total_lines-$train_lines-$test_lines | bc)
head -n "$train_lines" shuffled-meteorite-landings.csv > "meteorite_train.csv"
tail -n $((test_lines+validation_lines)) shuffled-meteorite-landings.csv | head -n "$test_lines" > "meteorite_test.csv"
tail -n "$validation_lines" shuffled-meteorite-landings.csv > "meteorite_validation.csv"
mkdir -p artifacts
mv meteorite-landings.csv shuffled-meteorite-landings.csv meteorite_test.csv meteorite_train.csv meteorite_validation.csv artifacts/