s464979/ium_464979
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added new jenkinsfile and kuggle download script

Browse Source
This commit is contained in:
AWieczarek 2024-03-24 17:49:57 +01:00
parent 68c3e170d0
commit 8143cba976
1 changed files with 16 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
kuggle_download.sh
View File

@ -1,30 +1,30 @@
#!/bin/bash #!/bin/bash
# Pobieranie i rozpakowanie
pip install kaggle pip install kaggle
kaggle datasets download -d thedevastator/1-5-million-beer-reviews-from-beer-advocate kaggle datasets download -d thedevastator/1-5-million-beer-reviews-from-beer-advocate
unzip -o 1-5-million-beer-reviews-from-beer-advocate.zip unzip -o 1-5-million-beer-reviews-from-beer-advocate.zip
DATASET_FILE="beer_reviews.csv"
shuf beer_reviews.csv > shuffled_dataset.csv head -n $1 $DATASET_FILE > cutoff_$DATASET_FILE
pwd # Mieszanie i podział
ls -a total_lines=$(tail -n +2 cutoff_$DATASET_FILE | wc -l)
split -l 80000 shuffled_dataset.csv train.csv train_lines=$((total_lines * 80 / 100))
split -l 10000 train.csv dev.csv dev_lines=$((total_lines * 10 / 100))
mv shuffled_dataset.csv test.csv test_lines=$((total_lines - train_lines - dev_lines))
head -n 1000 train.csv > train_head.csv shuf cutoff_$DATASET_FILE -o shuffled.csv
tail -n 1000 train.csv > train_tail.csv
if [ -n "$CUTOFF" ]; then head -n $train_lines shuffled.csv > train.csv
head -n "$CUTOFF" train.csv > train_cutoff.csv tail -n $((dev_lines + test_lines)) shuffled.csv | head -n $dev_lines > dev.csv
fi tail -n $test_lines shuffled.csv > test.csv
tar -czf artifacts.tar.gz train.csv dev.csv test.csv train_head.csv train_tail.csv train_cutoff.csv # Archiwizacja
tar -czf artifacts.tar.gz train.csv dev.csv test.csv
rm $DATASET_FILE shuffled_dataset.csv # Czyszczenie
rm cutoff_$DATASET_FILE shuffled.csv
echo "artifacts.tar.gz" echo "artifacts.tar.gz"