2022-03-27 17:43:59 +02:00
|
|
|
dataset_operation() {
|
2022-03-27 22:30:47 +02:00
|
|
|
tail -n +2 dataset.csv | shuf > dataset.csv.s
|
|
|
|
|
head -n $CUTOFF dataset.csv.s > ./dataset.csv.shuf
|
2022-03-27 17:43:59 +02:00
|
|
|
len1=$(cat ./imdb-dataset.csv.shuf | wc -l)
|
|
|
|
|
len2=$(($len1/10))
|
|
|
|
|
len3=$(($len2*2))
|
|
|
|
|
len4=$(($len3+1))
|
2022-03-27 22:30:47 +02:00
|
|
|
head -n $len2 dataset.csv.shuf > dataset_test.csv
|
|
|
|
|
head -n $len3 dataset.csv.shuf | tail -n $len2 > dataset_dev.csv.
|
|
|
|
|
tail -n +$len4 dataset.csv.shuf > dataset_train.csv
|
|
|
|
|
rm dataset.csv.shuf
|
|
|
|
|
wc -l dataset.csv.*
|
2022-03-27 17:43:59 +02:00
|
|
|
}
|
|
|
|
|
|
2022-03-27 22:30:47 +02:00
|
|
|
echo 'Start'
|
2022-03-27 17:43:59 +02:00
|
|
|
kaggle datasets download -d harshitshankhdhar/imdb-dataset-of-top-1000-movies-and-tv-shows
|
2022-03-27 22:30:47 +02:00
|
|
|
echo 'Dataset downloaded'
|
2022-03-27 17:43:59 +02:00
|
|
|
unzip imdb-dataset-of-top-1000-movies-and-tv-shows.zip
|
2022-03-27 22:30:47 +02:00
|
|
|
echo 'Dataset unziped'
|
|
|
|
|
mv imdb-dataset-of-top-1000-movies-and-tv-shows.zip dataset.zip
|
|
|
|
|
echo 'Dataset renamed'
|
|
|
|
|
archiveArtifacts artifacts: 'dataset_dev.csv, dataset_test.csv, dataset_train.csv', followSymlinks: false
|
