From b1d7953dfaf933ed2c7bc573a5d46bbd9df9fa95 Mon Sep 17 00:00:00 2001
From: Mateusz Kuc <matkuc4@st.amu.edu.pl>
Date: Fri, 30 Jun 2023 19:16:40 +0200
Subject: [PATCH] data preparation

---
 get-data.sh        |  5 +++++
 prepare_dataset.py | 22 ++++++++++++++++++++++
 2 files changed, 27 insertions(+)
 create mode 100644 get-data.sh
 create mode 100644 prepare_dataset.py

diff --git a/get-data.sh b/get-data.sh
new file mode 100644
index 0000000..5332f02
--- /dev/null
+++ b/get-data.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+kaggle datasets download zynicide/wine-reviews
+mkdir -p data
+unzip -o wine-reviews.zip -d data
+rm wine-reviews.zip
diff --git a/prepare_dataset.py b/prepare_dataset.py
new file mode 100644
index 0000000..6c29eb4
--- /dev/null
+++ b/prepare_dataset.py
@@ -0,0 +1,22 @@
+import pandas as pd
+import os
+from sklearn.model_selection import train_test_split
+
+CUTOFF = int(os.environ['CUTOFF'])
+
+wines = pd.read_csv('data/winemag-data_first150k.csv', engine='python', encoding='ISO-8859-1', sep=',')
+
+wines = wines.dropna()
+
+wines = wines.sample(100)
+
+X, Y = wines, wines
+
+# SPLIT BETWEEN DEV, TRAINS, AND TEST
+wines_train, wines_temp, wines_train, wines_temp = train_test_split(X, Y, test_size=0.2, random_state=1)
+wines_dev, wines_test, wines_dev, wines_test = train_test_split(wines_temp, wines_temp, test_size=0.2)
+
+wines_train.to_csv('wines_train.csv', index=False)
+wines_dev.to_csv('wines_dev.csv', index=False)
+wines_test.to_csv('wines_test.csv', index=False)
+