diff --git a/create_dataset.py b/create_dataset.py new file mode 100644 index 0000000..85f9e9a --- /dev/null +++ b/create_dataset.py @@ -0,0 +1,46 @@ + +#Downloading DataSet from kaggle +from kaggle.api.kaggle_api_extended import KaggleApi +api = KaggleApi() +api.authenticate() +api.dataset_download_file('mssmartypants/water-quality', file_name='waterQuality1.csv') + +#Reading DataSet from csv using pandas library +import pandas as pd +water = pd.read_csv('waterQuality1.csv') + +#water.describe(include='all') + +# Clearing DataSet from non digit values in is_safe column +water = water[water['is_safe'].apply(lambda x: str(x).isdigit())] +water['is_safe'].value_counts() + +# Splitting DataSet on train, dev, test parts +from sklearn.model_selection import train_test_split +water_train, water_test = train_test_split(water, train_size=0.8, random_state=1, stratify=water["is_safe"]) +water_test, water_dev = train_test_split(water_test, train_size=0.66, random_state=1, stratify=water_test["is_safe"]) +water_train["is_safe"].value_counts() +water_test["is_safe"].value_counts() +water_dev["is_safe"].value_counts() + +print(f''' +Statystyki zbioru: +Wielkość zbioru - {len(water)} +Wielkość podzbioru treningowego - {len(water_train)} +Wielkość podzbioru walidującego - {len(water_dev)} +Wielkość podzbioru testowego - {len(water_test)} + +Rozkład częstości parametru mówiącemu o zdantości picia wody (0 oznacza zdanty do picia): +''') + +#water["is_safe"].value_counts().plot(kind="bar") + +# Normalizing Dataset to [0.0, 1.0] float values +from sklearn import preprocessing +water_min_max = preprocessing.MinMaxScaler() +water_min_max = water_min_max.fit_transform(water) +water_min_max = pd.DataFrame(water_min_max, columns=water.columns) +waterNorm = water_min_max + +waterNorm.to_csv('waterQuality.csv', index=False) +