63 lines
2.2 KiB
Python
63 lines
2.2 KiB
Python
|
|
#Downloading DataSet from kaggle - Replaced by Jenkins
|
|
#from kaggle.api.kaggle_api_extended import KaggleApi
|
|
#api = KaggleApi()
|
|
#api.authenticate()
|
|
#api.dataset_download_file('mssmartypants/water-quality', file_name='waterQuality1.csv')
|
|
|
|
import argparse
|
|
parser = argparse.ArgumentParser(description='Program for Downloading Dataset from Kaggle and Normalisation')
|
|
parser.add_argument('CUT', type=int, nargs='?', default=7999, help='Cut this number of rows from csv file')
|
|
args = parser.parse_args()
|
|
|
|
#Reading DataSet from csv using pandas library
|
|
import pandas as pd
|
|
water = pd.read_csv('waterQuality1.csv', nrows = args.CUT)
|
|
|
|
|
|
#water.describe(include='all')
|
|
|
|
# Clearing DataSet from non digit values in is_safe column
|
|
water = water[water['is_safe'].apply(lambda x: str(x).isdigit())]
|
|
water['is_safe'].value_counts()
|
|
|
|
|
|
# Normalizing Dataset to [0.0, 1.0] float values
|
|
from sklearn import preprocessing
|
|
water_min_max = preprocessing.MinMaxScaler()
|
|
water_min_max = water_min_max.fit_transform(water)
|
|
water_min_max = pd.DataFrame(water_min_max, columns=water.columns)
|
|
waterNorm = water_min_max
|
|
|
|
# Splitting DataSet on train, dev, test parts
|
|
from sklearn.model_selection import train_test_split
|
|
water_train, water_test = train_test_split(waterNorm, train_size=0.8, random_state=1, stratify=waterNorm["is_safe"])
|
|
water_test, water_dev = train_test_split(water_test, train_size=0.66, random_state=1, stratify=water_test["is_safe"])
|
|
# water_train["is_safe"].value_counts()
|
|
# water_test["is_safe"].value_counts()
|
|
# water_dev["is_safe"].value_counts()
|
|
|
|
# print(f'''
|
|
# Statystyki zbioru:
|
|
# Wielkość zbioru - {len(water)}
|
|
# Wielkość podzbioru treningowego - {len(water_train)}
|
|
# Wielkość podzbioru walidującego - {len(water_dev)}
|
|
# Wielkość podzbioru testowego - {len(water_test)}
|
|
|
|
# Rozkład częstości parametru mówiącemu o zdantości picia wody (0 oznacza zdanty do picia):
|
|
# ''')
|
|
|
|
#water["is_safe"].value_counts().plot(kind="bar")
|
|
|
|
|
|
|
|
directory ='dane'
|
|
if not os.path.exists(directory):
|
|
os.makedirs('dane')
|
|
|
|
waterNorm.to_csv('waterQuality.csv', index=False)
|
|
water_train.to_csv('dane/diamonds_train.csv', index=False)
|
|
water_test.to_csv('dane/diamonds_test.csv', index=False)
|
|
water_dev.to_csv('dane/diamonds_dev.csv', index=False)
|
|
|