s478841/ium_478841
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

OHE added for categorical data in preprocessing

Browse Source
This commit is contained in:
MatOgr 2022-04-23 16:47:43 +02:00
parent 3587927003
commit 4cd5906fe3
1 changed files with 25 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
scripts/grab_avocado.py
View File

@ -1,22 +1,41 @@
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder
cols = list(pd.read_csv("data/avocado.csv", nrows=1))
# print("###\n", cols, "\n###")
avocados = pd.read_csv("data/avocado.csv", usecols=cols[1:])
avocados = pd.read_csv(
"data/avocado.csv").rename(columns={"Unnamed: 0": 'Week'})
avocados.describe(include="all")
float_cols = ['AveragePrice','Total Volume','4046','4225','4770','Total Bags','Small Bags','Large Bags','XLarge Bags']
# * columns containing float values to
float_cols = ['AveragePrice', 'Total Volume', '4046', '4225',
'4770', 'Total Bags', 'Small Bags', 'Large Bags', 'XLarge Bags']
avocados.loc[:, float_cols] = StandardScaler(
).fit_transform(avocados.loc[:, float_cols])
enc = OneHotEncoder(handle_unknown='ignore')
encoded_region = enc.fit_transform(
avocados['region'].to_numpy().reshape(-1, 1)).toarray()
encoded_region_frame = pd.DataFrame(
encoded_region, columns=enc.get_feature_names_out())
encoded_types = enc.fit_transform(
avocados['type'].to_numpy().reshape(-1, 1)).toarray()
encoded_types_frame = pd.DataFrame(
encoded_types, columns=enc.get_feature_names_out())
avocados = pd.concat([avocados, encoded_types_frame, encoded_region_frame], axis=1).drop(
['type', 'region', 'Date'], axis=1)
avocados.loc[:, float_cols] = StandardScaler().fit_transform(avocados.loc[:, float_cols])
print(avocados.head())
# avocados.loc[:, float_cols] = MinMaxScaler().fit_transform(avocados.loc[:, float_cols])
# print(avocados.head())
avocado_train, avocado_test = train_test_split(avocados, test_size=2000, random_state=3337)
avocado_train, avocado_valid = train_test_split(avocado_train, test_size=2249, random_state=3337)
avocado_train, avocado_test = train_test_split(
avocados, test_size=2000, random_state=3337)
avocado_train, avocado_valid = train_test_split(
avocado_train, test_size=2249, random_state=3337)
print("Train\n", avocado_train.describe(include="all"), "\n")
print("Valid\n", avocado_valid.describe(include="all"), "\n")