88 lines
2.6 KiB
Python
Executable File
88 lines
2.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
from kaggle.api.kaggle_api_extended import KaggleApi
|
|
import pandas as pd
|
|
import matplotlib.pyplot as plt
|
|
from sklearn.model_selection import train_test_split
|
|
|
|
pd.set_option("display.max_rows", None)
|
|
|
|
|
|
def column_stat(analyzed_set, column_name):
|
|
rating_min = analyzed_set[column_name].min()
|
|
rating_max = analyzed_set[column_name].max()
|
|
rating_mean = round(analyzed_set[column_name].mean(), 3)
|
|
rating_median = analyzed_set[column_name].median()
|
|
rating_std = round(analyzed_set[column_name].std(), 3)
|
|
|
|
output = ''
|
|
|
|
output += f"Dla kolumny '{column_name}':\n"
|
|
output += f"Minimum: {rating_min}\n"
|
|
output += f"Maximum: {rating_max}\n"
|
|
output += f"Średnia: {rating_mean}\n"
|
|
output += f"Mediana: {rating_median}\n"
|
|
output += f"Odchylenie standardowe: {rating_std}\n"
|
|
|
|
return output
|
|
|
|
|
|
# Pobieranie danych
|
|
api = KaggleApi()
|
|
api.authenticate()
|
|
api.dataset_download_files('arushchillar/disneyland-reviews', unzip=True)
|
|
disney = pd.read_csv('DisneylandReviews.csv', encoding='latin-1')
|
|
|
|
# Nie zauważyłem w pliku żadnych artefaktów, które trzeba wyczyścić
|
|
|
|
# Normalizacja kolumny 'Ratings' z przedziału [1;5] do przedziału [0;1]
|
|
disney['Rating'] = (disney['Rating'] - 1) / 4
|
|
|
|
# Normalizacja kolumny 'Review_Text' do lowercase
|
|
disney['Review_Text'] = disney['Review_Text'].str.lower()
|
|
|
|
|
|
# Podział na podzbiory: d_train, d_test, d_dev
|
|
d_train, d_test = train_test_split(disney, test_size=0.2, random_state=1, stratify=disney["Branch"])
|
|
d_dev, d_test = train_test_split(d_test, test_size=0.5, random_state=1, stratify=d_test["Branch"])
|
|
|
|
# Zapis do plików
|
|
d_train.to_csv('d_train.csv', index=False)
|
|
d_test.to_csv('d_test.csv', index=False)
|
|
d_dev.to_csv('d_dev.csv', index=False)
|
|
|
|
# Statystyki
|
|
temp = ''
|
|
temp += f"Wielkość całego zbioru: {disney.shape[0]}\n"
|
|
temp += f"Inne statystyki:\n"
|
|
temp += column_stat(disney, 'Rating')
|
|
temp += '\n'
|
|
|
|
temp += f"Wielkość zbioru trenującego: {d_train.shape[0]}\n"
|
|
temp += f"Inne statystyki:\n"
|
|
temp += column_stat(d_train, 'Rating')
|
|
temp += '\n'
|
|
|
|
temp += f"Wielkość zbioru walidującego: {d_dev.shape[0]}\n"
|
|
temp += f"Inne statystyki:\n"
|
|
temp += column_stat(d_dev, 'Rating')
|
|
temp += '\n'
|
|
|
|
temp += f"Wielkość zbioru testowego: {d_test.shape[0]}\n"
|
|
temp += f"Inne statystyki:\n"
|
|
temp += column_stat(d_test, 'Rating')
|
|
temp += '\n'
|
|
|
|
with open('stats.txt', 'w+', encoding="utf-8") as f:
|
|
print(temp)
|
|
f.write(temp)
|
|
|
|
# Rozkład ocen dla każdego oddziału
|
|
try:
|
|
disney.hist(column='Rating', by='Branch', legend=True)
|
|
plt.suptitle('Rozkład ocen w całym zbiorze')
|
|
plt.show()
|
|
except:
|
|
print("Error drawing hist plot (Powinno działać w Pycharmie)")
|
|
|