114 lines
3.4 KiB
Python
114 lines
3.4 KiB
Python
#!/usr/bin/env python
|
|
|
|
# Import bibliotek
|
|
|
|
import os
|
|
import shutil
|
|
import pandas as pd
|
|
from sklearn.model_selection import train_test_split
|
|
import requests
|
|
from sklearn.preprocessing import MinMaxScaler
|
|
from kaggle.api.kaggle_api_extended import KaggleApi
|
|
|
|
|
|
#funkcja pobierająca plik
|
|
|
|
def download_file(url, filename, destination_folder):
|
|
# Wersja dla datasetów kaggle
|
|
|
|
api = KaggleApi()
|
|
api.authenticate()
|
|
|
|
api.dataset_download_files('gulczas/spotify-dataset', path=destination_folder, unzip=True)
|
|
|
|
|
|
# funkcja dzieląca zbiór
|
|
|
|
def split_dataset(data, test_size=0.2, val_size=0.1, random_state=42):
|
|
#Podział na test i trening
|
|
train_data, test_data = train_test_split(data, test_size=test_size, random_state=random_state)
|
|
#Podział na walidacje i trening
|
|
train_data, val_data = train_test_split(train_data, test_size=val_size/(1-test_size), random_state=random_state)
|
|
|
|
return train_data, val_data, test_data
|
|
|
|
|
|
# Wyświetlanie statystyk zbioru
|
|
|
|
def print_dataset_stats(data, subset_name):
|
|
with open('stats.txt', 'a') as stats_file:
|
|
print(f"Statystyki dla zbioru {subset_name}:", file=stats_file)
|
|
print(f"Wielkość zbioru {subset_name}: {len(data)}", file=stats_file)
|
|
|
|
print("\nStatystyki wartości poszczególnych parametrów:", file=stats_file)
|
|
print(data.describe(), file=stats_file)
|
|
|
|
for column in data.columns:
|
|
print(f"Rozkład częstości dla kolumny '{column}':", file=stats_file)
|
|
print(data[column].value_counts(), file=stats_file)
|
|
print("\n", file=stats_file)
|
|
|
|
# Normalizacja danych
|
|
|
|
def normalize_data(data):
|
|
scaler = MinMaxScaler()
|
|
numeric_columns = data.select_dtypes(include=['int', 'float']).columns
|
|
scaler.fit(data[numeric_columns])
|
|
df_normalized = data.copy()
|
|
df_normalized[numeric_columns] = scaler.transform(df_normalized[numeric_columns])
|
|
return df_normalized
|
|
|
|
#Czyszczenie danych
|
|
|
|
def clean_dataset(data):
|
|
data.dropna(inplace=True)
|
|
data.drop_duplicates(inplace=True)
|
|
return data
|
|
|
|
|
|
# main
|
|
|
|
url = "https://www.kaggle.com/datasets/gulczas/spotify-dataset?select=Spotify_Dataset.csv"
|
|
filename = "Spotify_Dataset.csv"
|
|
destination_folder = "datasets"
|
|
|
|
# Pobieranie jeśli nie ma już pobranego pliku
|
|
if not os.path.exists(destination_folder):
|
|
os.makedirs(destination_folder)
|
|
print(f"Utworzono folder: {destination_folder}")
|
|
else:
|
|
print(f"Folder {destination_folder} już istnieje.")
|
|
|
|
if 'Spotify_Dataset.csv' not in os.listdir(destination_folder):
|
|
# Pobranie pliku
|
|
filepath = download_file(url, filename, destination_folder)
|
|
|
|
# Wczytanie danych z pliku CSV
|
|
data = pd.read_csv("datasets/Spotify_Dataset.csv", sep=";")
|
|
|
|
# Podział datasetu na zbiory treningowy, walidacyjny i testowy
|
|
train_data, val_data, test_data = split_dataset(data)
|
|
|
|
# Zapisanie podzielonych zbiorów danych do osobnych plików CSV
|
|
train_data.to_csv("datasets/train.csv", index=False)
|
|
val_data.to_csv("datasets/val.csv", index=False)
|
|
test_data.to_csv("datasets/test.csv", index=False)
|
|
|
|
# Wydrukowanie statystyk dla zbiorów
|
|
print_dataset_stats(train_data, "treningowego")
|
|
print("\n")
|
|
print_dataset_stats(val_data, "walidacyjnego")
|
|
print("\n")
|
|
print_dataset_stats(test_data, "testowego")
|
|
|
|
# Normalizacja i czyszczenie zbirów
|
|
train_data = normalize_data(train_data)
|
|
train_data = clean_dataset(train_data)
|
|
val_data = normalize_data(train_data)
|
|
val_data = clean_dataset(train_data)
|
|
test_data = normalize_data(train_data)
|
|
test_data = clean_dataset(train_data)
|
|
|
|
|
|
|