diff --git a/Dockerfile b/Dockerfile index e47e6ae..8ff1399 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,12 +11,16 @@ RUN apt-get update && \ unzip \ && rm -rf /var/lib/apt/lists/* -RUN pip3 install pandas scikit-learn requests kaggle +RUN pip3 install pandas scikit-learn requests kaggle numpy WORKDIR /app -COPY data_preparation_script.py /app/ +COPY model_creator.py /app/ +COPY use_model.py /app/ +COPY run_py_scripts.sh /app/ -RUN chmod +x data_preparation_script.py -CMD ["python3", "data_preparation_script.py"] \ No newline at end of file +RUN chmod +x model_creator.py +RUN chmod +x use_model.py + +CMD ["bash", "run_py_scripts.sh"] \ No newline at end of file diff --git a/model.pkl b/model.pkl new file mode 100644 index 0000000..b54a828 Binary files /dev/null and b/model.pkl differ diff --git a/model_creator.py b/model_creator.py new file mode 100644 index 0000000..81e9cd4 --- /dev/null +++ b/model_creator.py @@ -0,0 +1,127 @@ +import pandas as pd +import os +import numpy as np +from kaggle.api.kaggle_api_extended import KaggleApi +from sklearn.model_selection import train_test_split +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import accuracy_score +from sklearn.preprocessing import StandardScaler +from sklearn. preprocessing import LabelEncoder +import pickle + + +def download_dataset(dataset_address, destination_folder): + + api = KaggleApi() + api.authenticate() + + api.dataset_download_files(dataset_address, path=destination_folder, unzip=True) + + +def check_datasets_presence(): + + dataset_1 = "Spotify_Dataset.csv" + dataset_2 = "spotify_songs.csv" + destination_folder = "datasets" + + if not os.path.exists(destination_folder): + os.makedirs(destination_folder) + print(f"Utworzono folder: {destination_folder}") + else: + print(f"Folder {destination_folder} już istnieje.") + + if dataset_1 not in os.listdir(destination_folder): + download_dataset('gulczas/spotify-dataset', destination_folder) + + if dataset_2 not in os.listdir(destination_folder): + download_dataset('joebeachcapital/30000-spotify-songs', destination_folder) + + +def datasets_preparation(): + df_1 = pd.read_csv("datasets/spotify_songs.csv") + df_2 = pd.read_csv("datasets/Spotify_Dataset.csv", sep=";") + + df_1 = df_1.dropna() + df_2 = df_2.dropna() + df_2 = df_2.rename(columns={'Title': 'track_name'}) + + columns_to_remove_df_1 = ['track_id', 'track_album_id', 'track_album_name', 'track_album_release_date', + 'playlist_id', 'playlist_subgenre'] + columns_to_remove_df_2 = ['Date','# of Artist', 'Artist (Ind.)', '# of Nationality', + 'Nationality', 'Continent', 'Points (Total)', + 'Points (Ind for each Artist/Nat)', 'id', 'Song URL'] + + df_1 = df_1.drop(columns=columns_to_remove_df_1) + df_2 = df_2.drop(columns=columns_to_remove_df_2) + df_1 = df_1.drop_duplicates(subset=['track_name']) + df_2 = df_2.drop_duplicates(subset=['track_name']) + + le = LabelEncoder() + + unique_names_df2 = df_2['track_name'].unique() + diff_df = df_1[~df_1['track_name'].isin(unique_names_df2)] + diff_df = diff_df.iloc[:10000] + + #diff_df = pd.concat([diff_df, df_1.iloc[:20]], ignore_index=True) + diff_df['track_artist'] = le.fit_transform(diff_df.track_artist) + diff_df['playlist_name'] = le.fit_transform(diff_df.playlist_name) + diff_df['playlist_genre'] = le.fit_transform(diff_df.playlist_genre) + + #df_1 = df_1.iloc[20:] + + if "docker_test_dataset.csv" not in os.listdir("datasets"): + diff_df.to_csv("datasets/docker_test_dataset.csv", index=False) + + result_df = pd.merge(df_1, df_2, on='track_name', how='inner') + result_df = result_df.drop_duplicates(subset=['track_name']) + columns_to_remove_result_df = ['Rank', 'Artists', 'Danceability', 'Energy', 'Loudness', + 'Speechiness', 'Acousticness', 'Instrumentalness', 'Valence'] + result_df = result_df.drop(columns=columns_to_remove_result_df) + + result_df['track_artist'] = le.fit_transform(result_df.track_artist) + result_df['playlist_name'] = le.fit_transform(result_df.playlist_name) + result_df['playlist_genre'] = le.fit_transform(result_df.playlist_genre) + + return result_df + + +check_datasets_presence() +result_df = datasets_preparation() +Y = result_df[['playlist_genre']] +X = result_df.drop(columns='playlist_genre') +X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.10, random_state=42) + + +Y_train = np.ravel(Y_train) +Y_test = np.ravel(Y_test) + +scaler = StandardScaler() +numeric_columns = X_train.select_dtypes(include=['int', 'float']).columns +X_train_scaled = scaler.fit_transform(X_train[numeric_columns]) +X_test_scaled = scaler.transform(X_test[numeric_columns]) + +model = LogisticRegression(max_iter=1000) +model.fit(X_train_scaled, Y_train) + + +Y_pred = model.predict(X_test_scaled) + +accuracy = accuracy_score(Y_test, Y_pred) +print("Accuracy:", accuracy) + +file_path = 'model.pkl' + +if os.path.exists(file_path): + os.remove(file_path) + +if file_path not in os.listdir("./"): + with open(file_path, 'wb') as file: + pickle.dump(model, file) + +print("Model został zapisany do pliku:", file_path) + + + + + + diff --git a/model_predictions.txt b/model_predictions.txt new file mode 100644 index 0000000..2c42627 --- /dev/null +++ b/model_predictions.txt @@ -0,0 +1,3 @@ +Real:['edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm'] +Predicted: ['pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop'] +Accuracy:0.1521 \ No newline at end of file diff --git a/run_py_scripts.sh b/run_py_scripts.sh new file mode 100644 index 0000000..7762456 --- /dev/null +++ b/run_py_scripts.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +python3 model_creator.py +python3 use_model.py diff --git a/use_model.py b/use_model.py new file mode 100644 index 0000000..5cbff03 --- /dev/null +++ b/use_model.py @@ -0,0 +1,36 @@ +import pickle +import pandas as pd +import numpy as np +from sklearn.preprocessing import StandardScaler +from sklearn.metrics import accuracy_score + +np.set_printoptions(threshold=20) + +file_path = 'model.pkl' +with open(file_path, 'rb') as file: + model = pickle.load(file) +print("Model został wczytany z pliku:", file_path) + +test_df = pd.read_csv("datasets/docker_test_dataset.csv") + +Y_test = test_df[['playlist_genre']] +X_test = test_df.drop(columns='playlist_genre') +Y_test = np.ravel(Y_test) + +scaler = StandardScaler() +numeric_columns = X_test.select_dtypes(include=['int', 'float']).columns +X_test_scaled = scaler.fit_transform(X_test[numeric_columns]) + +Y_pred = model.predict(X_test_scaled) + +with open('model_predictions.txt', 'w') as f: + pass + +with open('model_predictions.txt', 'a') as f: + labels_dict = {0: 'edm', 1 : 'latin', 2 : 'pop', 3 : 'r&b', 4 : 'rap', 5 :'rock'} + Y_test_labels = [labels_dict[number] for number in Y_test] + Y_pred_labels = [labels_dict[number] for number in Y_pred] + f.write("Real:" + str(Y_test_labels[:20])+ " \nPredicted: "+ str(Y_pred_labels[:20])) + accuracy = accuracy_score(Y_test, Y_pred) + f.write("\nAccuracy:" + str(accuracy)) +