zad5 model i skrypty

This commit is contained in:
Michal Gulczynski 2024-04-14 21:59:34 +02:00
parent 941f1161c9
commit 390d6b118b
6 changed files with 178 additions and 4 deletions

View File

@ -11,12 +11,16 @@ RUN apt-get update && \
unzip \
&& rm -rf /var/lib/apt/lists/*
RUN pip3 install pandas scikit-learn requests kaggle
RUN pip3 install pandas scikit-learn requests kaggle numpy
COPY /app/
COPY /app/
COPY /app/
COPY /app/
RUN chmod +x
CMD ["python3", ""]
RUN chmod +x
RUN chmod +x
CMD ["bash", ""]

model.pkl Normal file

Binary file not shown.

127 Normal file
View File

@ -0,0 +1,127 @@
import pandas as pd
import os
import numpy as np
from kaggle.api.kaggle_api_extended import KaggleApi
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
from sklearn.preprocessing import StandardScaler
from sklearn. preprocessing import LabelEncoder
import pickle
def download_dataset(dataset_address, destination_folder):
api = KaggleApi()
api.dataset_download_files(dataset_address, path=destination_folder, unzip=True)
def check_datasets_presence():
dataset_1 = "Spotify_Dataset.csv"
dataset_2 = "spotify_songs.csv"
destination_folder = "datasets"
if not os.path.exists(destination_folder):
print(f"Utworzono folder: {destination_folder}")
print(f"Folder {destination_folder} już istnieje.")
if dataset_1 not in os.listdir(destination_folder):
download_dataset('gulczas/spotify-dataset', destination_folder)
if dataset_2 not in os.listdir(destination_folder):
download_dataset('joebeachcapital/30000-spotify-songs', destination_folder)
def datasets_preparation():
df_1 = pd.read_csv("datasets/spotify_songs.csv")
df_2 = pd.read_csv("datasets/Spotify_Dataset.csv", sep=";")
df_1 = df_1.dropna()
df_2 = df_2.dropna()
df_2 = df_2.rename(columns={'Title': 'track_name'})
columns_to_remove_df_1 = ['track_id', 'track_album_id', 'track_album_name', 'track_album_release_date',
'playlist_id', 'playlist_subgenre']
columns_to_remove_df_2 = ['Date','# of Artist', 'Artist (Ind.)', '# of Nationality',
'Nationality', 'Continent', 'Points (Total)',
'Points (Ind for each Artist/Nat)', 'id', 'Song URL']
df_1 = df_1.drop(columns=columns_to_remove_df_1)
df_2 = df_2.drop(columns=columns_to_remove_df_2)
df_1 = df_1.drop_duplicates(subset=['track_name'])
df_2 = df_2.drop_duplicates(subset=['track_name'])
le = LabelEncoder()
unique_names_df2 = df_2['track_name'].unique()
diff_df = df_1[~df_1['track_name'].isin(unique_names_df2)]
diff_df = diff_df.iloc[:10000]
#diff_df = pd.concat([diff_df, df_1.iloc[:20]], ignore_index=True)
diff_df['track_artist'] = le.fit_transform(diff_df.track_artist)
diff_df['playlist_name'] = le.fit_transform(diff_df.playlist_name)
diff_df['playlist_genre'] = le.fit_transform(diff_df.playlist_genre)
#df_1 = df_1.iloc[20:]
if "docker_test_dataset.csv" not in os.listdir("datasets"):
diff_df.to_csv("datasets/docker_test_dataset.csv", index=False)
result_df = pd.merge(df_1, df_2, on='track_name', how='inner')
result_df = result_df.drop_duplicates(subset=['track_name'])
columns_to_remove_result_df = ['Rank', 'Artists', 'Danceability', 'Energy', 'Loudness',
'Speechiness', 'Acousticness', 'Instrumentalness', 'Valence']
result_df = result_df.drop(columns=columns_to_remove_result_df)
result_df['track_artist'] = le.fit_transform(result_df.track_artist)
result_df['playlist_name'] = le.fit_transform(result_df.playlist_name)
result_df['playlist_genre'] = le.fit_transform(result_df.playlist_genre)
return result_df
result_df = datasets_preparation()
Y = result_df[['playlist_genre']]
X = result_df.drop(columns='playlist_genre')
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.10, random_state=42)
Y_train = np.ravel(Y_train)
Y_test = np.ravel(Y_test)
scaler = StandardScaler()
numeric_columns = X_train.select_dtypes(include=['int', 'float']).columns
X_train_scaled = scaler.fit_transform(X_train[numeric_columns])
X_test_scaled = scaler.transform(X_test[numeric_columns])
model = LogisticRegression(max_iter=1000), Y_train)
Y_pred = model.predict(X_test_scaled)
accuracy = accuracy_score(Y_test, Y_pred)
print("Accuracy:", accuracy)
file_path = 'model.pkl'
if os.path.exists(file_path):
if file_path not in os.listdir("./"):
with open(file_path, 'wb') as file:
pickle.dump(model, file)
print("Model został zapisany do pliku:", file_path)

model_predictions.txt Normal file
View File

@ -0,0 +1,3 @@
Real:['edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm', 'edm']
Predicted: ['pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop', 'pop']

4 Normal file
View File

@ -0,0 +1,4 @@

36 Normal file
View File

@ -0,0 +1,36 @@
import pickle
import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score
file_path = 'model.pkl'
with open(file_path, 'rb') as file:
model = pickle.load(file)
print("Model został wczytany z pliku:", file_path)
test_df = pd.read_csv("datasets/docker_test_dataset.csv")
Y_test = test_df[['playlist_genre']]
X_test = test_df.drop(columns='playlist_genre')
Y_test = np.ravel(Y_test)
scaler = StandardScaler()
numeric_columns = X_test.select_dtypes(include=['int', 'float']).columns
X_test_scaled = scaler.fit_transform(X_test[numeric_columns])
Y_pred = model.predict(X_test_scaled)
with open('model_predictions.txt', 'w') as f:
with open('model_predictions.txt', 'a') as f:
labels_dict = {0: 'edm', 1 : 'latin', 2 : 'pop', 3 : 'r&b', 4 : 'rap', 5 :'rock'}
Y_test_labels = [labels_dict[number] for number in Y_test]
Y_pred_labels = [labels_dict[number] for number in Y_pred]
f.write("Real:" + str(Y_test_labels[:20])+ " \nPredicted: "+ str(Y_pred_labels[:20]))
accuracy = accuracy_score(Y_test, Y_pred)
f.write("\nAccuracy:" + str(accuracy))