23 KiB
23 KiB
Self made simplified I-KNN
import helpers
import pandas as pd
import numpy as np
import scipy.sparse as sparse
from collections import defaultdict
from itertools import chain
import random
train_read=pd.read_csv('./Datasets/ml-100k/train.csv', sep='\t', header=None)
test_read=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\t', header=None)
train_ui, test_ui, user_code_id, user_id_code, item_code_id, item_id_code = helpers.data_to_csr(train_read, test_read)class IKNN():
def fit(self, train_ui):
self.train_ui=train_ui
train_iu=train_ui.transpose()
norms=np.linalg.norm(train_iu.A, axis=1) # here we compute lenth of each item ratings vector
norms=np.vectorize(lambda x: max(x,1))(norms[:,None]) # to avoid dividing by zero
normalized_train_iu=sparse.csr_matrix(train_iu/norms)
self.similarity_matrix_ii=normalized_train_iu*normalized_train_iu.transpose()
self.estimations=np.array(train_ui*self.similarity_matrix_ii/((train_ui>0)*self.similarity_matrix_ii))
def recommend(self, user_code_id, item_code_id, topK=10):
top_k = defaultdict(list)
for nb_user, user in enumerate(self.estimations):
user_rated=self.train_ui.indices[self.train_ui.indptr[nb_user]:self.train_ui.indptr[nb_user+1]]
for item, score in enumerate(user):
if item not in user_rated and not np.isnan(score):
top_k[user_code_id[nb_user]].append((item_code_id[item], score))
result=[]
# Let's choose k best items in the format: (user, item1, score1, item2, score2, ...)
for uid, item_scores in top_k.items():
item_scores.sort(key=lambda x: x[1], reverse=True)
result.append([uid]+list(chain(*item_scores[:topK])))
return result
def estimate(self, user_code_id, item_code_id, test_ui):
result=[]
for user, item in zip(*test_ui.nonzero()):
result.append([user_code_id[user], item_code_id[item],
self.estimations[user,item] if not np.isnan(self.estimations[user,item]) else 1])
return result# toy example
toy_train_read=pd.read_csv('./Datasets/toy-example/train.csv', sep='\t', header=None, names=['user', 'item', 'rating', 'timestamp'])
toy_test_read=pd.read_csv('./Datasets/toy-example/test.csv', sep='\t', header=None, names=['user', 'item', 'rating', 'timestamp'])
toy_train_ui, toy_test_ui, toy_user_code_id, toy_user_id_code, \
toy_item_code_id, toy_item_id_code = helpers.data_to_csr(toy_train_read, toy_test_read)
model=IKNN()
model.fit(toy_train_ui)
print('toy train ui:')
display(toy_train_ui.A)
print('similarity matrix:')
display(model.similarity_matrix_ii.A)
print('estimations matrix:')
display(model.estimations)
model.recommend(toy_user_code_id, toy_item_code_id)toy train ui:
array([[3, 4, 0, 0, 5, 0, 0, 4],
[0, 1, 2, 3, 0, 0, 0, 0],
[0, 0, 0, 5, 0, 3, 4, 0]], dtype=int64)similarity matrix:
array([[1. , 0.9701425 , 0. , 0. , 1. ,
0. , 0. , 1. ],
[0.9701425 , 1. , 0.24253563, 0.12478355, 0.9701425 ,
0. , 0. , 0.9701425 ],
[0. , 0.24253563, 1. , 0.51449576, 0. ,
0. , 0. , 0. ],
[0. , 0.12478355, 0.51449576, 1. , 0. ,
0.85749293, 0.85749293, 0. ],
[1. , 0.9701425 , 0. , 0. , 1. ,
0. , 0. , 1. ],
[0. , 0. , 0. , 0.85749293, 0. ,
1. , 1. , 0. ],
[0. , 0. , 0. , 0.85749293, 0. ,
1. , 1. , 0. ],
[1. , 0.9701425 , 0. , 0. , 1. ,
0. , 0. , 1. ]])estimations matrix:
array([[4. , 4. , 4. , 4. , 4. ,
nan, nan, 4. ],
[1. , 1.35990333, 2.15478388, 2.53390319, 1. ,
3. , 3. , 1. ],
[ nan, 5. , 5. , 4.05248907, nan,
3.95012863, 3.95012863, nan]])[[0, 20, 4.0, 30, 4.0], [10, 50, 3.0, 60, 3.0, 0, 1.0, 40, 1.0, 70, 1.0], [20, 10, 5.0, 20, 5.0]]
model=IKNN()
model.fit(train_ui)
top_n=pd.DataFrame(model.recommend(user_code_id, item_code_id, topK=10))
top_n.to_csv('Recommendations generated/ml-100k/Self_IKNN_reco.csv', index=False, header=False)
estimations=pd.DataFrame(model.estimate(user_code_id, item_code_id, test_ui))
estimations.to_csv('Recommendations generated/ml-100k/Self_IKNN_estimations.csv', index=False, header=False)import evaluation_measures as ev
estimations_df=pd.read_csv('Recommendations generated/ml-100k/Self_IKNN_estimations.csv', header=None)
reco=np.loadtxt('Recommendations generated/ml-100k/Self_IKNN_reco.csv', delimiter=',')
ev.evaluate(test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\t', header=None),
estimations_df=estimations_df,
reco=reco,
super_reactions=[4,5])943it [00:00, 8068.73it/s]
| RMSE | MAE | precision | recall | F_1 | F_05 | precision_super | recall_super | NDCG | mAP | MRR | LAUC | HR | Reco in test | Test coverage | Shannon | Gini | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 1.018363 | 0.808793 | 0.000318 | 0.000108 | 0.00014 | 0.000189 | 0.0 | 0.0 | 0.000214 | 0.000037 | 0.000368 | 0.496391 | 0.003181 | 0.392153 | 0.11544 | 4.174741 | 0.965327 |
import imp
imp.reload(ev)
import evaluation_measures as ev
dir_path="Recommendations generated/ml-100k/"
super_reactions=[4,5]
test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\t', header=None)
ev.evaluate_all(test, dir_path, super_reactions)943it [00:00, 8130.43it/s] 943it [00:00, 7928.93it/s] 943it [00:00, 7309.27it/s] 943it [00:00, 7687.73it/s] 943it [00:00, 8153.31it/s] 943it [00:00, 7785.12it/s]
| Model | RMSE | MAE | precision | recall | F_1 | F_05 | precision_super | recall_super | NDCG | mAP | MRR | LAUC | HR | Reco in test | Test coverage | Shannon | Gini | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | Self_TopPop | 2.508258 | 2.217909 | 0.188865 | 0.116919 | 0.118732 | 0.141584 | 0.130472 | 0.137473 | 0.214651 | 0.111707 | 0.400939 | 0.555546 | 0.765642 | 1.000000 | 0.038961 | 3.159079 | 0.987317 |
| 0 | Ready_Baseline | 0.949459 | 0.752487 | 0.091410 | 0.037652 | 0.046030 | 0.061286 | 0.079614 | 0.056463 | 0.095957 | 0.043178 | 0.198193 | 0.515501 | 0.437964 | 1.000000 | 0.033911 | 2.836513 | 0.991139 |
| 0 | Self_GlobalAvg | 1.125760 | 0.943534 | 0.061188 | 0.025968 | 0.031383 | 0.041343 | 0.040558 | 0.032107 | 0.067695 | 0.027470 | 0.171187 | 0.509546 | 0.384942 | 1.000000 | 0.025974 | 2.711772 | 0.992003 |
| 0 | Ready_Random | 1.517787 | 1.217953 | 0.047826 | 0.017861 | 0.022711 | 0.031080 | 0.028219 | 0.016982 | 0.051154 | 0.019551 | 0.125693 | 0.505448 | 0.318134 | 0.986426 | 0.186869 | 5.091730 | 0.908288 |
| 0 | Self_BaselineUI | 0.967585 | 0.762740 | 0.000954 | 0.000170 | 0.000278 | 0.000463 | 0.000644 | 0.000189 | 0.000752 | 0.000168 | 0.001677 | 0.496424 | 0.009544 | 0.600530 | 0.005051 | 1.803126 | 0.996380 |
| 0 | Self_IKNN | 1.018363 | 0.808793 | 0.000318 | 0.000108 | 0.000140 | 0.000189 | 0.000000 | 0.000000 | 0.000214 | 0.000037 | 0.000368 | 0.496391 | 0.003181 | 0.392153 | 0.115440 | 4.174741 | 0.965327 |
Ready-made KNNs - Surprise implementation
I-KNN - basic
import helpers
import surprise as sp
import imp
imp.reload(helpers)
sim_options = {'name': 'cosine',
'user_based': False} # compute similarities between items
algo = sp.KNNBasic(sim_options=sim_options)
helpers.ready_made(algo, reco_path='Recommendations generated/ml-100k/Ready_I-KNN_reco.csv',
estimations_path='Recommendations generated/ml-100k/Ready_I-KNN_estimations.csv')Computing the cosine similarity matrix... Done computing similarity matrix. Generating predictions... Generating top N recommendations... Generating predictions...
U-KNN - basic
import helpers
import surprise as sp
import imp
imp.reload(helpers)
sim_options = {'name': 'cosine',
'user_based': True} # compute similarities between users
algo = sp.KNNBasic(sim_options=sim_options)
helpers.ready_made(algo, reco_path='Recommendations generated/ml-100k/Ready_U-KNN_reco.csv',
estimations_path='Recommendations generated/ml-100k/Ready_U-KNN_estimations.csv')Computing the cosine similarity matrix... Done computing similarity matrix. Generating predictions... Generating top N recommendations... Generating predictions...
I-KNN - on top baseline
import helpers
import surprise as sp
import imp
imp.reload(helpers)
sim_options = {'name': 'cosine',
'user_based': False} # compute similarities between items
algo = sp.KNNBaseline()
helpers.ready_made(algo, reco_path='Recommendations generated/ml-100k/Ready_I-KNNBaseline_reco.csv',
estimations_path='Recommendations generated/ml-100k/Ready_I-KNNBaseline_estimations.csv')Estimating biases using als... Computing the msd similarity matrix... Done computing similarity matrix. Generating predictions... Generating top N recommendations... Generating predictions...
project task 4: use a version of your choice of Surprise KNNalgorithm
# read the docs and try to find best parameter configuration (let say in terms of RMSE)
# https://surprise.readthedocs.io/en/stable/knn_inspired.html##surprise.prediction_algorithms.knns.KNNBaseline
# the solution here can be similar to examples above
# please save the output in 'Recommendations generated/ml-100k/Self_KNNSurprisetask_reco.csv' and
# 'Recommendations generated/ml-100k/Self_KNNSurprisetask_estimations.csv'