46 KiB
46 KiB
Self made simplified I-KNN
pip install surpriseCollecting surprise Downloading https://files.pythonhosted.org/packages/61/de/e5cba8682201fcf9c3719a6fdda95693468ed061945493dea2dd37c5618b/surprise-0.1-py2.py3-none-any.whl Collecting scikit-surprise [?25l Downloading https://files.pythonhosted.org/packages/f5/da/b5700d96495fb4f092be497f02492768a3d96a3f4fa2ae7dea46d4081cfa/scikit-surprise-1.1.0.tar.gz (6.4MB) [K |████████████████████████████████| 6.5MB 3.1MB/s [?25hRequirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise->surprise) (0.15.1) Requirement already satisfied: numpy>=1.11.2 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise->surprise) (1.18.5) Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise->surprise) (1.4.1) Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise->surprise) (1.12.0) Building wheels for collected packages: scikit-surprise Building wheel for scikit-surprise (setup.py) ... [?25l[?25hdone Created wheel for scikit-surprise: filename=scikit_surprise-1.1.0-cp36-cp36m-linux_x86_64.whl size=1675356 sha256=d7c5907ea98c2add6a69db05dd4db7830a0b82a20313c621fbbfe0c7670db104 Stored in directory: /root/.cache/pip/wheels/cc/fa/8c/16c93fccce688ae1bde7d979ff102f7bee980d9cfeb8641bcf Successfully built scikit-surprise Installing collected packages: scikit-surprise, surprise Successfully installed scikit-surprise-1.1.0 surprise-0.1
import helpers
import pandas as pd
import numpy as np
import scipy.sparse as sparse
from collections import defaultdict
from itertools import chain
import random
train_read=pd.read_csv('./Datasets/ml-100k/train.csv', sep='\t', header=None)
test_read=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\t', header=None)
train_ui, test_ui, user_code_id, user_id_code, item_code_id, item_id_code = helpers.data_to_csr(train_read, test_read)class IKNN():
def fit(self, train_ui):
self.train_ui=train_ui
train_iu=train_ui.transpose()
norms=np.linalg.norm(train_iu.A, axis=1) # here we compute lenth of each item ratings vector
norms=np.vectorize(lambda x: max(x,1))(norms[:,None]) # to avoid dividing by zero
normalized_train_iu=sparse.csr_matrix(train_iu/norms)
self.similarity_matrix_ii=normalized_train_iu*normalized_train_iu.transpose()
self.estimations=np.array(train_ui*self.similarity_matrix_ii/((train_ui>0)*self.similarity_matrix_ii))
def recommend(self, user_code_id, item_code_id, topK=10):
top_k = defaultdict(list)
for nb_user, user in enumerate(self.estimations):
user_rated=self.train_ui.indices[self.train_ui.indptr[nb_user]:self.train_ui.indptr[nb_user+1]]
for item, score in enumerate(user):
if item not in user_rated and not np.isnan(score):
top_k[user_code_id[nb_user]].append((item_code_id[item], score))
result=[]
# Let's choose k best items in the format: (user, item1, score1, item2, score2, ...)
for uid, item_scores in top_k.items():
item_scores.sort(key=lambda x: x[1], reverse=True)
result.append([uid]+list(chain(*item_scores[:topK])))
return result
def estimate(self, user_code_id, item_code_id, test_ui):
result=[]
for user, item in zip(*test_ui.nonzero()):
result.append([user_code_id[user], item_code_id[item],
self.estimations[user,item] if not np.isnan(self.estimations[user,item]) else 1])
return result# toy example
toy_train_read=pd.read_csv('./Datasets/toy-example/train.csv', sep='\t', header=None, names=['user', 'item', 'rating', 'timestamp'])
toy_test_read=pd.read_csv('./Datasets/toy-example/test.csv', sep='\t', header=None, names=['user', 'item', 'rating', 'timestamp'])
toy_train_ui, toy_test_ui, toy_user_code_id, toy_user_id_code, \
toy_item_code_id, toy_item_id_code = helpers.data_to_csr(toy_train_read, toy_test_read)
model=IKNN()
model.fit(toy_train_ui)
print('toy train ui:')
display(toy_train_ui.A)
print('similarity matrix:')
display(model.similarity_matrix_ii.A)
print('estimations matrix:')
display(model.estimations)
model.recommend(toy_user_code_id, toy_item_code_id)toy train ui:
array([[3, 4, 0, 0, 5, 0, 0, 4],
[0, 1, 2, 3, 0, 0, 0, 0],
[0, 0, 0, 5, 0, 3, 4, 0]], dtype=int64)similarity matrix:
array([[1. , 0.9701425 , 0. , 0. , 1. ,
0. , 0. , 1. ],
[0.9701425 , 1. , 0.24253563, 0.12478355, 0.9701425 ,
0. , 0. , 0.9701425 ],
[0. , 0.24253563, 1. , 0.51449576, 0. ,
0. , 0. , 0. ],
[0. , 0.12478355, 0.51449576, 1. , 0. ,
0.85749293, 0.85749293, 0. ],
[1. , 0.9701425 , 0. , 0. , 1. ,
0. , 0. , 1. ],
[0. , 0. , 0. , 0.85749293, 0. ,
1. , 1. , 0. ],
[0. , 0. , 0. , 0.85749293, 0. ,
1. , 1. , 0. ],
[1. , 0.9701425 , 0. , 0. , 1. ,
0. , 0. , 1. ]])estimations matrix:
array([[4. , 4. , 4. , 4. , 4. ,
nan, nan, 4. ],
[1. , 1.35990333, 2.15478388, 2.53390319, 1. ,
3. , 3. , 1. ],
[ nan, 5. , 5. , 4.05248907, nan,
3.95012863, 3.95012863, nan]])[[0, 20, 4.0, 30, 4.0], [10, 50, 3.0, 60, 3.0, 0, 1.0, 40, 1.0, 70, 1.0], [20, 10, 5.0, 20, 5.0]]
model=IKNN()
model.fit(train_ui)
top_n=pd.DataFrame(model.recommend(user_code_id, item_code_id, topK=10))
top_n.to_csv('Recommendations generated/ml-100k/Self_IKNN_reco.csv', index=False, header=False)
estimations=pd.DataFrame(model.estimate(user_code_id, item_code_id, test_ui))
estimations.to_csv('Recommendations generated/ml-100k/Self_IKNN_estimations.csv', index=False, header=False)import evaluation_measures as ev
estimations_df=pd.read_csv('Recommendations generated/ml-100k/Self_IKNN_estimations.csv', header=None)
reco=np.loadtxt('Recommendations generated/ml-100k/Self_IKNN_reco.csv', delimiter=',')
ev.evaluate(test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\t', header=None),
estimations_df=estimations_df,
reco=reco,
super_reactions=[4,5])943it [00:00, 8432.28it/s]
| RMSE | MAE | precision | recall | F_1 | F_05 | precision_super | recall_super | NDCG | mAP | MRR | LAUC | HR | Reco in test | Test coverage | Shannon | Gini | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 1.018363 | 0.808793 | 0.000318 | 0.000108 | 0.00014 | 0.000189 | 0.0 | 0.0 | 0.000214 | 0.000037 | 0.000368 | 0.496391 | 0.003181 | 0.392153 | 0.11544 | 4.174741 | 0.965327 |
import imp
imp.reload(ev)
import evaluation_measures as ev
dir_path="Recommendations generated/ml-100k/"
super_reactions=[4,5]
test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\t', header=None)
ev.evaluate_all(test, dir_path, super_reactions)943it [00:00, 8766.27it/s]
| Model | RMSE | MAE | precision | recall | F_1 | F_05 | precision_super | recall_super | NDCG | mAP | MRR | LAUC | HR | Reco in test | Test coverage | Shannon | Gini | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | Self_IKNN | 1.018363 | 0.808793 | 0.000318 | 0.000108 | 0.00014 | 0.000189 | 0.0 | 0.0 | 0.000214 | 0.000037 | 0.000368 | 0.496391 | 0.003181 | 0.392153 | 0.11544 | 4.174741 | 0.965327 |
Ready-made KNNs - Surprise implementation
I-KNN - basic
import helpers
import surprise as sp
import imp
imp.reload(helpers)
sim_options = {'name': 'cosine',
'user_based': False} # compute similarities between items
algo = sp.KNNBasic(sim_options=sim_options)
helpers.ready_made(algo, reco_path='Recommendations generated/ml-100k/Ready_I-KNN_reco.csv',
estimations_path='Recommendations generated/ml-100k/Ready_I-KNN_estimations.csv')Computing the cosine similarity matrix... Done computing similarity matrix. Generating predictions... Generating top N recommendations... Generating predictions...
U-KNN - basic
import helpers
import surprise as sp
import imp
imp.reload(helpers)
sim_options = {'name': 'cosine',
'user_based': True} # compute similarities between users
algo = sp.KNNBasic(sim_options=sim_options)
helpers.ready_made(algo, reco_path='Recommendations generated/ml-100k/Ready_U-KNN_reco.csv',
estimations_path='Recommendations generated/ml-100k/Ready_U-KNN_estimations.csv')Computing the cosine similarity matrix... Done computing similarity matrix. Generating predictions... Generating top N recommendations... Generating predictions...
I-KNN - on top baseline
import helpers
import surprise as sp
import imp
imp.reload(helpers)
sim_options = {'name': 'cosine',
'user_based': False} # compute similarities between items
algo = sp.KNNBaseline()
helpers.ready_made(algo, reco_path='Recommendations generated/ml-100k/Ready_I-KNNBaseline_reco.csv',
estimations_path='Recommendations generated/ml-100k/Ready_I-KNNBaseline_estimations.csv')Estimating biases using als... Computing the msd similarity matrix... Done computing similarity matrix. Generating predictions... Generating top N recommendations... Generating predictions...
project task 4: use a version of your choice of Surprise KNNalgorithm
# read the docs and try to find best parameter configuration (let say in terms of RMSE)
# https://surprise.readthedocs.io/en/stable/knn_inspired.html##surprise.prediction_algorithms.knns.KNNBaseline
# the solution here can be similar to examples above
# please save the output in 'Recommendations generated/ml-100k/Self_KNNSurprisetask_reco.csv' and
# 'Recommendations generated/ml-100k/Self_KNNSurprisetask_estimations.csv'
import helpers
import surprise as sp
import imp
imp.reload(helpers)
sim_options = {'name': 'cosine',
'user_based': False}
algo = sp.KNNWithMeans() #KNNWithMeans parametr domyślny k = 40
helpers.ready_made(algo, reco_path='Recommendations generated/ml-100k/Ready_I-KNNWithMeans_reco.csv',
estimations_path='Recommendations generated/ml-100k/Ready_I-KNNWithMeans_estimations.csv')Computing the msd similarity matrix... Done computing similarity matrix. Generating predictions... Generating top N recommendations... Generating predictions...
import helpers
import surprise as sp
import imp
imp.reload(helpers)
sim_options = {'name': 'cosine',
'user_based': False}
algo = sp.KNNWithMeans(k=60) #KNNWithMeans parametr k =60
helpers.ready_made(algo, reco_path='Recommendations generated/ml-100k/Ready_I2-KNNWithMeans_reco.csv',
estimations_path='Recommendations generated/ml-100k/Ready_I2-KNNWithMeans_estimations.csv')Computing the msd similarity matrix... Done computing similarity matrix. Generating predictions... Generating top N recommendations... Generating predictions...
import helpers
import surprise as sp
import imp
imp.reload(helpers)
sim_options = {'name': 'cosine',
'user_based': False}
algo = sp.KNNWithZScore() #KNNWithMeans parametr domyślny k = 40
helpers.ready_made(algo, reco_path='Recommendations generated/ml-100k/Ready_I-KNNWithZScore_reco.csv',
estimations_path='Recommendations generated/ml-100k/Ready_I-KNNWithZScore_estimations.csv')Computing the msd similarity matrix... Done computing similarity matrix. Generating predictions... Generating top N recommendations... Generating predictions...
import helpers
import surprise as sp
import imp
imp.reload(helpers)
sim_options = {'name': 'cosine',
'user_based': False}
algo = sp.KNNWithZScore(k=60) #KNNWithScore parametr k =60
helpers.ready_made(algo, reco_path='Recommendations generated/ml-100k/Ready_I2-KNNWithZScore_reco.csv',
estimations_path='Recommendations generated/ml-100k/Ready_I2-KNNWithZScore_estimations.csv')Computing the msd similarity matrix... Done computing similarity matrix. Generating predictions... Generating top N recommendations... Generating predictions...
import helpers
import surprise as sp
import imp
imp.reload(helpers)
sim_options = {'name': 'cosine',
'user_based': False} #KNNWithMeans parametr domyślny k = 40
algo = sp.KNNBaseline()
helpers.ready_made(algo, reco_path='Recommendations generated/ml-100k/Ready_I-KNNBaseline_reco.csv',
estimations_path='Recommendations generated/ml-100k/Ready_I-KNNBaseline_estimations.csv')Estimating biases using als... Computing the msd similarity matrix... Done computing similarity matrix. Generating predictions... Generating top N recommendations... Generating predictions...
import helpers
import surprise as sp
import imp
imp.reload(helpers)
sim_options = {'name': 'cosine',
'user_based': False}
algo = sp.KNNBaseline(k=20) #KNNWithMeans parametr k =20
helpers.ready_made(algo, reco_path='Recommendations generated/ml-100k/Ready_I-KNNBaseline2_reco.csv',
estimations_path='Recommendations generated/ml-100k/Ready_I-KNNBaseline2_estimations.csv')Estimating biases using als... Computing the msd similarity matrix... Done computing similarity matrix. Generating predictions... Generating top N recommendations... Generating predictions...
import helpers
import surprise as sp
import imp
imp.reload(helpers)
sim_options = {'name': 'cosine',
'user_based': False}
algo = sp.KNNBaseline(k=60) #KNNWithBaseline parametr k =60
helpers.ready_made(algo, reco_path='Recommendations generated/ml-100k/Ready_I-KNNBaseline3_reco.csv',
estimations_path='Recommendations generated/ml-100k/Ready_I-KNNBaseline3_estimations.csv')Estimating biases using als... Computing the msd similarity matrix... Done computing similarity matrix. Generating predictions... Generating top N recommendations... Generating predictions...
import imp
imp.reload(ev)
import evaluation_measures as ev
dir_path="Recommendations generated/ml-100k/"
super_reactions=[4,5]
test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\t', header=None)
ev.evaluate_all(test, dir_path, super_reactions)
# baseline z parametrem domyślnym k = 40 ma najmniejszy RMSE, MAE i recall, posiada za to dość niską precyzję943it [00:00, 8651.21it/s] 943it [00:00, 7695.97it/s] 943it [00:00, 8273.01it/s] 943it [00:00, 8476.77it/s] 943it [00:00, 8678.60it/s] 943it [00:00, 8516.82it/s] 943it [00:00, 7812.26it/s] 943it [00:00, 8528.21it/s] 943it [00:00, 8808.80it/s] 943it [00:00, 8706.93it/s]
| Model | RMSE | MAE | precision | recall | F_1 | F_05 | precision_super | recall_super | NDCG | mAP | MRR | LAUC | HR | Reco in test | Test coverage | Shannon | Gini | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | Ready_I-KNN | 1.030386 | 0.813067 | 0.026087 | 0.006908 | 0.010593 | 0.016046 | 0.021137 | 0.009522 | 0.024214 | 0.008958 | 0.048068 | 0.499885 | 0.154825 | 0.402333 | 0.434343 | 5.133650 | 0.877999 |
| 0 | Ready_I-KNNWithMeans | 0.955921 | 0.754037 | 0.004984 | 0.003225 | 0.003406 | 0.003956 | 0.004506 | 0.003861 | 0.006815 | 0.002906 | 0.020332 | 0.497969 | 0.039236 | 0.587699 | 0.071429 | 2.699278 | 0.991353 |
| 0 | Ready_I2-KNNWithMeans | 0.955530 | 0.753259 | 0.004666 | 0.002998 | 0.003190 | 0.003716 | 0.004185 | 0.003621 | 0.006575 | 0.002757 | 0.020919 | 0.497854 | 0.037116 | 0.587275 | 0.067821 | 2.675131 | 0.991838 |
| 0 | Ready_I2-KNNWithZScore | 0.956736 | 0.751215 | 0.003924 | 0.002134 | 0.002513 | 0.003078 | 0.003755 | 0.002633 | 0.004906 | 0.002065 | 0.013621 | 0.497419 | 0.026511 | 0.387275 | 0.061328 | 2.427288 | 0.993420 |
| 0 | Ready_I-KNNWithZScore | 0.957701 | 0.752387 | 0.003712 | 0.001994 | 0.002380 | 0.002919 | 0.003433 | 0.002401 | 0.005137 | 0.002158 | 0.016458 | 0.497349 | 0.027572 | 0.389926 | 0.067821 | 2.475747 | 0.992793 |
| 0 | Ready_I-KNNBaseline2 | 0.939085 | 0.740225 | 0.003818 | 0.001196 | 0.001716 | 0.002455 | 0.003541 | 0.001513 | 0.004876 | 0.002211 | 0.013878 | 0.496949 | 0.025451 | 0.493531 | 0.082251 | 2.386656 | 0.992495 |
| 0 | Ready_I-KNNBaseline3 | 0.935828 | 0.737925 | 0.002757 | 0.000874 | 0.001255 | 0.001785 | 0.002468 | 0.001071 | 0.003760 | 0.001593 | 0.012014 | 0.496784 | 0.021209 | 0.480382 | 0.054113 | 2.206297 | 0.994802 |
| 0 | Ready_I-KNNBaseline | 0.935327 | 0.737424 | 0.002545 | 0.000755 | 0.001105 | 0.001602 | 0.002253 | 0.000930 | 0.003444 | 0.001362 | 0.011760 | 0.496724 | 0.021209 | 0.482821 | 0.059885 | 2.232578 | 0.994487 |
| 0 | Ready_U-KNN | 1.023495 | 0.807913 | 0.000742 | 0.000205 | 0.000305 | 0.000449 | 0.000536 | 0.000198 | 0.000845 | 0.000274 | 0.002744 | 0.496441 | 0.007423 | 0.602121 | 0.010823 | 2.089186 | 0.995706 |
| 0 | Self_IKNN | 1.018363 | 0.808793 | 0.000318 | 0.000108 | 0.000140 | 0.000189 | 0.000000 | 0.000000 | 0.000214 | 0.000037 | 0.000368 | 0.496391 | 0.003181 | 0.392153 | 0.115440 | 4.174741 | 0.965327 |