WSS-project/P6. WRMF (Implicit ALS).ipynb

import helpers
import pandas as pd
import numpy as np
import scipy.sparse as sparse
from collections import defaultdict
from itertools import chain
import random
import time
import matplotlib.pyplot as plt
import implicit
import evaluation_measures as ev

train_read=pd.read_csv('./Datasets/ml-100k/train.csv', sep='\t', header=None)
test_read=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\t', header=None)
train_ui, test_ui, user_code_id, user_id_code, item_code_id, item_id_code = helpers.data_to_csr(train_read, test_read)

alpha = 30
train_ui*=alpha
train_iu=train_ui.transpose().tocsr()

model = implicit.als.AlternatingLeastSquares(factors=200, regularization=0.1, iterations=10)
model.fit(train_iu)

WARNING:root:Intel MKL BLAS detected. Its highly recommend to set the environment variable 'export MKL_NUM_THREADS=1' to disable its internal multithreading

HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=10.0), HTML(value='')))

def top_k_recommendations(model, user_code_id, item_code_id, topK=10):
    recommendations=[]
    for u in range(train_ui.shape[0]):
        u_recommended_items=model.recommend(u, train_ui, N=10, filter_already_liked_items=True)
        recommendations.append([user_code_id[u]]+list(chain(*u_recommended_items)))
    reco=pd.DataFrame(recommendations)
    reco.iloc[:,1::2]=reco.iloc[:,1::2].applymap(lambda x: item_code_id[x])
    return reco

def estimate(model, user_code_id, item_code_id, test_ui):
    result=[]
    for user, item in zip(*test_ui.nonzero()):
        result.append([user_code_id[user], item_code_id[item], 
            model.rank_items(userid=user, user_items=train_ui, selected_items=[item])[0][1]])
    return result

reco=top_k_recommendations(model, user_code_id, item_code_id, topK=10)
reco.to_csv('Recommendations generated/ml-100k/Ready_ImplicitALS_reco.csv', index=False, header=False)

estimations_df=pd.DataFrame(estimate(model, user_code_id, item_code_id, test_ui))
estimations_df.to_csv('Recommendations generated/ml-100k/Ready_ImplicitALS_estimations.csv', index=False, header=False)

import evaluation_measures as ev
import imp
imp.reload(ev)

estimations_df=pd.read_csv('Recommendations generated/ml-100k/Ready_ImplicitALS_estimations.csv', header=None)
reco=np.loadtxt('Recommendations generated/ml-100k/Ready_ImplicitALS_reco.csv', delimiter=',')

ev.evaluate(test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\t', header=None),
            estimations_df=estimations_df, 
            reco=reco,
            super_reactions=[4,5])
#also you can just type ev.evaluate_all(estimations_df, reco) - I put above values as default

943it [00:00, 9827.12it/s]

	RMSE	MAE	precision	recall	F_1	F_05	precision_super	recall_super	NDCG	mAP	MRR	LAUC	HR	Reco in test	Test coverage	Shannon	Gini
0	3.269156	3.070003	0.257582	0.18664	0.178445	0.202974	0.171137	0.216258	0.308415	0.175796	0.532835	0.590709	0.878049	0.999788	0.504329	5.761941	0.820874

Hyperparameters tuning

Number of latent factors

from tqdm import tqdm
result=[]
for factors in tqdm([i for i in np.arange(25,400,25)]):
    train_read=pd.read_csv('./Datasets/ml-100k/train.csv', sep='\t', header=None)
    test_read=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\t', header=None)
    train_ui, test_ui, user_code_id, user_id_code, item_code_id, item_id_code = helpers.data_to_csr(train_read, test_read)
    
    train_ui*=alpha
    train_iu=train_ui.transpose().tocsr()
    
    model = implicit.als.AlternatingLeastSquares(factors=factors, regularization=0.1, iterations=10)
    model.fit(train_iu, show_progress=False)
    
    reco=top_k_recommendations(model, user_code_id, item_code_id, topK=10)
    estimations_df=pd.DataFrame(estimate(model, user_code_id, item_code_id, test_ui))
    
    to_append=ev.evaluate(test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\t', header=None),
            estimations_df=estimations_df, 
            reco=np.array(reco),
            super_reactions=[4,5])
    to_append.insert(0, "Factors", factors)
    result.append(to_append)
    
result=pd.concat(result)
result

  0%|          | 0/15 [00:00<?, ?it/s]
943it [00:00, 10189.63it/s]
  7%|▋         | 1/15 [00:00<00:13,  1.01it/s]
943it [00:00, 10892.14it/s]
 13%|█▎        | 2/15 [00:02<00:13,  1.02s/it]
943it [00:00, 9878.59it/s]
 20%|██        | 3/15 [00:03<00:12,  1.05s/it]
0it [00:00, ?it/s]
943it [00:00, 8448.36it/s]
 27%|██▋       | 4/15 [00:04<00:11,  1.09s/it]
943it [00:00, 10309.18it/s]
 33%|███▎      | 5/15 [00:05<00:11,  1.12s/it]
943it [00:00, 9627.27it/s]
 40%|████      | 6/15 [00:06<00:10,  1.17s/it]
943it [00:00, 10064.14it/s]
 47%|████▋     | 7/15 [00:08<00:10,  1.27s/it]
0it [00:00, ?it/s]
943it [00:00, 7773.81it/s]
 53%|█████▎    | 8/15 [00:09<00:09,  1.35s/it]
943it [00:00, 10183.39it/s]
 60%|██████    | 9/15 [00:11<00:08,  1.42s/it]
943it [00:00, 10380.88it/s]
 67%|██████▋   | 10/15 [00:13<00:07,  1.49s/it]
0it [00:00, ?it/s]
943it [00:00, 8598.01it/s]
 73%|███████▎  | 11/15 [00:14<00:06,  1.60s/it]
0it [00:00, ?it/s]
943it [00:00, 8767.21it/s]
 80%|████████  | 12/15 [00:16<00:05,  1.69s/it]
943it [00:00, 10008.50it/s]
 87%|████████▋ | 13/15 [00:18<00:03,  1.77s/it]
943it [00:00, 10347.28it/s]
 93%|█████████▎| 14/15 [00:20<00:01,  1.80s/it]
943it [00:00, 10531.63it/s]
100%|██████████| 15/15 [00:22<00:00,  1.51s/it]

	Factors	RMSE	MAE	precision	recall	F_1	F_05	precision_super	recall_super	NDCG	mAP	MRR	LAUC	HR	Reco in test	Test coverage	Shannon	Gini
0	25	2.883547	2.662389	0.160445	0.151301	0.130827	0.138574	0.099785	0.173271	0.186102	0.088639	0.321376	0.572603	0.772004	0.998940	0.670996	6.307257	0.691244
0	50	2.959637	2.742050	0.185366	0.164963	0.146859	0.157846	0.115987	0.187880	0.215360	0.107503	0.367134	0.579544	0.813362	0.999788	0.612554	6.128382	0.740882
0	75	3.035941	2.822359	0.210180	0.180590	0.161700	0.175574	0.135944	0.207044	0.249073	0.128013	0.429789	0.587471	0.864263	1.000000	0.561328	5.947086	0.783018
0	100	3.101731	2.891083	0.226829	0.186581	0.169359	0.186513	0.146674	0.213931	0.276863	0.147407	0.493030	0.590562	0.888653	0.999894	0.528860	5.872001	0.802429
0	125	3.162336	2.954918	0.240297	0.182437	0.172145	0.193263	0.156438	0.211291	0.281856	0.152367	0.482915	0.588517	0.872747	0.999894	0.506494	5.805071	0.814995
0	150	3.203476	2.999182	0.243584	0.183088	0.171628	0.193841	0.158047	0.210114	0.288516	0.157567	0.496442	0.588875	0.873807	0.999576	0.499278	5.766679	0.820675
0	175	3.241546	3.039763	0.245281	0.184557	0.172602	0.194957	0.162017	0.209834	0.294425	0.163943	0.512398	0.589619	0.875928	0.999682	0.512266	5.792640	0.816326
0	200	3.264120	3.064814	0.247932	0.178100	0.170734	0.195007	0.164485	0.204995	0.294926	0.166334	0.508886	0.586400	0.858961	1.000000	0.507215	5.751094	0.822069
0	225	3.291242	3.094641	0.251326	0.179623	0.172184	0.197367	0.163948	0.207106	0.304110	0.171370	0.532284	0.587198	0.879109	0.999576	0.489899	5.747192	0.822054
0	250	3.301411	3.105478	0.259703	0.186612	0.177475	0.202945	0.172961	0.217178	0.312328	0.179115	0.534315	0.590712	0.879109	0.999682	0.491342	5.709178	0.829702
0	275	3.321122	3.127153	0.265429	0.190661	0.180937	0.207261	0.175107	0.214145	0.315759	0.182169	0.528570	0.592760	0.875928	0.999682	0.495671	5.676727	0.832700
0	300	3.328603	3.135320	0.260976	0.187770	0.179302	0.204948	0.174356	0.217609	0.312703	0.178007	0.537737	0.591298	0.876988	0.999788	0.494949	5.618692	0.838369
0	325	3.340508	3.147901	0.267444	0.189835	0.181971	0.208957	0.176180	0.220058	0.319151	0.184651	0.541390	0.592342	0.872747	0.999682	0.492063	5.650832	0.834855
0	350	3.347853	3.155924	0.273065	0.194921	0.185570	0.213002	0.182511	0.226180	0.325784	0.190838	0.543873	0.594921	0.885472	0.999894	0.484848	5.633676	0.839884
0	375	3.355182	3.164099	0.268717	0.186555	0.179748	0.207861	0.178541	0.219777	0.319529	0.187262	0.531547	0.590719	0.878049	0.999364	0.483405	5.569003	0.845486

metrics=list(result.columns[[i not in ['Factors'] for i in result.columns]])

charts_per_row=6
charts_per_column=3

fig, axes = plt.subplots(nrows=charts_per_row, ncols=charts_per_column,figsize=(18, 7*charts_per_row ))
import itertools
to_iter=[i for i in itertools.product(range(charts_per_row), range(charts_per_column))]

for i in range(len(metrics)):
    df=result[['Factors', metrics[i]]]
    df.plot(ax=axes[to_iter[i]], title=metrics[i], x=0, y=1)

Alpha

from tqdm import tqdm
result=[]
for alpha in tqdm([1, 3, 7]+[i for i in np.arange(10,200,20)]):
    train_read=pd.read_csv('./Datasets/ml-100k/train.csv', sep='\t', header=None)
    test_read=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\t', header=None)
    train_ui, test_ui, user_code_id, user_id_code, item_code_id, item_id_code = helpers.data_to_csr(train_read, test_read)
    
    train_ui*=alpha
    train_iu=train_ui.transpose().tocsr()
    
    model = implicit.als.AlternatingLeastSquares(factors=factors, regularization=0.1, iterations=10)
    model.fit(train_iu, show_progress=False)
    
    reco=top_k_recommendations(model, user_code_id, item_code_id, topK=10)
    estimations_df=pd.DataFrame(estimate(model, user_code_id, item_code_id, test_ui))
    
    to_append=ev.evaluate(test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\t', header=None),
            estimations_df=estimations_df, 
            reco=np.array(reco),
            super_reactions=[4,5])
    to_append.insert(0, "Alpha", alpha)
    result.append(to_append)
    
result=pd.concat(result)
result

  0%|          | 0/13 [00:00<?, ?it/s]
943it [00:00, 9631.70it/s]
  8%|▊         | 1/13 [00:01<00:21,  1.81s/it]
943it [00:00, 9579.52it/s]
 15%|█▌        | 2/13 [00:04<00:21,  1.96s/it]
943it [00:00, 10126.66it/s]
 23%|██▎       | 3/13 [00:06<00:19,  1.97s/it]
943it [00:00, 10327.24it/s]
 31%|███       | 4/13 [00:08<00:17,  2.00s/it]
943it [00:00, 10078.35it/s]
 38%|███▊      | 5/13 [00:10<00:16,  2.00s/it]
943it [00:00, 9576.25it/s]
 46%|████▌     | 6/13 [00:12<00:13,  1.99s/it]
943it [00:00, 9950.61it/s]
 54%|█████▍    | 7/13 [00:14<00:11,  1.96s/it]
0it [00:00, ?it/s]
943it [00:00, 9270.47it/s]
 62%|██████▏   | 8/13 [00:16<00:09,  1.96s/it]
943it [00:00, 10790.95it/s]
 69%|██████▉   | 9/13 [00:17<00:07,  1.92s/it]
943it [00:00, 9690.89it/s]
 77%|███████▋  | 10/13 [00:19<00:05,  1.90s/it]
943it [00:00, 10651.40it/s]
 85%|████████▍ | 11/13 [00:21<00:03,  1.94s/it]
943it [00:00, 10100.25it/s]
 92%|█████████▏| 12/13 [00:23<00:01,  1.94s/it]
943it [00:00, 10501.13it/s]
100%|██████████| 13/13 [00:25<00:00,  1.96s/it]

	Alpha	RMSE	MAE	precision	recall	F_1	F_05	precision_super	recall_super	NDCG	mAP	MRR	LAUC	HR	Reco in test	Test coverage	Shannon	Gini
0	1	3.667029	3.488836	0.096713	0.071467	0.067186	0.076112	0.054506	0.074237	0.118179	0.053270	0.260293	0.532465	0.568399	0.994168	0.734488	6.693136	0.541379
0	3	3.628018	3.448919	0.147826	0.103487	0.099285	0.114498	0.090021	0.107763	0.176286	0.085984	0.353319	0.548672	0.709438	0.994168	0.697691	6.626478	0.573536
0	7	3.556791	3.375833	0.218982	0.146258	0.143279	0.167046	0.142811	0.167806	0.263200	0.146602	0.481264	0.570360	0.812301	0.997667	0.628427	6.318295	0.692941
0	10	3.510641	3.328261	0.243902	0.164535	0.160648	0.187002	0.162768	0.191288	0.295153	0.169161	0.524880	0.579620	0.848356	0.997879	0.585137	6.098749	0.751148
0	30	3.355705	3.164734	0.267550	0.188253	0.180257	0.207965	0.179185	0.218573	0.319477	0.184459	0.542908	0.591560	0.888653	1.000000	0.484848	5.570362	0.845232
0	50	3.273295	3.075884	0.267444	0.190215	0.180829	0.207961	0.176288	0.218506	0.317305	0.183281	0.525159	0.592538	0.880170	1.000000	0.455267	5.520912	0.854004
0	70	3.216138	3.014393	0.250795	0.177880	0.170805	0.196090	0.162017	0.197930	0.295512	0.165824	0.506624	0.586287	0.862142	1.000000	0.453102	5.501145	0.858617
0	90	3.173423	2.969030	0.248993	0.176408	0.169005	0.194434	0.164056	0.205219	0.294456	0.166556	0.504511	0.585555	0.871686	1.000000	0.453102	5.524114	0.856353
0	110	3.142262	2.935499	0.229056	0.161752	0.153913	0.177522	0.147961	0.179891	0.264254	0.142434	0.451158	0.578115	0.846235	0.999894	0.462482	5.455059	0.863686
0	130	3.111596	2.902799	0.225769	0.159152	0.152693	0.175730	0.146674	0.181274	0.259610	0.139655	0.443082	0.576798	0.834571	1.000000	0.471140	5.525633	0.858103
0	150	3.089802	2.878570	0.222587	0.153552	0.149134	0.172670	0.141524	0.173480	0.254501	0.134119	0.446513	0.573975	0.825027	1.000000	0.448773	5.497452	0.862723
0	170	3.071050	2.859003	0.218028	0.155008	0.147994	0.170192	0.142167	0.169686	0.247942	0.131564	0.420853	0.574683	0.831389	0.999894	0.459596	5.513748	0.859285
0	190	3.054821	2.841333	0.209544	0.148378	0.142746	0.164267	0.138948	0.172691	0.242354	0.126092	0.432985	0.571345	0.815483	1.000000	0.485570	5.525070	0.855854

metrics=list(result.columns[[i not in ['Alpha'] for i in result.columns]])

charts_per_row=6
charts_per_column=3

fig, axes = plt.subplots(nrows=charts_per_row, ncols=charts_per_column,figsize=(18, 7*charts_per_row ))
import itertools
to_iter=[i for i in itertools.product(range(charts_per_row), range(charts_per_column))]

for i in range(len(metrics)):
    df=result[['Alpha', metrics[i]]]
    df.plot(ax=axes[to_iter[i]], title=metrics[i], x=0, y=1)

import evaluation_measures as ev

dir_path="Recommendations generated/ml-100k/"
super_reactions=[4,5]
test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\t', header=None)

ev.evaluate_all(test, dir_path, super_reactions)

943it [00:00, 10551.89it/s]
943it [00:00, 12056.38it/s]
943it [00:00, 11744.01it/s]
943it [00:00, 11899.93it/s]
943it [00:00, 11421.00it/s]
943it [00:00, 12041.26it/s]
943it [00:00, 10597.41it/s]
943it [00:00, 10562.60it/s]
943it [00:00, 12095.76it/s]
943it [00:00, 11264.41it/s]
943it [00:00, 11571.70it/s]
943it [00:00, 11714.27it/s]
943it [00:00, 11105.58it/s]
943it [00:00, 11365.66it/s]
943it [00:00, 11612.74it/s]

	Model	RMSE	MAE	precision	recall	F_1	F_05	precision_super	recall_super	NDCG	mAP	MRR	LAUC	HR	Reco in test	Test coverage	Shannon	Gini
0	Self_P3	3.702446	3.527273	0.282185	0.192092	0.186749	0.216980	0.204185	0.240096	0.339114	0.204905	0.572157	0.593544	0.875928	1.000000	0.077201	3.875892	0.974947
0	Ready_ImplicitALS	3.269156	3.070003	0.257582	0.186640	0.178445	0.202974	0.171137	0.216258	0.308415	0.175796	0.532835	0.590709	0.878049	0.999788	0.504329	5.761941	0.820874
0	Self_TopPop	2.508258	2.217909	0.188865	0.116919	0.118732	0.141584	0.130472	0.137473	0.214651	0.111707	0.400939	0.555546	0.765642	1.000000	0.038961	3.159079	0.987317
0	Ready_SVD	0.951652	0.750975	0.096394	0.047252	0.052870	0.067257	0.085515	0.074754	0.109578	0.051562	0.235567	0.520341	0.496288	0.995546	0.208514	4.455755	0.951624
0	Self_SVD	0.914393	0.717199	0.101697	0.042334	0.051787	0.068811	0.092489	0.072360	0.104839	0.048970	0.196117	0.517889	0.480382	0.867338	0.147186	3.852545	0.972694
0	Ready_Baseline	0.949459	0.752487	0.091410	0.037652	0.046030	0.061286	0.079614	0.056463	0.095957	0.043178	0.198193	0.515501	0.437964	1.000000	0.033911	2.836513	0.991139
0	Ready_SVDBiased	0.940413	0.739571	0.086002	0.035478	0.043196	0.057507	0.075751	0.053460	0.094897	0.043361	0.209124	0.514405	0.428420	0.997349	0.177489	4.212509	0.962656
0	Ready_Random	1.527935	1.225393	0.049311	0.020479	0.024944	0.032990	0.032189	0.024725	0.053647	0.020462	0.136036	0.506763	0.339343	0.986108	0.191198	5.101215	0.907796
0	Ready_I-KNN	1.030386	0.813067	0.026087	0.006908	0.010593	0.016046	0.021137	0.009522	0.024214	0.008958	0.048068	0.499885	0.154825	0.402333	0.434343	5.133650	0.877999
0	Ready_I-KNNBaseline	0.935327	0.737424	0.002545	0.000755	0.001105	0.001602	0.002253	0.000930	0.003444	0.001362	0.011760	0.496724	0.021209	0.482821	0.059885	2.232578	0.994487
0	Ready_U-KNN	1.023495	0.807913	0.000742	0.000205	0.000305	0.000449	0.000536	0.000198	0.000845	0.000274	0.002744	0.496441	0.007423	0.602121	0.010823	2.089186	0.995706
0	Self_BaselineIU	0.958136	0.754051	0.000954	0.000188	0.000298	0.000481	0.000644	0.000223	0.001043	0.000335	0.003348	0.496433	0.009544	0.699046	0.005051	1.945910	0.995669
0	Self_TopRated	1.030712	0.820904	0.000954	0.000188	0.000298	0.000481	0.000644	0.000223	0.001043	0.000335	0.003348	0.496433	0.009544	0.699046	0.005051	1.945910	0.995669
0	Self_BaselineUI	0.967585	0.762740	0.000954	0.000170	0.000278	0.000463	0.000644	0.000189	0.000752	0.000168	0.001677	0.496424	0.009544	0.600530	0.005051	1.803126	0.996380
0	Self_IKNN	1.018363	0.808793	0.000318	0.000108	0.000140	0.000189	0.000000	0.000000	0.000214	0.000037	0.000368	0.496391	0.003181	0.392153	0.115440	4.174741	0.965327

project task 7: Check how number of iterations of WRMF model influence the evaluation metrics

# we already checked how parameters alpha and factors influence the model performance
# your task is to do a similar thing with param iterations

# expected output is a table reporting the evaluation metrics