s416082/WSS-project
1
0
Fork 0
forked from robkw/introduction_to_recommender_systems
Code Issues Pull Requests Projects Releases Wiki Activity
c3c563e3b6
WSS-project/P2. Evaluation.ipynb

2746 lines
92 KiB
Plaintext
Raw Normal View History

first 2 lessons
2021-03-20 20:01:22 +01:00
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Prepare test set"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"slideshow": {
"slide_type": "-"
}
},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import scipy.sparse as sparse\n",
"from collections import defaultdict\n",
"from itertools import chain\n",
"import random\n",
"from tqdm import tqdm\n",
"\n",
"# In evaluation we do not load train set - it is not needed\n",
"test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None)\n",
"test.columns=['user', 'item', 'rating', 'timestamp']\n",
"\n",
"test['user_code'] = test['user'].astype(\"category\").cat.codes\n",
"test['item_code'] = test['item'].astype(\"category\").cat.codes\n",
"\n",
"user_code_id = dict(enumerate(test['user'].astype(\"category\").cat.categories))\n",
"user_id_code = dict((v, k) for k, v in user_code_id.items())\n",
"item_code_id = dict(enumerate(test['item'].astype(\"category\").cat.categories))\n",
"item_id_code = dict((v, k) for k, v in item_code_id.items())\n",
"\n",
"test_ui = sparse.csr_matrix((test['rating'], (test['user_code'], test['item_code'])))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Estimations metrics"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"estimations_df=pd.read_csv('Recommendations generated/ml-100k/Ready_Baseline_estimations.csv', header=None)\n",
"estimations_df.columns=['user', 'item' ,'score']\n",
"\n",
"estimations_df['user_code']=[user_id_code[user] for user in estimations_df['user']]\n",
"estimations_df['item_code']=[item_id_code[item] for item in estimations_df['item']]\n",
"estimations=sparse.csr_matrix((estimations_df['score'], (estimations_df['user_code'], estimations_df['item_code'])), shape=test_ui.shape)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"def estimations_metrics(test_ui, estimations):\n",
" result=[]\n",
"\n",
" RMSE=(np.sum((estimations.data-test_ui.data)**2)/estimations.nnz)**(1/2)\n",
" result.append(['RMSE', RMSE])\n",
"\n",
" MAE=np.sum(abs(estimations.data-test_ui.data))/estimations.nnz\n",
" result.append(['MAE', MAE])\n",
" \n",
" df_result=(pd.DataFrame(list(zip(*result))[1])).T\n",
" df_result.columns=list(zip(*result))[0]\n",
" return df_result"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" RMSE MAE\n",
"0 0.949459 0.752487"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# in case of error (in the laboratories) you might have to switch to the other version of pandas\n",
"# try !pip3 install pandas=='1.0.3' (or pip if you use python 2) and restart the kernel\n",
"\n",
"estimations_metrics(test_ui, estimations)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Ranking metrics"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[663, 475, 62, ..., 472, 269, 503],\n",
" [ 48, 313, 475, ..., 591, 175, 466],\n",
" [351, 313, 475, ..., 591, 175, 466],\n",
" ...,\n",
" [259, 313, 475, ..., 11, 591, 175],\n",
" [ 33, 313, 475, ..., 11, 591, 175],\n",
" [ 77, 313, 475, ..., 11, 591, 175]])"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import numpy as np\n",
"reco = np.loadtxt('Recommendations generated/ml-100k/Ready_Baseline_reco.csv', delimiter=',')\n",
"# Let's ignore scores - they are not used in evaluation: \n",
"users=reco[:,:1]\n",
"items=reco[:,1::2]\n",
"# Let's use inner ids instead of real ones\n",
"users=np.vectorize(lambda x: user_id_code.setdefault(x, -1))(users)\n",
"items=np.vectorize(lambda x: item_id_code.setdefault(x, -1))(items) # maybe items we recommend are not in test set\n",
"# Let's put them into one array\n",
"reco=np.concatenate((users, items), axis=1)\n",
"reco"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"def ranking_metrics(test_ui, reco, super_reactions=[], topK=10):\n",
" \n",
" nb_items=test_ui.shape[1]\n",
" relevant_users, super_relevant_users, prec, rec, F_1, F_05, prec_super, rec_super, ndcg, mAP, MRR, LAUC, HR=\\\n",
" 0,0,0,0,0,0,0,0,0,0,0,0,0\n",
" \n",
" cg = (1.0 / np.log2(np.arange(2, topK + 2)))\n",
" cg_sum = np.cumsum(cg)\n",
" \n",
" for (nb_user, user) in tqdm(enumerate(reco[:,0])):\n",
" u_rated_items=test_ui.indices[test_ui.indptr[user]:test_ui.indptr[user+1]]\n",
" nb_u_rated_items=len(u_rated_items)\n",
" if nb_u_rated_items>0: # skip users with no items in test set (still possible that there will be no super items)\n",
" relevant_users+=1\n",
" \n",
" u_super_items=u_rated_items[np.vectorize(lambda x: x in super_reactions)\\\n",
" (test_ui.data[test_ui.indptr[user]:test_ui.indptr[user+1]])]\n",
" # more natural seems u_super_items=[item for item in u_rated_items if test_ui[user,item] in super_reactions]\n",
" # but accesing test_ui[user,item] is expensive -we should avoid doing it\n",
" if len(u_super_items)>0:\n",
" super_relevant_users+=1\n",
" \n",
" user_successes=np.zeros(topK)\n",
" nb_user_successes=0\n",
" user_super_successes=np.zeros(topK)\n",
" nb_user_super_successes=0\n",
" \n",
" # evaluation\n",
" for (item_position,item) in enumerate(reco[nb_user,1:topK+1]):\n",
" if item in u_rated_items:\n",
" user_successes[item_position]=1\n",
" nb_user_successes+=1\n",
" if item in u_super_items:\n",
" user_super_successes[item_position]=1\n",
" nb_user_super_successes+=1\n",
" \n",
" prec_u=nb_user_successes/topK \n",
" prec+=prec_u\n",
" \n",
" rec_u=nb_user_successes/nb_u_rated_items\n",
" rec+=rec_u\n",
" \n",
" F_1+=2*(prec_u*rec_u)/(prec_u+rec_u) if prec_u+rec_u>0 else 0\n",
" F_05+=(0.5**2+1)*(prec_u*rec_u)/(0.5**2*prec_u+rec_u) if prec_u+rec_u>0 else 0\n",
" \n",
" prec_super+=nb_user_super_successes/topK\n",
" rec_super+=nb_user_super_successes/max(len(u_super_items),1) # to set 0 if no super items\n",
" ndcg+=np.dot(user_successes,cg)/cg_sum[min(topK, nb_u_rated_items)-1]\n",
" \n",
" cumsum_successes=np.cumsum(user_successes)\n",
" mAP+=np.dot(cumsum_successes/np.arange(1,topK+1), user_successes)/min(topK, nb_u_rated_items)\n",
" MRR+=1/(user_successes.nonzero()[0][0]+1) if user_successes.nonzero()[0].size>0 else 0\n",
" LAUC+=(np.dot(cumsum_successes, 1-user_successes)+\\\n",
" (nb_user_successes+nb_u_rated_items)/2*((nb_items-nb_u_rated_items)-(topK-nb_user_successes)))/\\\n",
" ((nb_items-nb_u_rated_items)*nb_u_rated_items)\n",
" \n",
" HR+=nb_user_successes>0\n",
" \n",
" \n",
" result=[]\n",
" result.append(('precision', prec/relevant_users))\n",
" result.append(('recall', rec/relevant_users))\n",
" result.append(('F_1', F_1/relevant_users))\n",
" result.append(('F_05', F_05/relevant_users))\n",
" result.append(('precision_super', prec_super/super_relevant_users))\n",
" result.append(('recall_super', rec_super/super_relevant_users))\n",
" result.append(('NDCG', ndcg/relevant_users))\n",
" result.append(('mAP', mAP/relevant_users))\n",
" result.append(('MRR', MRR/relevant_users))\n",
" result.append(('LAUC', LAUC/relevant_users))\n",
" result.append(('HR', HR/relevant_users))\n",
"\n",
" df_result=(pd.DataFrame(list(zip(*result))[1])).T\n",
" df_result.columns=list(zip(*result))[0]\n",
" return df_result"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"943it [00:00, 7647.02it/s]\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.09141</td>\n",
" <td>0.037652</td>\n",
" <td>0.04603</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" precision recall F_1 F_05 precision_super recall_super \\\n",
"0 0.09141 0.037652 0.04603 0.061286 0.079614 0.056463 \n",
"\n",
" NDCG mAP MRR LAUC HR \n",
"0 0.095957 0.043178 0.198193 0.515501 0.437964 "
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ranking_metrics(test_ui, reco, super_reactions=[4,5], topK=10)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Diversity metrics"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"def diversity_metrics(test_ui, reco, topK=10):\n",
" \n",
" frequencies=defaultdict(int)\n",
" \n",
" # let's assign 0 to all items in test set\n",
" for item in list(set(test_ui.indices)):\n",
" frequencies[item]=0\n",
" \n",
" # counting frequencies\n",
" for item in reco[:,1:].flat:\n",
" frequencies[item]+=1\n",
" \n",
" nb_reco_outside_test=frequencies[-1]\n",
" del frequencies[-1]\n",
" \n",
" frequencies=np.array(list(frequencies.values()))\n",
" \n",
" nb_rec_items=len(frequencies[frequencies>0])\n",
" nb_reco_inside_test=np.sum(frequencies)\n",
" \n",
" frequencies=frequencies/np.sum(frequencies)\n",
" frequencies=np.sort(frequencies)\n",
" \n",
" with np.errstate(divide='ignore'): # let's put zeros put items with 0 frequency and ignore division warning\n",
" log_frequencies=np.nan_to_num(np.log(frequencies), posinf=0, neginf=0)\n",
" \n",
" result=[]\n",
" result.append(('Reco in test', nb_reco_inside_test/(nb_reco_inside_test+nb_reco_outside_test)))\n",
" result.append(('Test coverage', nb_rec_items/test_ui.shape[1]))\n",
" result.append(('Shannon', -np.dot(frequencies, log_frequencies)))\n",
" result.append(('Gini', np.dot(frequencies, np.arange(1-len(frequencies), len(frequencies), 2))/(len(frequencies)-1)))\n",
" \n",
" df_result=(pd.DataFrame(list(zip(*result))[1])).T\n",
" df_result.columns=list(zip(*result))[0]\n",
" return df_result"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1.0</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Reco in test Test coverage Shannon Gini\n",
"0 1.0 0.033911 2.836513 0.991139"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# in case of errors try !pip3 install numpy==1.18.4 (or pip if you use python 2) and restart the kernel\n",
"\n",
"import evaluation_measures as ev\n",
"import imp\n",
"imp.reload(ev)\n",
"\n",
"x=diversity_metrics(test_ui, reco, topK=10)\n",
"x"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# To be used in other notebooks"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"943it [00:00, 7829.39it/s]\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" <td>0.09141</td>\n",
" <td>0.037652</td>\n",
" <td>0.04603</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
" <td>1.0</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" RMSE MAE precision recall F_1 F_05 \\\n",
"0 0.949459 0.752487 0.09141 0.037652 0.04603 0.061286 \n",
"\n",
" precision_super recall_super NDCG mAP MRR LAUC \\\n",
"0 0.079614 0.056463 0.095957 0.043178 0.198193 0.515501 \n",
"\n",
" HR Reco in test Test coverage Shannon Gini \n",
"0 0.437964 1.0 0.033911 2.836513 0.991139 "
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import evaluation_measures as ev\n",
"import imp\n",
"imp.reload(ev)\n",
"\n",
"estimations_df=pd.read_csv('Recommendations generated/ml-100k/Ready_Baseline_estimations.csv', header=None)\n",
"reco=np.loadtxt('Recommendations generated/ml-100k/Ready_Baseline_reco.csv', delimiter=',')\n",
"\n",
"ev.evaluate(test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None),\n",
" estimations_df=estimations_df, \n",
" reco=reco,\n",
" super_reactions=[4,5])\n",
"#also you can just type ev.evaluate_all(estimations_df, reco) - I put above values as default"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"943it [00:00, 7954.38it/s]\n",
"943it [00:00, 4698.62it/s]\n",
"943it [00:00, 5104.10it/s]\n",
"943it [00:00, 4853.27it/s]\n",
"943it [00:00, 4669.78it/s]\n",
"943it [00:00, 4207.34it/s]\n",
"943it [00:00, 5248.26it/s]\n",
"943it [00:00, 4477.59it/s]\n",
"943it [00:00, 4280.31it/s]\n",
"943it [00:00, 3915.20it/s]\n",
"943it [00:00, 4648.51it/s]\n",
"943it [00:00, 3819.45it/s]\n",
"943it [00:00, 4405.24it/s]\n",
"943it [00:00, 4725.10it/s]\n",
"943it [00:00, 4426.18it/s]\n",
"943it [00:00, 4179.78it/s]\n",
"943it [00:00, 4919.92it/s]\n"
]
}
],
"source": [
"import evaluation_measures as ev\n",
"import imp\n",
"imp.reload(ev)\n",
"\n",
"dir_path=\"Recommendations generated/ml-100k/\"\n",
"super_reactions=[4,5]\n",
"test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None)\n",
"\n",
"df=ev.evaluate_all(test, dir_path, super_reactions)\n",
"#also you can just type ev.evaluate_all() - I put above values as default"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_LightFM</td>\n",
" <td>162.703697</td>\n",
" <td>160.837311</td>\n",
" <td>0.349523</td>\n",
" <td>0.226193</td>\n",
" <td>0.225202</td>\n",
" <td>0.265538</td>\n",
" <td>0.246459</td>\n",
" <td>0.266934</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_LightFMpureMF</td>\n",
" <td>8.015665</td>\n",
" <td>7.520402</td>\n",
" <td>0.333934</td>\n",
" <td>0.216047</td>\n",
" <td>0.214731</td>\n",
" <td>0.253177</td>\n",
" <td>0.232725</td>\n",
" <td>0.254485</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_P3</td>\n",
" <td>3.702446</td>\n",
" <td>3.527273</td>\n",
" <td>0.282185</td>\n",
" <td>0.192092</td>\n",
" <td>0.186749</td>\n",
" <td>0.216980</td>\n",
" <td>0.204185</td>\n",
" <td>0.240096</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_ImplicitALS</td>\n",
" <td>3.267237</td>\n",
" <td>3.068493</td>\n",
" <td>0.252068</td>\n",
" <td>0.182639</td>\n",
" <td>0.175182</td>\n",
" <td>0.199457</td>\n",
" <td>0.167167</td>\n",
" <td>0.216308</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_TopPop</td>\n",
" <td>2.508258</td>\n",
" <td>2.217909</td>\n",
" <td>0.188865</td>\n",
" <td>0.116919</td>\n",
" <td>0.118732</td>\n",
" <td>0.141584</td>\n",
" <td>0.130472</td>\n",
" <td>0.137473</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_LightFMcontent</td>\n",
" <td>182.840876</td>\n",
" <td>180.771141</td>\n",
" <td>0.161294</td>\n",
" <td>0.100424</td>\n",
" <td>0.101736</td>\n",
" <td>0.121096</td>\n",
" <td>0.101395</td>\n",
" <td>0.110660</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_SVD</td>\n",
" <td>0.953076</td>\n",
" <td>0.750219</td>\n",
" <td>0.094804</td>\n",
" <td>0.045302</td>\n",
" <td>0.051519</td>\n",
" <td>0.065833</td>\n",
" <td>0.083691</td>\n",
" <td>0.074336</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_SVD</td>\n",
" <td>0.913840</td>\n",
" <td>0.717167</td>\n",
" <td>0.105620</td>\n",
" <td>0.044070</td>\n",
" <td>0.053839</td>\n",
" <td>0.071381</td>\n",
" <td>0.096030</td>\n",
" <td>0.074982</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Baseline</td>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" <td>0.091410</td>\n",
" <td>0.037652</td>\n",
" <td>0.046030</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_SVDBiased</td>\n",
" <td>0.941830</td>\n",
" <td>0.742841</td>\n",
" <td>0.083033</td>\n",
" <td>0.034867</td>\n",
" <td>0.041967</td>\n",
" <td>0.055644</td>\n",
" <td>0.072425</td>\n",
" <td>0.054271</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_GlobalAvg</td>\n",
" <td>1.125760</td>\n",
" <td>0.943534</td>\n",
" <td>0.061188</td>\n",
" <td>0.025968</td>\n",
" <td>0.031383</td>\n",
" <td>0.041343</td>\n",
" <td>0.040558</td>\n",
" <td>0.032107</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Random</td>\n",
" <td>1.513348</td>\n",
" <td>1.214309</td>\n",
" <td>0.044221</td>\n",
" <td>0.019366</td>\n",
" <td>0.022599</td>\n",
" <td>0.029593</td>\n",
" <td>0.026288</td>\n",
" <td>0.018226</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_I-KNN</td>\n",
" <td>1.030386</td>\n",
" <td>0.813067</td>\n",
" <td>0.026087</td>\n",
" <td>0.006908</td>\n",
" <td>0.010593</td>\n",
" <td>0.016046</td>\n",
" <td>0.021137</td>\n",
" <td>0.009522</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_I-KNNBaseline</td>\n",
" <td>0.935327</td>\n",
" <td>0.737424</td>\n",
" <td>0.002545</td>\n",
" <td>0.000755</td>\n",
" <td>0.001105</td>\n",
" <td>0.001602</td>\n",
" <td>0.002253</td>\n",
" <td>0.000930</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_U-KNN</td>\n",
" <td>1.023495</td>\n",
" <td>0.807913</td>\n",
" <td>0.000742</td>\n",
" <td>0.000205</td>\n",
" <td>0.000305</td>\n",
" <td>0.000449</td>\n",
" <td>0.000536</td>\n",
" <td>0.000198</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineUI</td>\n",
" <td>0.967585</td>\n",
" <td>0.762740</td>\n",
" <td>0.000954</td>\n",
" <td>0.000170</td>\n",
" <td>0.000278</td>\n",
" <td>0.000463</td>\n",
" <td>0.000644</td>\n",
" <td>0.000189</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_IKNN</td>\n",
" <td>1.018363</td>\n",
" <td>0.808793</td>\n",
" <td>0.000318</td>\n",
" <td>0.000108</td>\n",
" <td>0.000140</td>\n",
" <td>0.000189</td>\n",
" <td>0.000000</td>\n",
" <td>0.000000</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Model RMSE MAE precision recall \\\n",
"0 Ready_LightFM 162.703697 160.837311 0.349523 0.226193 \n",
"0 Ready_LightFMpureMF 8.015665 7.520402 0.333934 0.216047 \n",
"0 Self_P3 3.702446 3.527273 0.282185 0.192092 \n",
"0 Ready_ImplicitALS 3.267237 3.068493 0.252068 0.182639 \n",
"0 Self_TopPop 2.508258 2.217909 0.188865 0.116919 \n",
"0 Ready_LightFMcontent 182.840876 180.771141 0.161294 0.100424 \n",
"0 Ready_SVD 0.953076 0.750219 0.094804 0.045302 \n",
"0 Self_SVD 0.913840 0.717167 0.105620 0.044070 \n",
"0 Ready_Baseline 0.949459 0.752487 0.091410 0.037652 \n",
"0 Ready_SVDBiased 0.941830 0.742841 0.083033 0.034867 \n",
"0 Self_GlobalAvg 1.125760 0.943534 0.061188 0.025968 \n",
"0 Ready_Random 1.513348 1.214309 0.044221 0.019366 \n",
"0 Ready_I-KNN 1.030386 0.813067 0.026087 0.006908 \n",
"0 Ready_I-KNNBaseline 0.935327 0.737424 0.002545 0.000755 \n",
"0 Ready_U-KNN 1.023495 0.807913 0.000742 0.000205 \n",
"0 Self_BaselineUI 0.967585 0.762740 0.000954 0.000170 \n",
"0 Self_IKNN 1.018363 0.808793 0.000318 0.000108 \n",
"\n",
" F_1 F_05 precision_super recall_super \n",
"0 0.225202 0.265538 0.246459 0.266934 \n",
"0 0.214731 0.253177 0.232725 0.254485 \n",
"0 0.186749 0.216980 0.204185 0.240096 \n",
"0 0.175182 0.199457 0.167167 0.216308 \n",
"0 0.118732 0.141584 0.130472 0.137473 \n",
"0 0.101736 0.121096 0.101395 0.110660 \n",
"0 0.051519 0.065833 0.083691 0.074336 \n",
"0 0.053839 0.071381 0.096030 0.074982 \n",
"0 0.046030 0.061286 0.079614 0.056463 \n",
"0 0.041967 0.055644 0.072425 0.054271 \n",
"0 0.031383 0.041343 0.040558 0.032107 \n",
"0 0.022599 0.029593 0.026288 0.018226 \n",
"0 0.010593 0.016046 0.021137 0.009522 \n",
"0 0.001105 0.001602 0.002253 0.000930 \n",
"0 0.000305 0.000449 0.000536 0.000198 \n",
"0 0.000278 0.000463 0.000644 0.000189 \n",
"0 0.000140 0.000189 0.000000 0.000000 "
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.iloc[:,:9]"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_LightFM</td>\n",
" <td>0.413969</td>\n",
" <td>0.277036</td>\n",
" <td>0.648029</td>\n",
" <td>0.610845</td>\n",
" <td>0.916225</td>\n",
" <td>1.000000</td>\n",
" <td>0.352814</td>\n",
" <td>5.363070</td>\n",
" <td>0.885116</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_LightFMpureMF</td>\n",
" <td>0.391316</td>\n",
" <td>0.257793</td>\n",
" <td>0.606204</td>\n",
" <td>0.605708</td>\n",
" <td>0.906681</td>\n",
" <td>1.000000</td>\n",
" <td>0.272006</td>\n",
" <td>5.031437</td>\n",
" <td>0.918177</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_P3</td>\n",
" <td>0.339114</td>\n",
" <td>0.204905</td>\n",
" <td>0.572157</td>\n",
" <td>0.593544</td>\n",
" <td>0.875928</td>\n",
" <td>1.000000</td>\n",
" <td>0.077201</td>\n",
" <td>3.875892</td>\n",
" <td>0.974947</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_ImplicitALS</td>\n",
" <td>0.295331</td>\n",
" <td>0.163847</td>\n",
" <td>0.500282</td>\n",
" <td>0.588672</td>\n",
" <td>0.873807</td>\n",
" <td>0.999894</td>\n",
" <td>0.497835</td>\n",
" <td>5.727745</td>\n",
" <td>0.825683</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_TopPop</td>\n",
" <td>0.214651</td>\n",
" <td>0.111707</td>\n",
" <td>0.400939</td>\n",
" <td>0.555546</td>\n",
" <td>0.765642</td>\n",
" <td>1.000000</td>\n",
" <td>0.038961</td>\n",
" <td>3.159079</td>\n",
" <td>0.987317</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_LightFMcontent</td>\n",
" <td>0.184311</td>\n",
" <td>0.091346</td>\n",
" <td>0.352019</td>\n",
" <td>0.547187</td>\n",
" <td>0.705196</td>\n",
" <td>0.979533</td>\n",
" <td>0.269120</td>\n",
" <td>4.940084</td>\n",
" <td>0.924146</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_SVD</td>\n",
" <td>0.107620</td>\n",
" <td>0.051155</td>\n",
" <td>0.234251</td>\n",
" <td>0.519361</td>\n",
" <td>0.490986</td>\n",
" <td>0.993425</td>\n",
" <td>0.206349</td>\n",
" <td>4.406898</td>\n",
" <td>0.953781</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_SVD</td>\n",
" <td>0.109138</td>\n",
" <td>0.051857</td>\n",
" <td>0.202054</td>\n",
" <td>0.518772</td>\n",
" <td>0.478261</td>\n",
" <td>0.872959</td>\n",
" <td>0.144300</td>\n",
" <td>3.912577</td>\n",
" <td>0.971609</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Baseline</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
" <td>1.000000</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_SVDBiased</td>\n",
" <td>0.090974</td>\n",
" <td>0.041243</td>\n",
" <td>0.195741</td>\n",
" <td>0.514084</td>\n",
" <td>0.418876</td>\n",
" <td>0.998409</td>\n",
" <td>0.168831</td>\n",
" <td>4.152102</td>\n",
" <td>0.964603</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_GlobalAvg</td>\n",
" <td>0.067695</td>\n",
" <td>0.027470</td>\n",
" <td>0.171187</td>\n",
" <td>0.509546</td>\n",
" <td>0.384942</td>\n",
" <td>1.000000</td>\n",
" <td>0.025974</td>\n",
" <td>2.711772</td>\n",
" <td>0.992003</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Random</td>\n",
" <td>0.047273</td>\n",
" <td>0.017729</td>\n",
" <td>0.114687</td>\n",
" <td>0.506181</td>\n",
" <td>0.301166</td>\n",
" <td>0.986002</td>\n",
" <td>0.184704</td>\n",
" <td>5.093324</td>\n",
" <td>0.907405</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_I-KNN</td>\n",
" <td>0.024214</td>\n",
" <td>0.008958</td>\n",
" <td>0.048068</td>\n",
" <td>0.499885</td>\n",
" <td>0.154825</td>\n",
" <td>0.402333</td>\n",
" <td>0.434343</td>\n",
" <td>5.133650</td>\n",
" <td>0.877999</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_I-KNNBaseline</td>\n",
" <td>0.003444</td>\n",
" <td>0.001362</td>\n",
" <td>0.011760</td>\n",
" <td>0.496724</td>\n",
" <td>0.021209</td>\n",
" <td>0.482821</td>\n",
" <td>0.059885</td>\n",
" <td>2.232578</td>\n",
" <td>0.994487</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_U-KNN</td>\n",
" <td>0.000845</td>\n",
" <td>0.000274</td>\n",
" <td>0.002744</td>\n",
" <td>0.496441</td>\n",
" <td>0.007423</td>\n",
" <td>0.602121</td>\n",
" <td>0.010823</td>\n",
" <td>2.089186</td>\n",
" <td>0.995706</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineUI</td>\n",
" <td>0.000752</td>\n",
" <td>0.000168</td>\n",
" <td>0.001677</td>\n",
" <td>0.496424</td>\n",
" <td>0.009544</td>\n",
" <td>0.600530</td>\n",
" <td>0.005051</td>\n",
" <td>1.803126</td>\n",
" <td>0.996380</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_IKNN</td>\n",
" <td>0.000214</td>\n",
" <td>0.000037</td>\n",
" <td>0.000368</td>\n",
" <td>0.496391</td>\n",
" <td>0.003181</td>\n",
" <td>0.392153</td>\n",
" <td>0.115440</td>\n",
" <td>4.174741</td>\n",
" <td>0.965327</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Model NDCG mAP MRR LAUC HR \\\n",
"0 Ready_LightFM 0.413969 0.277036 0.648029 0.610845 0.916225 \n",
"0 Ready_LightFMpureMF 0.391316 0.257793 0.606204 0.605708 0.906681 \n",
"0 Self_P3 0.339114 0.204905 0.572157 0.593544 0.875928 \n",
"0 Ready_ImplicitALS 0.295331 0.163847 0.500282 0.588672 0.873807 \n",
"0 Self_TopPop 0.214651 0.111707 0.400939 0.555546 0.765642 \n",
"0 Ready_LightFMcontent 0.184311 0.091346 0.352019 0.547187 0.705196 \n",
"0 Ready_SVD 0.107620 0.051155 0.234251 0.519361 0.490986 \n",
"0 Self_SVD 0.109138 0.051857 0.202054 0.518772 0.478261 \n",
"0 Ready_Baseline 0.095957 0.043178 0.198193 0.515501 0.437964 \n",
"0 Ready_SVDBiased 0.090974 0.041243 0.195741 0.514084 0.418876 \n",
"0 Self_GlobalAvg 0.067695 0.027470 0.171187 0.509546 0.384942 \n",
"0 Ready_Random 0.047273 0.017729 0.114687 0.506181 0.301166 \n",
"0 Ready_I-KNN 0.024214 0.008958 0.048068 0.499885 0.154825 \n",
"0 Ready_I-KNNBaseline 0.003444 0.001362 0.011760 0.496724 0.021209 \n",
"0 Ready_U-KNN 0.000845 0.000274 0.002744 0.496441 0.007423 \n",
"0 Self_BaselineUI 0.000752 0.000168 0.001677 0.496424 0.009544 \n",
"0 Self_IKNN 0.000214 0.000037 0.000368 0.496391 0.003181 \n",
"\n",
" Reco in test Test coverage Shannon Gini \n",
"0 1.000000 0.352814 5.363070 0.885116 \n",
"0 1.000000 0.272006 5.031437 0.918177 \n",
"0 1.000000 0.077201 3.875892 0.974947 \n",
"0 0.999894 0.497835 5.727745 0.825683 \n",
"0 1.000000 0.038961 3.159079 0.987317 \n",
"0 0.979533 0.269120 4.940084 0.924146 \n",
"0 0.993425 0.206349 4.406898 0.953781 \n",
"0 0.872959 0.144300 3.912577 0.971609 \n",
"0 1.000000 0.033911 2.836513 0.991139 \n",
"0 0.998409 0.168831 4.152102 0.964603 \n",
"0 1.000000 0.025974 2.711772 0.992003 \n",
"0 0.986002 0.184704 5.093324 0.907405 \n",
"0 0.402333 0.434343 5.133650 0.877999 \n",
"0 0.482821 0.059885 2.232578 0.994487 \n",
"0 0.602121 0.010823 2.089186 0.995706 \n",
"0 0.600530 0.005051 1.803126 0.996380 \n",
"0 0.392153 0.115440 4.174741 0.965327 "
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.iloc[:,np.append(0,np.arange(9, df.shape[1]))]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Check metrics on toy dataset"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"3it [00:00, 4233.82it/s]\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineUI</td>\n",
" <td>1.612452</td>\n",
" <td>1.4</td>\n",
" <td>0.444444</td>\n",
" <td>0.888889</td>\n",
" <td>0.555556</td>\n",
" <td>0.478632</td>\n",
" <td>0.333333</td>\n",
" <td>0.75</td>\n",
" <td>0.676907</td>\n",
" <td>0.574074</td>\n",
" <td>0.611111</td>\n",
" <td>0.638889</td>\n",
" <td>1.0</td>\n",
" <td>0.888889</td>\n",
" <td>0.8</td>\n",
" <td>1.386294</td>\n",
" <td>0.25</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Model RMSE MAE precision recall F_1 F_05 \\\n",
"0 Self_BaselineUI 1.612452 1.4 0.444444 0.888889 0.555556 0.478632 \n",
"\n",
" precision_super recall_super NDCG mAP MRR LAUC HR \\\n",
"0 0.333333 0.75 0.676907 0.574074 0.611111 0.638889 1.0 \n",
"\n",
" Reco in test Test coverage Shannon Gini \n",
"0 0.888889 0.8 1.386294 0.25 "
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Training data:\n"
]
},
{
"data": {
"text/plain": [
"matrix([[3, 4, 0, 0, 5, 0, 0, 4],\n",
" [0, 1, 2, 3, 0, 0, 0, 0],\n",
" [0, 0, 0, 5, 0, 3, 4, 0]])"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Test data:\n"
]
},
{
"data": {
"text/plain": [
"matrix([[0, 0, 0, 0, 0, 0, 3, 0],\n",
" [0, 0, 0, 0, 5, 0, 0, 0],\n",
" [5, 0, 4, 0, 0, 0, 0, 2]])"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Recommendations:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>0</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" <th>3</th>\n",
" <th>4</th>\n",
" <th>5</th>\n",
" <th>6</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0</td>\n",
" <td>30</td>\n",
" <td>5.0</td>\n",
" <td>20</td>\n",
" <td>4.0</td>\n",
" <td>60</td>\n",
" <td>4.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>10</td>\n",
" <td>40</td>\n",
" <td>3.0</td>\n",
" <td>60</td>\n",
" <td>2.0</td>\n",
" <td>70</td>\n",
" <td>2.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>20</td>\n",
" <td>40</td>\n",
" <td>5.0</td>\n",
" <td>20</td>\n",
" <td>4.0</td>\n",
" <td>70</td>\n",
" <td>4.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" 0 1 2 3 4 5 6\n",
"0 0 30 5.0 20 4.0 60 4.0\n",
"1 10 40 3.0 60 2.0 70 2.0\n",
"2 20 40 5.0 20 4.0 70 4.0"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Estimations:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>user</th>\n",
" <th>item</th>\n",
" <th>est_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0</td>\n",
" <td>60</td>\n",
" <td>4.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>10</td>\n",
" <td>40</td>\n",
" <td>3.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>20</td>\n",
" <td>0</td>\n",
" <td>3.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>20</td>\n",
" <td>20</td>\n",
" <td>4.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>20</td>\n",
" <td>70</td>\n",
" <td>4.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" user item est_score\n",
"0 0 60 4.0\n",
"1 10 40 3.0\n",
"2 20 0 3.0\n",
"3 20 20 4.0\n",
"4 20 70 4.0"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import evaluation_measures as ev\n",
"import imp\n",
"import helpers\n",
"imp.reload(ev)\n",
"\n",
"dir_path=\"Recommendations generated/toy-example/\"\n",
"super_reactions=[4,5]\n",
"test=pd.read_csv('./Datasets/toy-example/test.csv', sep='\\t', header=None)\n",
"\n",
"display(ev.evaluate_all(test, dir_path, super_reactions, topK=3))\n",
"#also you can just type ev.evaluate_all() - I put above values as default\n",
"\n",
"toy_train_read=pd.read_csv('./Datasets/toy-example/train.csv', sep='\\t', header=None, names=['user', 'item', 'rating', 'timestamp'])\n",
"toy_test_read=pd.read_csv('./Datasets/toy-example/test.csv', sep='\\t', header=None, names=['user', 'item', 'rating', 'timestamp'])\n",
"reco=pd.read_csv('Recommendations generated/toy-example/Self_BaselineUI_reco.csv', header=None)\n",
"estimations=pd.read_csv('Recommendations generated/toy-example/Self_BaselineUI_estimations.csv', names=['user', 'item', 'est_score'])\n",
"toy_train_ui, toy_test_ui, toy_user_code_id, toy_user_id_code, \\\n",
"toy_item_code_id, toy_item_id_code = helpers.data_to_csr(toy_train_read, toy_test_read)\n",
"\n",
"print('Training data:')\n",
"display(toy_train_ui.todense())\n",
"\n",
"print('Test data:')\n",
"display(toy_test_ui.todense())\n",
"\n",
"print('Recommendations:')\n",
"display(reco)\n",
"\n",
"print('Estimations:')\n",
"display(estimations)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# A/B testing"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"# Suppose we had\n",
"A_successes=1000\n",
"A_failures=9000\n",
"\n",
"B_successes=1500\n",
"B_failures=12000"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Confidence intervals"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>successes</th>\n",
" <th>failures</th>\n",
" <th>conversion</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>A</th>\n",
" <td>1000</td>\n",
" <td>1500</td>\n",
" <td>0.4000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>B</th>\n",
" <td>9000</td>\n",
" <td>12000</td>\n",
" <td>0.4286</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" successes failures conversion\n",
"A 1000 1500 0.4000\n",
"B 9000 12000 0.4286"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df=pd.DataFrame({'successes': [A_successes, A_failures],'failures': [B_successes,B_failures]}, index=['A','B'])\n",
"df['conversion']=df.apply(lambda x: round(x['successes']/(x['successes']+x['failures']),4), axis=1)\n",
"df"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>successes</th>\n",
" <th>failures</th>\n",
" <th>conversion</th>\n",
" <th>conf_interval</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>A</th>\n",
" <td>1000</td>\n",
" <td>1500</td>\n",
" <td>0.4000</td>\n",
" <td>[0.3808, 0.4194]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>B</th>\n",
" <td>9000</td>\n",
" <td>12000</td>\n",
" <td>0.4286</td>\n",
" <td>[0.4219, 0.4353]</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" successes failures conversion conf_interval\n",
"A 1000 1500 0.4000 [0.3808, 0.4194]\n",
"B 9000 12000 0.4286 [0.4219, 0.4353]"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"significance=0.95\n",
"\n",
"from statsmodels.stats.proportion import proportion_confint\n",
"df['conf_interval']=df.apply(lambda x: [round(i,4) for i in proportion_confint(count=x['successes'], nobs=x['successes']+x['failures'], alpha=1-significance, method='binom_test')], axis=1)\n",
"df"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"p-value: 0.006729080907452261\n"
]
}
],
"source": [
"from scipy.stats import chi2_contingency\n",
"cond = np.array([[A_successes, A_failures], [B_successes, B_failures]])\n",
"print(f'p-value: {chi2_contingency(cond)[1]}')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### How many observations do we need? Power analysis "
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Effect size: 0.02041241452319317\n",
"Samples needed: 18837\n"
]
}
],
"source": [
"# sample size calculator: https://www.evanmiller.org/ab-testing/sample-size.html \n",
"# for now let's assume conversion from control group is known\n",
"\n",
"from statsmodels.stats.power import GofChisquarePower\n",
"from statsmodels.stats.gof import chisquare_effectsize\n",
"\n",
"effect_size=chisquare_effectsize([df['conversion']['A'], 1-df['conversion']['A']], \n",
" [df['conversion']['A']+0.01, 1-df['conversion']['A']-0.01])\n",
"print(f'Effect size: {effect_size}')\n",
"print(f'Samples needed: {round(GofChisquarePower().solve_power(effect_size, power=.8, n_bins=2, alpha=0.05))}')"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Effect size: 0.07001400420140048\n",
"Samples needed: 1601\n"
]
}
],
"source": [
"# for now let's assume conversion from control group is known\n",
"# it's not correct looking at https://www.evanmiller.org/ab-testing/sample-size.html\n",
"from statsmodels.stats.power import GofChisquarePower\n",
"from statsmodels.stats.gof import chisquare_effectsize\n",
"n_levels_variable_a = 1 # to verify\n",
"n_levels_variable_b = 2\n",
"\n",
"effect_size=chisquare_effectsize([0.15, 0.85], [0.125,0.875])\n",
"print(f'Effect size: {effect_size}')\n",
"print(f'Samples needed: {round(GofChisquarePower().solve_power(effect_size, power=.8, n_bins=(n_levels_variable_a)*(n_levels_variable_b), alpha=0.05))}')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Sample recommendations"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Here is what user rated high:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>user</th>\n",
" <th>rating</th>\n",
" <th>title</th>\n",
" <th>genres</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>41281</th>\n",
" <td>437</td>\n",
" <td>5</td>\n",
" <td>Gone with the Wind (1939)</td>\n",
" <td>Drama, Romance, War</td>\n",
" </tr>\n",
" <tr>\n",
" <th>28880</th>\n",
" <td>437</td>\n",
" <td>5</td>\n",
" <td>Pinocchio (1940)</td>\n",
" <td>Animation, Children's</td>\n",
" </tr>\n",
" <tr>\n",
" <th>36888</th>\n",
" <td>437</td>\n",
" <td>5</td>\n",
" <td>Backbeat (1993)</td>\n",
" <td>Drama, Musical</td>\n",
" </tr>\n",
" <tr>\n",
" <th>36713</th>\n",
" <td>437</td>\n",
" <td>5</td>\n",
" <td>Lone Star (1996)</td>\n",
" <td>Drama, Mystery</td>\n",
" </tr>\n",
" <tr>\n",
" <th>36122</th>\n",
" <td>437</td>\n",
" <td>5</td>\n",
" <td>Silence of the Lambs, The (1991)</td>\n",
" <td>Drama, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
" <th>32783</th>\n",
" <td>437</td>\n",
" <td>5</td>\n",
" <td>Muriel's Wedding (1994)</td>\n",
" <td>Comedy, Romance</td>\n",
" </tr>\n",
" <tr>\n",
" <th>30950</th>\n",
" <td>437</td>\n",
" <td>5</td>\n",
" <td>Rosewood (1997)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
" <th>30386</th>\n",
" <td>437</td>\n",
" <td>5</td>\n",
" <td>Manchurian Candidate, The (1962)</td>\n",
" <td>Film-Noir, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
" <th>29411</th>\n",
" <td>437</td>\n",
" <td>5</td>\n",
" <td>Psycho (1960)</td>\n",
" <td>Horror, Romance, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
" <th>27655</th>\n",
" <td>437</td>\n",
" <td>5</td>\n",
" <td>Vertigo (1958)</td>\n",
" <td>Mystery, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14735</th>\n",
" <td>437</td>\n",
" <td>5</td>\n",
" <td>Raising Arizona (1987)</td>\n",
" <td>Comedy</td>\n",
" </tr>\n",
" <tr>\n",
" <th>27563</th>\n",
" <td>437</td>\n",
" <td>5</td>\n",
" <td>Young Frankenstein (1974)</td>\n",
" <td>Comedy, Horror</td>\n",
" </tr>\n",
" <tr>\n",
" <th>26524</th>\n",
" <td>437</td>\n",
" <td>5</td>\n",
" <td>Everyone Says I Love You (1996)</td>\n",
" <td>Comedy, Musical, Romance</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25618</th>\n",
" <td>437</td>\n",
" <td>5</td>\n",
" <td>Citizen Kane (1941)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
" <th>23714</th>\n",
" <td>437</td>\n",
" <td>5</td>\n",
" <td>Casablanca (1942)</td>\n",
" <td>Drama, Romance, War</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" user rating title \\\n",
"41281 437 5 Gone with the Wind (1939) \n",
"28880 437 5 Pinocchio (1940) \n",
"36888 437 5 Backbeat (1993) \n",
"36713 437 5 Lone Star (1996) \n",
"36122 437 5 Silence of the Lambs, The (1991) \n",
"32783 437 5 Muriel's Wedding (1994) \n",
"30950 437 5 Rosewood (1997) \n",
"30386 437 5 Manchurian Candidate, The (1962) \n",
"29411 437 5 Psycho (1960) \n",
"27655 437 5 Vertigo (1958) \n",
"14735 437 5 Raising Arizona (1987) \n",
"27563 437 5 Young Frankenstein (1974) \n",
"26524 437 5 Everyone Says I Love You (1996) \n",
"25618 437 5 Citizen Kane (1941) \n",
"23714 437 5 Casablanca (1942) \n",
"\n",
" genres \n",
"41281 Drama, Romance, War \n",
"28880 Animation, Children's \n",
"36888 Drama, Musical \n",
"36713 Drama, Mystery \n",
"36122 Drama, Thriller \n",
"32783 Comedy, Romance \n",
"30950 Drama \n",
"30386 Film-Noir, Thriller \n",
"29411 Horror, Romance, Thriller \n",
"27655 Mystery, Thriller \n",
"14735 Comedy \n",
"27563 Comedy, Horror \n",
"26524 Comedy, Musical, Romance \n",
"25618 Drama \n",
"23714 Drama, Romance, War "
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Here is what we recommend:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>user</th>\n",
" <th>rec_nb</th>\n",
" <th>title</th>\n",
" <th>genres</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>435</th>\n",
" <td>437.0</td>\n",
" <td>1</td>\n",
" <td>Great Day in Harlem, A (1994)</td>\n",
" <td>Documentary</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1377</th>\n",
" <td>437.0</td>\n",
" <td>2</td>\n",
" <td>Tough and Deadly (1995)</td>\n",
" <td>Action, Drama, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2319</th>\n",
" <td>437.0</td>\n",
" <td>3</td>\n",
" <td>Aiqing wansui (1994)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3261</th>\n",
" <td>437.0</td>\n",
" <td>4</td>\n",
" <td>Delta of Venus (1994)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5145</th>\n",
" <td>437.0</td>\n",
" <td>5</td>\n",
" <td>Saint of Fort Washington, The (1993)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6087</th>\n",
" <td>437.0</td>\n",
" <td>6</td>\n",
" <td>Celestial Clockwork (1994)</td>\n",
" <td>Comedy</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7030</th>\n",
" <td>437.0</td>\n",
" <td>7</td>\n",
" <td>Some Mother's Son (1996)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8924</th>\n",
" <td>437.0</td>\n",
" <td>8</td>\n",
" <td>Maya Lin: A Strong Clear Vision (1994)</td>\n",
" <td>Documentary</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7970</th>\n",
" <td>437.0</td>\n",
" <td>9</td>\n",
" <td>Prefontaine (1997)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8485</th>\n",
" <td>437.0</td>\n",
" <td>10</td>\n",
" <td>Santa with Muscles (1996)</td>\n",
" <td>Comedy</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" user rec_nb title \\\n",
"435 437.0 1 Great Day in Harlem, A (1994) \n",
"1377 437.0 2 Tough and Deadly (1995) \n",
"2319 437.0 3 Aiqing wansui (1994) \n",
"3261 437.0 4 Delta of Venus (1994) \n",
"5145 437.0 5 Saint of Fort Washington, The (1993) \n",
"6087 437.0 6 Celestial Clockwork (1994) \n",
"7030 437.0 7 Some Mother's Son (1996) \n",
"8924 437.0 8 Maya Lin: A Strong Clear Vision (1994) \n",
"7970 437.0 9 Prefontaine (1997) \n",
"8485 437.0 10 Santa with Muscles (1996) \n",
"\n",
" genres \n",
"435 Documentary \n",
"1377 Action, Drama, Thriller \n",
"2319 Drama \n",
"3261 Drama \n",
"5145 Drama \n",
"6087 Comedy \n",
"7030 Drama \n",
"8924 Documentary \n",
"7970 Drama \n",
"8485 Comedy "
]
},
"execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"train=pd.read_csv('./Datasets/ml-100k/train.csv', sep='\\t', header=None, names=['user', 'item', 'rating', 'timestamp'])\n",
"items=pd.read_csv('./Datasets/ml-100k/movies.csv')\n",
"\n",
"user=random.choice(list(set(train['user'])))\n",
"\n",
"train_content=pd.merge(train, items, left_on='item', right_on='id')\n",
"\n",
"print('Here is what user rated high:')\n",
"display(train_content[train_content['user']==user][['user', 'rating', 'title', 'genres']]\\\n",
" .sort_values(by='rating', ascending=False)[:15])\n",
"\n",
"reco = np.loadtxt('Recommendations generated/ml-100k/Self_BaselineUI_reco.csv', delimiter=',')\n",
"items=pd.read_csv('./Datasets/ml-100k/movies.csv')\n",
"\n",
"# Let's ignore scores - they are not used in evaluation: \n",
"reco_users=reco[:,:1]\n",
"reco_items=reco[:,1::2]\n",
"# Let's put them into one array\n",
"reco=np.concatenate((reco_users, reco_items), axis=1)\n",
"\n",
"# Let's rebuild it user-item dataframe\n",
"recommended=[]\n",
"for row in reco:\n",
" for rec_nb, entry in enumerate(row[1:]):\n",
" recommended.append((row[0], rec_nb+1, entry))\n",
"recommended=pd.DataFrame(recommended, columns=['user','rec_nb', 'item'])\n",
"\n",
"recommended_content=pd.merge(recommended, items, left_on='item', right_on='id')\n",
"\n",
"print('Here is what we recommend:')\n",
"recommended_content[recommended_content['user']==user][['user', 'rec_nb', 'title', 'genres']].sort_values(by='rec_nb')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# project task 3: implement some other evaluation measure"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
"# it may be your idea, modification of what we have already implemented \n",
"# (for example Hit2 rate which would count as a success users whoreceived at least 2 relevant recommendations) \n",
"# or something well-known\n",
"# expected output: modification of evaluation_measures.py such that evaluate_all will also display your measure"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"943it [00:00, 5476.88it/s]\n",
"943it [00:00, 4421.14it/s]\n",
"943it [00:00, 5056.87it/s]\n",
"943it [00:00, 5642.22it/s]\n",
"943it [00:00, 2776.13it/s]\n",
"943it [00:00, 3004.22it/s]\n",
"943it [00:00, 3802.86it/s]\n",
"943it [00:00, 3421.26it/s]\n",
"943it [00:00, 5077.51it/s]\n",
"943it [00:00, 4927.51it/s]\n",
"943it [00:00, 4246.38it/s]\n",
"943it [00:00, 4295.31it/s]\n",
"943it [00:00, 4362.79it/s]\n",
"943it [00:00, 6241.10it/s]\n",
"943it [00:00, 4318.95it/s]\n",
"943it [00:00, 5054.75it/s]\n",
"943it [00:00, 3839.80it/s]\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_LightFM</td>\n",
" <td>162.703697</td>\n",
" <td>160.837311</td>\n",
" <td>0.349523</td>\n",
" <td>0.226193</td>\n",
" <td>0.225202</td>\n",
" <td>0.265538</td>\n",
" <td>0.246459</td>\n",
" <td>0.266934</td>\n",
" <td>0.413969</td>\n",
" <td>0.277036</td>\n",
" <td>0.648029</td>\n",
" <td>0.610845</td>\n",
" <td>0.916225</td>\n",
" <td>1.000000</td>\n",
" <td>0.352814</td>\n",
" <td>5.363070</td>\n",
" <td>0.885116</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_LightFMpureMF</td>\n",
" <td>8.015665</td>\n",
" <td>7.520402</td>\n",
" <td>0.333934</td>\n",
" <td>0.216047</td>\n",
" <td>0.214731</td>\n",
" <td>0.253177</td>\n",
" <td>0.232725</td>\n",
" <td>0.254485</td>\n",
" <td>0.391316</td>\n",
" <td>0.257793</td>\n",
" <td>0.606204</td>\n",
" <td>0.605708</td>\n",
" <td>0.906681</td>\n",
" <td>1.000000</td>\n",
" <td>0.272006</td>\n",
" <td>5.031437</td>\n",
" <td>0.918177</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_P3</td>\n",
" <td>3.702446</td>\n",
" <td>3.527273</td>\n",
" <td>0.282185</td>\n",
" <td>0.192092</td>\n",
" <td>0.186749</td>\n",
" <td>0.216980</td>\n",
" <td>0.204185</td>\n",
" <td>0.240096</td>\n",
" <td>0.339114</td>\n",
" <td>0.204905</td>\n",
" <td>0.572157</td>\n",
" <td>0.593544</td>\n",
" <td>0.875928</td>\n",
" <td>1.000000</td>\n",
" <td>0.077201</td>\n",
" <td>3.875892</td>\n",
" <td>0.974947</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_ImplicitALS</td>\n",
" <td>3.267237</td>\n",
" <td>3.068493</td>\n",
" <td>0.252068</td>\n",
" <td>0.182639</td>\n",
" <td>0.175182</td>\n",
" <td>0.199457</td>\n",
" <td>0.167167</td>\n",
" <td>0.216308</td>\n",
" <td>0.295331</td>\n",
" <td>0.163847</td>\n",
" <td>0.500282</td>\n",
" <td>0.588672</td>\n",
" <td>0.873807</td>\n",
" <td>0.999894</td>\n",
" <td>0.497835</td>\n",
" <td>5.727745</td>\n",
" <td>0.825683</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_TopPop</td>\n",
" <td>2.508258</td>\n",
" <td>2.217909</td>\n",
" <td>0.188865</td>\n",
" <td>0.116919</td>\n",
" <td>0.118732</td>\n",
" <td>0.141584</td>\n",
" <td>0.130472</td>\n",
" <td>0.137473</td>\n",
" <td>0.214651</td>\n",
" <td>0.111707</td>\n",
" <td>0.400939</td>\n",
" <td>0.555546</td>\n",
" <td>0.765642</td>\n",
" <td>1.000000</td>\n",
" <td>0.038961</td>\n",
" <td>3.159079</td>\n",
" <td>0.987317</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_LightFMcontent</td>\n",
" <td>182.840876</td>\n",
" <td>180.771141</td>\n",
" <td>0.161294</td>\n",
" <td>0.100424</td>\n",
" <td>0.101736</td>\n",
" <td>0.121096</td>\n",
" <td>0.101395</td>\n",
" <td>0.110660</td>\n",
" <td>0.184311</td>\n",
" <td>0.091346</td>\n",
" <td>0.352019</td>\n",
" <td>0.547187</td>\n",
" <td>0.705196</td>\n",
" <td>0.979533</td>\n",
" <td>0.269120</td>\n",
" <td>4.940084</td>\n",
" <td>0.924146</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_SVD</td>\n",
" <td>0.953076</td>\n",
" <td>0.750219</td>\n",
" <td>0.094804</td>\n",
" <td>0.045302</td>\n",
" <td>0.051519</td>\n",
" <td>0.065833</td>\n",
" <td>0.083691</td>\n",
" <td>0.074336</td>\n",
" <td>0.107620</td>\n",
" <td>0.051155</td>\n",
" <td>0.234251</td>\n",
" <td>0.519361</td>\n",
" <td>0.490986</td>\n",
" <td>0.993425</td>\n",
" <td>0.206349</td>\n",
" <td>4.406898</td>\n",
" <td>0.953781</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_SVD</td>\n",
" <td>0.913840</td>\n",
" <td>0.717167</td>\n",
" <td>0.105620</td>\n",
" <td>0.044070</td>\n",
" <td>0.053839</td>\n",
" <td>0.071381</td>\n",
" <td>0.096030</td>\n",
" <td>0.074982</td>\n",
" <td>0.109138</td>\n",
" <td>0.051857</td>\n",
" <td>0.202054</td>\n",
" <td>0.518772</td>\n",
" <td>0.478261</td>\n",
" <td>0.872959</td>\n",
" <td>0.144300</td>\n",
" <td>3.912577</td>\n",
" <td>0.971609</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Baseline</td>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" <td>0.091410</td>\n",
" <td>0.037652</td>\n",
" <td>0.046030</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
" <td>1.000000</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_SVDBiased</td>\n",
" <td>0.941830</td>\n",
" <td>0.742841</td>\n",
" <td>0.083033</td>\n",
" <td>0.034867</td>\n",
" <td>0.041967</td>\n",
" <td>0.055644</td>\n",
" <td>0.072425</td>\n",
" <td>0.054271</td>\n",
" <td>0.090974</td>\n",
" <td>0.041243</td>\n",
" <td>0.195741</td>\n",
" <td>0.514084</td>\n",
" <td>0.418876</td>\n",
" <td>0.998409</td>\n",
" <td>0.168831</td>\n",
" <td>4.152102</td>\n",
" <td>0.964603</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_GlobalAvg</td>\n",
" <td>1.125760</td>\n",
" <td>0.943534</td>\n",
" <td>0.061188</td>\n",
" <td>0.025968</td>\n",
" <td>0.031383</td>\n",
" <td>0.041343</td>\n",
" <td>0.040558</td>\n",
" <td>0.032107</td>\n",
" <td>0.067695</td>\n",
" <td>0.027470</td>\n",
" <td>0.171187</td>\n",
" <td>0.509546</td>\n",
" <td>0.384942</td>\n",
" <td>1.000000</td>\n",
" <td>0.025974</td>\n",
" <td>2.711772</td>\n",
" <td>0.992003</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Random</td>\n",
" <td>1.513348</td>\n",
" <td>1.214309</td>\n",
" <td>0.044221</td>\n",
" <td>0.019366</td>\n",
" <td>0.022599</td>\n",
" <td>0.029593</td>\n",
" <td>0.026288</td>\n",
" <td>0.018226</td>\n",
" <td>0.047273</td>\n",
" <td>0.017729</td>\n",
" <td>0.114687</td>\n",
" <td>0.506181</td>\n",
" <td>0.301166</td>\n",
" <td>0.986002</td>\n",
" <td>0.184704</td>\n",
" <td>5.093324</td>\n",
" <td>0.907405</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_I-KNN</td>\n",
" <td>1.030386</td>\n",
" <td>0.813067</td>\n",
" <td>0.026087</td>\n",
" <td>0.006908</td>\n",
" <td>0.010593</td>\n",
" <td>0.016046</td>\n",
" <td>0.021137</td>\n",
" <td>0.009522</td>\n",
" <td>0.024214</td>\n",
" <td>0.008958</td>\n",
" <td>0.048068</td>\n",
" <td>0.499885</td>\n",
" <td>0.154825</td>\n",
" <td>0.402333</td>\n",
" <td>0.434343</td>\n",
" <td>5.133650</td>\n",
" <td>0.877999</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_I-KNNBaseline</td>\n",
" <td>0.935327</td>\n",
" <td>0.737424</td>\n",
" <td>0.002545</td>\n",
" <td>0.000755</td>\n",
" <td>0.001105</td>\n",
" <td>0.001602</td>\n",
" <td>0.002253</td>\n",
" <td>0.000930</td>\n",
" <td>0.003444</td>\n",
" <td>0.001362</td>\n",
" <td>0.011760</td>\n",
" <td>0.496724</td>\n",
" <td>0.021209</td>\n",
" <td>0.482821</td>\n",
" <td>0.059885</td>\n",
" <td>2.232578</td>\n",
" <td>0.994487</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_U-KNN</td>\n",
" <td>1.023495</td>\n",
" <td>0.807913</td>\n",
" <td>0.000742</td>\n",
" <td>0.000205</td>\n",
" <td>0.000305</td>\n",
" <td>0.000449</td>\n",
" <td>0.000536</td>\n",
" <td>0.000198</td>\n",
" <td>0.000845</td>\n",
" <td>0.000274</td>\n",
" <td>0.002744</td>\n",
" <td>0.496441</td>\n",
" <td>0.007423</td>\n",
" <td>0.602121</td>\n",
" <td>0.010823</td>\n",
" <td>2.089186</td>\n",
" <td>0.995706</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineUI</td>\n",
" <td>0.967585</td>\n",
" <td>0.762740</td>\n",
" <td>0.000954</td>\n",
" <td>0.000170</td>\n",
" <td>0.000278</td>\n",
" <td>0.000463</td>\n",
" <td>0.000644</td>\n",
" <td>0.000189</td>\n",
" <td>0.000752</td>\n",
" <td>0.000168</td>\n",
" <td>0.001677</td>\n",
" <td>0.496424</td>\n",
" <td>0.009544</td>\n",
" <td>0.600530</td>\n",
" <td>0.005051</td>\n",
" <td>1.803126</td>\n",
" <td>0.996380</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_IKNN</td>\n",
" <td>1.018363</td>\n",
" <td>0.808793</td>\n",
" <td>0.000318</td>\n",
" <td>0.000108</td>\n",
" <td>0.000140</td>\n",
" <td>0.000189</td>\n",
" <td>0.000000</td>\n",
" <td>0.000000</td>\n",
" <td>0.000214</td>\n",
" <td>0.000037</td>\n",
" <td>0.000368</td>\n",
" <td>0.496391</td>\n",
" <td>0.003181</td>\n",
" <td>0.392153</td>\n",
" <td>0.115440</td>\n",
" <td>4.174741</td>\n",
" <td>0.965327</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Model RMSE MAE precision recall \\\n",
"0 Ready_LightFM 162.703697 160.837311 0.349523 0.226193 \n",
"0 Ready_LightFMpureMF 8.015665 7.520402 0.333934 0.216047 \n",
"0 Self_P3 3.702446 3.527273 0.282185 0.192092 \n",
"0 Ready_ImplicitALS 3.267237 3.068493 0.252068 0.182639 \n",
"0 Self_TopPop 2.508258 2.217909 0.188865 0.116919 \n",
"0 Ready_LightFMcontent 182.840876 180.771141 0.161294 0.100424 \n",
"0 Ready_SVD 0.953076 0.750219 0.094804 0.045302 \n",
"0 Self_SVD 0.913840 0.717167 0.105620 0.044070 \n",
"0 Ready_Baseline 0.949459 0.752487 0.091410 0.037652 \n",
"0 Ready_SVDBiased 0.941830 0.742841 0.083033 0.034867 \n",
"0 Self_GlobalAvg 1.125760 0.943534 0.061188 0.025968 \n",
"0 Ready_Random 1.513348 1.214309 0.044221 0.019366 \n",
"0 Ready_I-KNN 1.030386 0.813067 0.026087 0.006908 \n",
"0 Ready_I-KNNBaseline 0.935327 0.737424 0.002545 0.000755 \n",
"0 Ready_U-KNN 1.023495 0.807913 0.000742 0.000205 \n",
"0 Self_BaselineUI 0.967585 0.762740 0.000954 0.000170 \n",
"0 Self_IKNN 1.018363 0.808793 0.000318 0.000108 \n",
"\n",
" F_1 F_05 precision_super recall_super NDCG mAP \\\n",
"0 0.225202 0.265538 0.246459 0.266934 0.413969 0.277036 \n",
"0 0.214731 0.253177 0.232725 0.254485 0.391316 0.257793 \n",
"0 0.186749 0.216980 0.204185 0.240096 0.339114 0.204905 \n",
"0 0.175182 0.199457 0.167167 0.216308 0.295331 0.163847 \n",
"0 0.118732 0.141584 0.130472 0.137473 0.214651 0.111707 \n",
"0 0.101736 0.121096 0.101395 0.110660 0.184311 0.091346 \n",
"0 0.051519 0.065833 0.083691 0.074336 0.107620 0.051155 \n",
"0 0.053839 0.071381 0.096030 0.074982 0.109138 0.051857 \n",
"0 0.046030 0.061286 0.079614 0.056463 0.095957 0.043178 \n",
"0 0.041967 0.055644 0.072425 0.054271 0.090974 0.041243 \n",
"0 0.031383 0.041343 0.040558 0.032107 0.067695 0.027470 \n",
"0 0.022599 0.029593 0.026288 0.018226 0.047273 0.017729 \n",
"0 0.010593 0.016046 0.021137 0.009522 0.024214 0.008958 \n",
"0 0.001105 0.001602 0.002253 0.000930 0.003444 0.001362 \n",
"0 0.000305 0.000449 0.000536 0.000198 0.000845 0.000274 \n",
"0 0.000278 0.000463 0.000644 0.000189 0.000752 0.000168 \n",
"0 0.000140 0.000189 0.000000 0.000000 0.000214 0.000037 \n",
"\n",
" MRR LAUC HR Reco in test Test coverage Shannon \\\n",
"0 0.648029 0.610845 0.916225 1.000000 0.352814 5.363070 \n",
"0 0.606204 0.605708 0.906681 1.000000 0.272006 5.031437 \n",
"0 0.572157 0.593544 0.875928 1.000000 0.077201 3.875892 \n",
"0 0.500282 0.588672 0.873807 0.999894 0.497835 5.727745 \n",
"0 0.400939 0.555546 0.765642 1.000000 0.038961 3.159079 \n",
"0 0.352019 0.547187 0.705196 0.979533 0.269120 4.940084 \n",
"0 0.234251 0.519361 0.490986 0.993425 0.206349 4.406898 \n",
"0 0.202054 0.518772 0.478261 0.872959 0.144300 3.912577 \n",
"0 0.198193 0.515501 0.437964 1.000000 0.033911 2.836513 \n",
"0 0.195741 0.514084 0.418876 0.998409 0.168831 4.152102 \n",
"0 0.171187 0.509546 0.384942 1.000000 0.025974 2.711772 \n",
"0 0.114687 0.506181 0.301166 0.986002 0.184704 5.093324 \n",
"0 0.048068 0.499885 0.154825 0.402333 0.434343 5.133650 \n",
"0 0.011760 0.496724 0.021209 0.482821 0.059885 2.232578 \n",
"0 0.002744 0.496441 0.007423 0.602121 0.010823 2.089186 \n",
"0 0.001677 0.496424 0.009544 0.600530 0.005051 1.803126 \n",
"0 0.000368 0.496391 0.003181 0.392153 0.115440 4.174741 \n",
"\n",
" Gini \n",
"0 0.885116 \n",
"0 0.918177 \n",
"0 0.974947 \n",
"0 0.825683 \n",
"0 0.987317 \n",
"0 0.924146 \n",
"0 0.953781 \n",
"0 0.971609 \n",
"0 0.991139 \n",
"0 0.964603 \n",
"0 0.992003 \n",
"0 0.907405 \n",
"0 0.877999 \n",
"0 0.994487 \n",
"0 0.995706 \n",
"0 0.996380 \n",
"0 0.965327 "
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dir_path=\"Recommendations generated/ml-100k/\"\n",
"super_reactions=[4,5]\n",
"test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None)\n",
"\n",
"ev.evaluate_all(test, dir_path, super_reactions)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.9"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Reference in New Issue Copy Permalink