s406917/workshops_recommender_systems
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
8ad2469287
workshops_recommender_systems/P2. Evaluation.ipynb

1815 lines
58 KiB
Plaintext
Raw Normal View History

Codes added
2020-05-21 13:42:50 +02:00
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Prepare test set"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 1,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {
"slideshow": {
"slide_type": "-"
}
},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import scipy.sparse as sparse\n",
"from collections import defaultdict\n",
"from itertools import chain\n",
"import random\n",
"from tqdm import tqdm\n",
"\n",
"# In evaluation we do not load train set - it is not needed\n",
"test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None)\n",
"test.columns=['user', 'item', 'rating', 'timestamp']\n",
"\n",
"test['user_code'] = test['user'].astype(\"category\").cat.codes\n",
"test['item_code'] = test['item'].astype(\"category\").cat.codes\n",
"\n",
"user_code_id = dict(enumerate(test['user'].astype(\"category\").cat.categories))\n",
"user_id_code = dict((v, k) for k, v in user_code_id.items())\n",
"item_code_id = dict(enumerate(test['item'].astype(\"category\").cat.categories))\n",
"item_id_code = dict((v, k) for k, v in item_code_id.items())\n",
"\n",
"test_ui = sparse.csr_matrix((test['rating'], (test['user_code'], test['item_code'])))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Estimations metrics"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 2,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [],
"source": [
"estimations_df=pd.read_csv('Recommendations generated/ml-100k/Ready_Baseline_estimations.csv', header=None)\n",
"estimations_df.columns=['user', 'item' ,'score']\n",
"\n",
"estimations_df['user_code']=[user_id_code[user] for user in estimations_df['user']]\n",
"estimations_df['item_code']=[item_id_code[item] for item in estimations_df['item']]\n",
"estimations=sparse.csr_matrix((estimations_df['score'], (estimations_df['user_code'], estimations_df['item_code'])), shape=test_ui.shape)"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 3,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [],
"source": [
"def estimations_metrics(test_ui, estimations):\n",
" result=[]\n",
"\n",
" RMSE=(np.sum((estimations.data-test_ui.data)**2)/estimations.nnz)**(1/2)\n",
" result.append(['RMSE', RMSE])\n",
"\n",
" MAE=np.sum(abs(estimations.data-test_ui.data))/estimations.nnz\n",
" result.append(['MAE', MAE])\n",
" \n",
" df_result=(pd.DataFrame(list(zip(*result))[1])).T\n",
" df_result.columns=list(zip(*result))[0]\n",
" return df_result"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 4,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" RMSE MAE\n",
"0 0.949459 0.752487"
]
},
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 4,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
Slides added
2020-05-21 16:20:12 +02:00
"# in case of error (in the laboratories) you might have to switch to the other version of pandas\n",
"# try !pip3 install pandas=='1.0.3' (or pip if you use python 2) and restart the kernel\n",
"\n",
Codes added
2020-05-21 13:42:50 +02:00
"estimations_metrics(test_ui, estimations)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Ranking metrics"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 5,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[663, 475, 62, ..., 472, 269, 503],\n",
" [ 48, 313, 475, ..., 591, 175, 466],\n",
" [351, 313, 475, ..., 591, 175, 466],\n",
" ...,\n",
" [259, 313, 475, ..., 11, 591, 175],\n",
" [ 33, 313, 475, ..., 11, 591, 175],\n",
" [ 77, 313, 475, ..., 11, 591, 175]])"
]
},
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 5,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import numpy as np\n",
"reco = np.loadtxt('Recommendations generated/ml-100k/Ready_Baseline_reco.csv', delimiter=',')\n",
"# Let's ignore scores - they are not used in evaluation: \n",
"users=reco[:,:1]\n",
"items=reco[:,1::2]\n",
"# Let's use inner ids instead of real ones\n",
"users=np.vectorize(lambda x: user_id_code.setdefault(x, -1))(users)\n",
"items=np.vectorize(lambda x: item_id_code.setdefault(x, -1))(items) # maybe items we recommend are not in test set\n",
"# Let's put them into one array\n",
"reco=np.concatenate((users, items), axis=1)\n",
"reco"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 6,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [],
"source": [
"def ranking_metrics(test_ui, reco, super_reactions=[], topK=10):\n",
" \n",
" nb_items=test_ui.shape[1]\n",
" relevant_users, super_relevant_users, prec, rec, F_1, F_05, prec_super, rec_super, ndcg, mAP, MRR, LAUC, HR=\\\n",
" 0,0,0,0,0,0,0,0,0,0,0,0,0\n",
" \n",
" cg = (1.0 / np.log2(np.arange(2, topK + 2)))\n",
" cg_sum = np.cumsum(cg)\n",
" \n",
" for (nb_user, user) in tqdm(enumerate(reco[:,0])):\n",
" u_rated_items=test_ui.indices[test_ui.indptr[user]:test_ui.indptr[user+1]]\n",
" nb_u_rated_items=len(u_rated_items)\n",
" if nb_u_rated_items>0: # skip users with no items in test set (still possible that there will be no super items)\n",
" relevant_users+=1\n",
" \n",
" u_super_items=u_rated_items[np.vectorize(lambda x: x in super_reactions)\\\n",
" (test_ui.data[test_ui.indptr[user]:test_ui.indptr[user+1]])]\n",
" # more natural seems u_super_items=[item for item in u_rated_items if test_ui[user,item] in super_reactions]\n",
" # but accesing test_ui[user,item] is expensive -we should avoid doing it\n",
" if len(u_super_items)>0:\n",
" super_relevant_users+=1\n",
" \n",
" user_successes=np.zeros(topK)\n",
" nb_user_successes=0\n",
" user_super_successes=np.zeros(topK)\n",
" nb_user_super_successes=0\n",
" \n",
" # evaluation\n",
" for (item_position,item) in enumerate(reco[nb_user,1:topK+1]):\n",
" if item in u_rated_items:\n",
" user_successes[item_position]=1\n",
" nb_user_successes+=1\n",
" if item in u_super_items:\n",
" user_super_successes[item_position]=1\n",
" nb_user_super_successes+=1\n",
" \n",
" prec_u=nb_user_successes/topK \n",
" prec+=prec_u\n",
" \n",
" rec_u=nb_user_successes/nb_u_rated_items\n",
" rec+=rec_u\n",
" \n",
" F_1+=2*(prec_u*rec_u)/(prec_u+rec_u) if prec_u+rec_u>0 else 0\n",
" F_05+=(0.5**2+1)*(prec_u*rec_u)/(0.5**2*prec_u+rec_u) if prec_u+rec_u>0 else 0\n",
" \n",
" prec_super+=nb_user_super_successes/topK\n",
" rec_super+=nb_user_super_successes/max(len(u_super_items),1) # to set 0 if no super items\n",
" ndcg+=np.dot(user_successes,cg)/cg_sum[min(topK, nb_u_rated_items)-1]\n",
" \n",
" cumsum_successes=np.cumsum(user_successes)\n",
" mAP+=np.dot(cumsum_successes/np.arange(1,topK+1), user_successes)/min(topK, nb_u_rated_items)\n",
" MRR+=1/(user_successes.nonzero()[0][0]+1) if user_successes.nonzero()[0].size>0 else 0\n",
" LAUC+=(np.dot(cumsum_successes, 1-user_successes)+\\\n",
" (nb_user_successes+nb_u_rated_items)/2*((nb_items-nb_u_rated_items)-(topK-nb_user_successes)))/\\\n",
" ((nb_items-nb_u_rated_items)*nb_u_rated_items)\n",
" \n",
" HR+=nb_user_successes>0\n",
" \n",
" \n",
" result=[]\n",
" result.append(('precision', prec/relevant_users))\n",
" result.append(('recall', rec/relevant_users))\n",
" result.append(('F_1', F_1/relevant_users))\n",
" result.append(('F_05', F_05/relevant_users))\n",
" result.append(('precision_super', prec_super/super_relevant_users))\n",
" result.append(('recall_super', rec_super/super_relevant_users))\n",
" result.append(('NDCG', ndcg/relevant_users))\n",
" result.append(('mAP', mAP/relevant_users))\n",
" result.append(('MRR', MRR/relevant_users))\n",
" result.append(('LAUC', LAUC/relevant_users))\n",
" result.append(('HR', HR/relevant_users))\n",
"\n",
" df_result=(pd.DataFrame(list(zip(*result))[1])).T\n",
" df_result.columns=list(zip(*result))[0]\n",
" return df_result"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 7,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
Slides added
2020-05-21 16:20:12 +02:00
"943it [00:00, 9409.37it/s]\n"
Codes added
2020-05-21 13:42:50 +02:00
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.09141</td>\n",
" <td>0.037652</td>\n",
" <td>0.04603</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" precision recall F_1 F_05 precision_super recall_super \\\n",
"0 0.09141 0.037652 0.04603 0.061286 0.079614 0.056463 \n",
"\n",
" NDCG mAP MRR LAUC HR \n",
"0 0.095957 0.043178 0.198193 0.515501 0.437964 "
]
},
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 7,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ranking_metrics(test_ui, reco, super_reactions=[4,5], topK=10)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Diversity metrics"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"def diversity_metrics(test_ui, reco, topK=10):\n",
" \n",
" frequencies=defaultdict(int)\n",
" \n",
" # let's assign 0 to all items in test set\n",
" for item in list(set(test_ui.indices)):\n",
" frequencies[item]=0\n",
" \n",
" # counting frequencies\n",
" for item in reco[:,1:].flat:\n",
" frequencies[item]+=1\n",
" \n",
" nb_reco_outside_test=frequencies[-1]\n",
" del frequencies[-1]\n",
" \n",
" frequencies=np.array(list(frequencies.values()))\n",
" \n",
" nb_rec_items=len(frequencies[frequencies>0])\n",
" nb_reco_inside_test=np.sum(frequencies)\n",
" \n",
" frequencies=frequencies/np.sum(frequencies)\n",
" frequencies=np.sort(frequencies)\n",
" \n",
" with np.errstate(divide='ignore'): # let's put zeros put items with 0 frequency and ignore division warning\n",
" log_frequencies=np.nan_to_num(np.log(frequencies), posinf=0, neginf=0)\n",
" \n",
" result=[]\n",
" result.append(('Reco in test', nb_reco_inside_test/(nb_reco_inside_test+nb_reco_outside_test)))\n",
" result.append(('Test coverage', nb_rec_items/test_ui.shape[1]))\n",
" result.append(('Shannon', -np.dot(frequencies, log_frequencies)))\n",
" result.append(('Gini', np.dot(frequencies, np.arange(1-len(frequencies), len(frequencies), 2))/(len(frequencies)-1)))\n",
" \n",
" df_result=(pd.DataFrame(list(zip(*result))[1])).T\n",
" df_result.columns=list(zip(*result))[0]\n",
" return df_result"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 14,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1.0</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Reco in test Test coverage Shannon Gini\n",
"0 1.0 0.033911 2.836513 0.991139"
]
},
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 14,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
Slides added
2020-05-21 16:20:12 +02:00
"# in case of errors try !pip3 install numpy==1.18.4 (or pip if you use python 2) and restart the kernel\n",
"\n",
Codes added
2020-05-21 13:42:50 +02:00
"import evaluation_measures as ev\n",
"import imp\n",
"imp.reload(ev)\n",
"\n",
"x=diversity_metrics(test_ui, reco, topK=10)\n",
"x"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# To be used in other notebooks"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 15,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
Slides added
2020-05-21 16:20:12 +02:00
"943it [00:00, 8290.29it/s]\n"
Codes added
2020-05-21 13:42:50 +02:00
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" <td>0.09141</td>\n",
" <td>0.037652</td>\n",
" <td>0.04603</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
" <td>1.0</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" RMSE MAE precision recall F_1 F_05 \\\n",
"0 0.949459 0.752487 0.09141 0.037652 0.04603 0.061286 \n",
"\n",
" precision_super recall_super NDCG mAP MRR LAUC \\\n",
"0 0.079614 0.056463 0.095957 0.043178 0.198193 0.515501 \n",
"\n",
" HR Reco in test Test coverage Shannon Gini \n",
"0 0.437964 1.0 0.033911 2.836513 0.991139 "
]
},
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 15,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import evaluation_measures as ev\n",
"import imp\n",
"imp.reload(ev)\n",
"\n",
"estimations_df=pd.read_csv('Recommendations generated/ml-100k/Ready_Baseline_estimations.csv', header=None)\n",
"reco=np.loadtxt('Recommendations generated/ml-100k/Ready_Baseline_reco.csv', delimiter=',')\n",
"\n",
"ev.evaluate(test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None),\n",
" estimations_df=estimations_df, \n",
" reco=reco,\n",
" super_reactions=[4,5])\n",
"#also you can just type ev.evaluate_all(estimations_df, reco) - I put above values as default"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 16,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
Slides added
2020-05-21 16:20:12 +02:00
"943it [00:00, 9608.49it/s]\n",
"943it [00:00, 9837.12it/s]\n",
"943it [00:00, 10292.46it/s]\n",
"943it [00:00, 9906.94it/s]\n",
"943it [00:00, 9162.09it/s]\n"
Codes added
2020-05-21 13:42:50 +02:00
]
}
],
"source": [
"import evaluation_measures as ev\n",
"import imp\n",
"imp.reload(ev)\n",
"\n",
"dir_path=\"Recommendations generated/ml-100k/\"\n",
"super_reactions=[4,5]\n",
"test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None)\n",
"\n",
"df=ev.evaluate_all(test, dir_path, super_reactions)\n",
"#also you can just type ev.evaluate_all() - I put above values as default"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 17,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_TopPop</td>\n",
" <td>2.508258</td>\n",
" <td>2.217909</td>\n",
" <td>0.188865</td>\n",
" <td>0.116919</td>\n",
" <td>0.118732</td>\n",
" <td>0.141584</td>\n",
" <td>0.130472</td>\n",
" <td>0.137473</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Baseline</td>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" <td>0.091410</td>\n",
" <td>0.037652</td>\n",
" <td>0.046030</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_GlobalAvg</td>\n",
" <td>1.125760</td>\n",
" <td>0.943534</td>\n",
" <td>0.061188</td>\n",
" <td>0.025968</td>\n",
" <td>0.031383</td>\n",
" <td>0.041343</td>\n",
" <td>0.040558</td>\n",
" <td>0.032107</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Random</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>1.518964</td>\n",
" <td>1.222159</td>\n",
" <td>0.046554</td>\n",
" <td>0.020603</td>\n",
" <td>0.023679</td>\n",
" <td>0.031216</td>\n",
" <td>0.028970</td>\n",
" <td>0.021179</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineUI</td>\n",
" <td>0.967585</td>\n",
" <td>0.762740</td>\n",
" <td>0.000954</td>\n",
" <td>0.000170</td>\n",
" <td>0.000278</td>\n",
" <td>0.000463</td>\n",
" <td>0.000644</td>\n",
" <td>0.000189</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
Slides added
2020-05-21 16:20:12 +02:00
" Model RMSE MAE precision recall F_1 \\\n",
"0 Self_TopPop 2.508258 2.217909 0.188865 0.116919 0.118732 \n",
"0 Ready_Baseline 0.949459 0.752487 0.091410 0.037652 0.046030 \n",
"0 Self_GlobalAvg 1.125760 0.943534 0.061188 0.025968 0.031383 \n",
"0 Ready_Random 1.518964 1.222159 0.046554 0.020603 0.023679 \n",
"0 Self_BaselineUI 0.967585 0.762740 0.000954 0.000170 0.000278 \n",
Codes added
2020-05-21 13:42:50 +02:00
"\n",
" F_05 precision_super recall_super \n",
"0 0.141584 0.130472 0.137473 \n",
"0 0.061286 0.079614 0.056463 \n",
"0 0.041343 0.040558 0.032107 \n",
Slides added
2020-05-21 16:20:12 +02:00
"0 0.031216 0.028970 0.021179 \n",
"0 0.000463 0.000644 0.000189 "
Codes added
2020-05-21 13:42:50 +02:00
]
},
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 17,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.iloc[:,:9]"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 18,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_TopPop</td>\n",
" <td>0.214651</td>\n",
" <td>0.111707</td>\n",
" <td>0.400939</td>\n",
" <td>0.555546</td>\n",
" <td>0.765642</td>\n",
" <td>1.000000</td>\n",
" <td>0.038961</td>\n",
" <td>3.159079</td>\n",
" <td>0.987317</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Baseline</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
" <td>1.000000</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_GlobalAvg</td>\n",
" <td>0.067695</td>\n",
" <td>0.027470</td>\n",
" <td>0.171187</td>\n",
" <td>0.509546</td>\n",
" <td>0.384942</td>\n",
" <td>1.000000</td>\n",
" <td>0.025974</td>\n",
" <td>2.711772</td>\n",
" <td>0.992003</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Random</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>0.050489</td>\n",
" <td>0.019185</td>\n",
" <td>0.123856</td>\n",
" <td>0.506812</td>\n",
" <td>0.322375</td>\n",
" <td>0.987805</td>\n",
" <td>0.184704</td>\n",
" <td>5.103172</td>\n",
" <td>0.906873</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineUI</td>\n",
" <td>0.000752</td>\n",
" <td>0.000168</td>\n",
" <td>0.001677</td>\n",
" <td>0.496424</td>\n",
" <td>0.009544</td>\n",
" <td>0.600530</td>\n",
" <td>0.005051</td>\n",
" <td>1.803126</td>\n",
" <td>0.996380</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
Slides added
2020-05-21 16:20:12 +02:00
" Model NDCG mAP MRR LAUC HR \\\n",
"0 Self_TopPop 0.214651 0.111707 0.400939 0.555546 0.765642 \n",
"0 Ready_Baseline 0.095957 0.043178 0.198193 0.515501 0.437964 \n",
"0 Self_GlobalAvg 0.067695 0.027470 0.171187 0.509546 0.384942 \n",
"0 Ready_Random 0.050489 0.019185 0.123856 0.506812 0.322375 \n",
"0 Self_BaselineUI 0.000752 0.000168 0.001677 0.496424 0.009544 \n",
Codes added
2020-05-21 13:42:50 +02:00
"\n",
" Reco in test Test coverage Shannon Gini \n",
"0 1.000000 0.038961 3.159079 0.987317 \n",
"0 1.000000 0.033911 2.836513 0.991139 \n",
"0 1.000000 0.025974 2.711772 0.992003 \n",
Slides added
2020-05-21 16:20:12 +02:00
"0 0.987805 0.184704 5.103172 0.906873 \n",
"0 0.600530 0.005051 1.803126 0.996380 "
Codes added
2020-05-21 13:42:50 +02:00
]
},
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 18,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.iloc[:,np.append(0,np.arange(9, df.shape[1]))]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Check metrics on toy dataset"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 19,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
Slides added
2020-05-21 16:20:12 +02:00
"3it [00:00, 1651.52it/s]\n"
Codes added
2020-05-21 13:42:50 +02:00
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineUI</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>1.612452</td>\n",
" <td>1.4</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>0.444444</td>\n",
" <td>0.888889</td>\n",
" <td>0.555556</td>\n",
" <td>0.478632</td>\n",
" <td>0.333333</td>\n",
" <td>0.75</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>0.676907</td>\n",
" <td>0.574074</td>\n",
" <td>0.611111</td>\n",
" <td>0.638889</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>1.0</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>0.888889</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>0.8</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>1.386294</td>\n",
" <td>0.25</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
Slides added
2020-05-21 16:20:12 +02:00
" Model RMSE MAE precision recall F_1 F_05 \\\n",
"0 Self_BaselineUI 1.612452 1.4 0.444444 0.888889 0.555556 0.478632 \n",
Codes added
2020-05-21 13:42:50 +02:00
"\n",
Slides added
2020-05-21 16:20:12 +02:00
" precision_super recall_super NDCG mAP MRR LAUC HR \\\n",
"0 0.333333 0.75 0.676907 0.574074 0.611111 0.638889 1.0 \n",
Codes added
2020-05-21 13:42:50 +02:00
"\n",
Slides added
2020-05-21 16:20:12 +02:00
" Reco in test Test coverage Shannon Gini \n",
"0 0.888889 0.8 1.386294 0.25 "
Codes added
2020-05-21 13:42:50 +02:00
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Training data:\n"
]
},
{
"data": {
"text/plain": [
"matrix([[3, 4, 0, 0, 5, 0, 0, 4],\n",
" [0, 1, 2, 3, 0, 0, 0, 0],\n",
" [0, 0, 0, 5, 0, 3, 4, 0]], dtype=int64)"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Test data:\n"
]
},
{
"data": {
"text/plain": [
"matrix([[0, 0, 0, 0, 0, 0, 3, 0],\n",
" [0, 0, 0, 0, 5, 0, 0, 0],\n",
" [5, 0, 4, 0, 0, 0, 0, 2]], dtype=int64)"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Recommendations:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>0</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" <th>3</th>\n",
" <th>4</th>\n",
" <th>5</th>\n",
" <th>6</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0</td>\n",
" <td>30</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>5.0</td>\n",
" <td>20</td>\n",
" <td>4.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>60</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>4.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>10</td>\n",
" <td>40</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>3.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>60</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>2.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>70</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>2.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>20</td>\n",
" <td>40</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>5.0</td>\n",
" <td>20</td>\n",
" <td>4.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>70</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>4.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
Slides added
2020-05-21 16:20:12 +02:00
" 0 1 2 3 4 5 6\n",
"0 0 30 5.0 20 4.0 60 4.0\n",
"1 10 40 3.0 60 2.0 70 2.0\n",
"2 20 40 5.0 20 4.0 70 4.0"
Codes added
2020-05-21 13:42:50 +02:00
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Estimations:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>user</th>\n",
" <th>item</th>\n",
" <th>est_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0</td>\n",
" <td>60</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>4.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>10</td>\n",
" <td>40</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>3.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>20</td>\n",
" <td>0</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>3.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>20</td>\n",
" <td>20</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>4.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>20</td>\n",
" <td>70</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>4.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" user item est_score\n",
Slides added
2020-05-21 16:20:12 +02:00
"0 0 60 4.0\n",
"1 10 40 3.0\n",
"2 20 0 3.0\n",
"3 20 20 4.0\n",
"4 20 70 4.0"
Codes added
2020-05-21 13:42:50 +02:00
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import evaluation_measures as ev\n",
"import imp\n",
"import helpers\n",
"imp.reload(ev)\n",
"\n",
"dir_path=\"Recommendations generated/toy-example/\"\n",
"super_reactions=[4,5]\n",
"test=pd.read_csv('./Datasets/toy-example/test.csv', sep='\\t', header=None)\n",
"\n",
"display(ev.evaluate_all(test, dir_path, super_reactions, topK=3))\n",
"#also you can just type ev.evaluate_all() - I put above values as default\n",
"\n",
"toy_train_read=pd.read_csv('./Datasets/toy-example/train.csv', sep='\\t', header=None, names=['user', 'item', 'rating', 'timestamp'])\n",
"toy_test_read=pd.read_csv('./Datasets/toy-example/test.csv', sep='\\t', header=None, names=['user', 'item', 'rating', 'timestamp'])\n",
"reco=pd.read_csv('Recommendations generated/toy-example/Self_BaselineUI_reco.csv', header=None)\n",
"estimations=pd.read_csv('Recommendations generated/toy-example/Self_BaselineUI_estimations.csv', names=['user', 'item', 'est_score'])\n",
"toy_train_ui, toy_test_ui, toy_user_code_id, toy_user_id_code, \\\n",
"toy_item_code_id, toy_item_id_code = helpers.data_to_csr(toy_train_read, toy_test_read)\n",
"\n",
"print('Training data:')\n",
"display(toy_train_ui.todense())\n",
"\n",
"print('Test data:')\n",
"display(toy_test_ui.todense())\n",
"\n",
"print('Recommendations:')\n",
"display(reco)\n",
"\n",
"print('Estimations:')\n",
"display(estimations)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Sample recommendations"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 20,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Here is what user rated high:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>user</th>\n",
" <th>rating</th>\n",
" <th>title</th>\n",
" <th>genres</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>24754</th>\n",
" <td>105</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>5</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>Gattaca (1997)</td>\n",
" <td>Drama, Sci-Fi, Thriller</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>37073</th>\n",
" <td>105</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>5</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>Contact (1997)</td>\n",
" <td>Drama, Sci-Fi</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>40592</th>\n",
" <td>105</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>5</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>Titanic (1997)</td>\n",
" <td>Action, Drama, Romance</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>46032</th>\n",
" <td>105</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>5</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>L.A. Confidential (1997)</td>\n",
" <td>Crime, Film-Noir, Mystery, Thriller</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>17916</th>\n",
" <td>105</td>\n",
" <td>4</td>\n",
" <td>English Patient, The (1996)</td>\n",
" <td>Drama, Romance, War</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>23767</th>\n",
" <td>105</td>\n",
" <td>4</td>\n",
" <td>Chasing Amy (1997)</td>\n",
" <td>Drama, Romance</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>47003</th>\n",
" <td>105</td>\n",
" <td>4</td>\n",
" <td>Cop Land (1997)</td>\n",
" <td>Crime, Drama, Mystery</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>48891</th>\n",
" <td>105</td>\n",
" <td>4</td>\n",
" <td>Lost Highway (1997)</td>\n",
" <td>Mystery</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>67355</th>\n",
" <td>105</td>\n",
" <td>4</td>\n",
" <td>Good Will Hunting (1997)</td>\n",
" <td>Drama</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>54121</th>\n",
" <td>105</td>\n",
" <td>3</td>\n",
" <td>Boogie Nights (1997)</td>\n",
" <td>Drama</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>55439</th>\n",
" <td>105</td>\n",
" <td>3</td>\n",
" <td>Seven Years in Tibet (1997)</td>\n",
" <td>Drama, War</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>69687</th>\n",
" <td>105</td>\n",
" <td>3</td>\n",
" <td>Wag the Dog (1997)</td>\n",
" <td>Comedy, Drama</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>7321</th>\n",
" <td>105</td>\n",
" <td>2</td>\n",
" <td>Saint, The (1997)</td>\n",
" <td>Action, Romance, Thriller</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>45056</th>\n",
" <td>105</td>\n",
" <td>2</td>\n",
" <td>Tomorrow Never Dies (1997)</td>\n",
" <td>Action, Romance, Thriller</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>64052</th>\n",
" <td>105</td>\n",
" <td>2</td>\n",
" <td>Alien: Resurrection (1997)</td>\n",
" <td>Action, Horror, Sci-Fi</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
Slides added
2020-05-21 16:20:12 +02:00
" user rating title \\\n",
"24754 105 5 Gattaca (1997) \n",
"37073 105 5 Contact (1997) \n",
"40592 105 5 Titanic (1997) \n",
"46032 105 5 L.A. Confidential (1997) \n",
"17916 105 4 English Patient, The (1996) \n",
"23767 105 4 Chasing Amy (1997) \n",
"47003 105 4 Cop Land (1997) \n",
"48891 105 4 Lost Highway (1997) \n",
"67355 105 4 Good Will Hunting (1997) \n",
"54121 105 3 Boogie Nights (1997) \n",
"55439 105 3 Seven Years in Tibet (1997) \n",
"69687 105 3 Wag the Dog (1997) \n",
"7321 105 2 Saint, The (1997) \n",
"45056 105 2 Tomorrow Never Dies (1997) \n",
"64052 105 2 Alien: Resurrection (1997) \n",
Codes added
2020-05-21 13:42:50 +02:00
"\n",
Slides added
2020-05-21 16:20:12 +02:00
" genres \n",
"24754 Drama, Sci-Fi, Thriller \n",
"37073 Drama, Sci-Fi \n",
"40592 Action, Drama, Romance \n",
"46032 Crime, Film-Noir, Mystery, Thriller \n",
"17916 Drama, Romance, War \n",
"23767 Drama, Romance \n",
"47003 Crime, Drama, Mystery \n",
"48891 Mystery \n",
"67355 Drama \n",
"54121 Drama \n",
"55439 Drama, War \n",
"69687 Comedy, Drama \n",
"7321 Action, Romance, Thriller \n",
"45056 Action, Romance, Thriller \n",
"64052 Action, Horror, Sci-Fi "
Codes added
2020-05-21 13:42:50 +02:00
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Here is what we recommend:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>user</th>\n",
" <th>rec_nb</th>\n",
" <th>title</th>\n",
" <th>genres</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>103</th>\n",
" <td>105.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>1</td>\n",
" <td>Great Day in Harlem, A (1994)</td>\n",
" <td>Documentary</td>\n",
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>1046</th>\n",
" <td>105.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>2</td>\n",
" <td>Tough and Deadly (1995)</td>\n",
" <td>Action, Drama, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>1988</th>\n",
" <td>105.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>3</td>\n",
" <td>Aiqing wansui (1994)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>2930</th>\n",
" <td>105.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>4</td>\n",
" <td>Delta of Venus (1994)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>3872</th>\n",
" <td>105.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>5</td>\n",
" <td>Someone Else's America (1995)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>4814</th>\n",
" <td>105.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>6</td>\n",
" <td>Saint of Fort Washington, The (1993)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>5755</th>\n",
" <td>105.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>7</td>\n",
" <td>Celestial Clockwork (1994)</td>\n",
" <td>Comedy</td>\n",
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>6698</th>\n",
" <td>105.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>8</td>\n",
" <td>Some Mother's Son (1996)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>8592</th>\n",
" <td>105.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>9</td>\n",
" <td>Maya Lin: A Strong Clear Vision (1994)</td>\n",
" <td>Documentary</td>\n",
" </tr>\n",
" <tr>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <th>7638</th>\n",
" <td>105.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>10</td>\n",
" <td>Prefontaine (1997)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" user rec_nb title \\\n",
Slides added
2020-05-21 16:20:12 +02:00
"103 105.0 1 Great Day in Harlem, A (1994) \n",
"1046 105.0 2 Tough and Deadly (1995) \n",
"1988 105.0 3 Aiqing wansui (1994) \n",
"2930 105.0 4 Delta of Venus (1994) \n",
"3872 105.0 5 Someone Else's America (1995) \n",
"4814 105.0 6 Saint of Fort Washington, The (1993) \n",
"5755 105.0 7 Celestial Clockwork (1994) \n",
"6698 105.0 8 Some Mother's Son (1996) \n",
"8592 105.0 9 Maya Lin: A Strong Clear Vision (1994) \n",
"7638 105.0 10 Prefontaine (1997) \n",
Codes added
2020-05-21 13:42:50 +02:00
"\n",
" genres \n",
Slides added
2020-05-21 16:20:12 +02:00
"103 Documentary \n",
"1046 Action, Drama, Thriller \n",
"1988 Drama \n",
"2930 Drama \n",
"3872 Drama \n",
"4814 Drama \n",
"5755 Comedy \n",
"6698 Drama \n",
"8592 Documentary \n",
"7638 Drama "
Codes added
2020-05-21 13:42:50 +02:00
]
},
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 20,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"train=pd.read_csv('./Datasets/ml-100k/train.csv', sep='\\t', header=None, names=['user', 'item', 'rating', 'timestamp'])\n",
"items=pd.read_csv('./Datasets/ml-100k/movies.csv')\n",
"\n",
"user=random.choice(list(set(train['user'])))\n",
"\n",
"train_content=pd.merge(train, items, left_on='item', right_on='id')\n",
"\n",
"print('Here is what user rated high:')\n",
"display(train_content[train_content['user']==user][['user', 'rating', 'title', 'genres']]\\\n",
" .sort_values(by='rating', ascending=False)[:15])\n",
"\n",
"reco = np.loadtxt('Recommendations generated/ml-100k/Self_BaselineUI_reco.csv', delimiter=',')\n",
"items=pd.read_csv('./Datasets/ml-100k/movies.csv')\n",
"\n",
"# Let's ignore scores - they are not used in evaluation: \n",
"reco_users=reco[:,:1]\n",
"reco_items=reco[:,1::2]\n",
"# Let's put them into one array\n",
"reco=np.concatenate((reco_users, reco_items), axis=1)\n",
"\n",
"# Let's rebuild it user-item dataframe\n",
"recommended=[]\n",
"for row in reco:\n",
" for rec_nb, entry in enumerate(row[1:]):\n",
" recommended.append((row[0], rec_nb+1, entry))\n",
"recommended=pd.DataFrame(recommended, columns=['user','rec_nb', 'item'])\n",
"\n",
"recommended_content=pd.merge(recommended, items, left_on='item', right_on='id')\n",
"\n",
"print('Here is what we recommend:')\n",
"recommended_content[recommended_content['user']==user][['user', 'rec_nb', 'title', 'genres']].sort_values(by='rec_nb')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# project task 3: implement some other evaluation measure"
]
},
{
"cell_type": "code",
"execution_count": 71,
"metadata": {},
"outputs": [],
"source": [
"# it may be your idea, modification of what we have already implemented \n",
"# (for example Hit2 rate which would count as a success users whoreceived at least 2 relevant recommendations) \n",
"# or something well-known\n",
"# expected output: modification of evaluation_measures.py such that evaluate_all will also display your measure"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 21,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
Slides added
2020-05-21 16:20:12 +02:00
"943it [00:00, 9023.32it/s]\n",
"943it [00:00, 9004.98it/s]\n",
"943it [00:00, 8532.40it/s]\n",
"943it [00:00, 8974.06it/s]\n",
"943it [00:00, 8818.01it/s]\n"
Codes added
2020-05-21 13:42:50 +02:00
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_TopPop</td>\n",
" <td>2.508258</td>\n",
" <td>2.217909</td>\n",
" <td>0.188865</td>\n",
" <td>0.116919</td>\n",
" <td>0.118732</td>\n",
" <td>0.141584</td>\n",
" <td>0.130472</td>\n",
" <td>0.137473</td>\n",
" <td>0.214651</td>\n",
" <td>0.111707</td>\n",
" <td>0.400939</td>\n",
" <td>0.555546</td>\n",
" <td>0.765642</td>\n",
" <td>1.000000</td>\n",
" <td>0.038961</td>\n",
" <td>3.159079</td>\n",
" <td>0.987317</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Baseline</td>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" <td>0.091410</td>\n",
" <td>0.037652</td>\n",
" <td>0.046030</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
" <td>1.000000</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_GlobalAvg</td>\n",
" <td>1.125760</td>\n",
" <td>0.943534</td>\n",
" <td>0.061188</td>\n",
" <td>0.025968</td>\n",
" <td>0.031383</td>\n",
" <td>0.041343</td>\n",
" <td>0.040558</td>\n",
" <td>0.032107</td>\n",
" <td>0.067695</td>\n",
" <td>0.027470</td>\n",
" <td>0.171187</td>\n",
" <td>0.509546</td>\n",
" <td>0.384942</td>\n",
" <td>1.000000</td>\n",
" <td>0.025974</td>\n",
" <td>2.711772</td>\n",
" <td>0.992003</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Random</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>1.518964</td>\n",
" <td>1.222159</td>\n",
" <td>0.046554</td>\n",
" <td>0.020603</td>\n",
" <td>0.023679</td>\n",
" <td>0.031216</td>\n",
" <td>0.028970</td>\n",
" <td>0.021179</td>\n",
" <td>0.050489</td>\n",
" <td>0.019185</td>\n",
" <td>0.123856</td>\n",
" <td>0.506812</td>\n",
" <td>0.322375</td>\n",
" <td>0.987805</td>\n",
" <td>0.184704</td>\n",
" <td>5.103172</td>\n",
" <td>0.906873</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineUI</td>\n",
" <td>0.967585</td>\n",
" <td>0.762740</td>\n",
" <td>0.000954</td>\n",
" <td>0.000170</td>\n",
" <td>0.000278</td>\n",
" <td>0.000463</td>\n",
" <td>0.000644</td>\n",
" <td>0.000189</td>\n",
" <td>0.000752</td>\n",
" <td>0.000168</td>\n",
" <td>0.001677</td>\n",
" <td>0.496424</td>\n",
" <td>0.009544</td>\n",
" <td>0.600530</td>\n",
" <td>0.005051</td>\n",
" <td>1.803126</td>\n",
" <td>0.996380</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
Slides added
2020-05-21 16:20:12 +02:00
" Model RMSE MAE precision recall F_1 \\\n",
"0 Self_TopPop 2.508258 2.217909 0.188865 0.116919 0.118732 \n",
"0 Ready_Baseline 0.949459 0.752487 0.091410 0.037652 0.046030 \n",
"0 Self_GlobalAvg 1.125760 0.943534 0.061188 0.025968 0.031383 \n",
"0 Ready_Random 1.518964 1.222159 0.046554 0.020603 0.023679 \n",
"0 Self_BaselineUI 0.967585 0.762740 0.000954 0.000170 0.000278 \n",
Codes added
2020-05-21 13:42:50 +02:00
"\n",
" F_05 precision_super recall_super NDCG mAP MRR \\\n",
"0 0.141584 0.130472 0.137473 0.214651 0.111707 0.400939 \n",
"0 0.061286 0.079614 0.056463 0.095957 0.043178 0.198193 \n",
"0 0.041343 0.040558 0.032107 0.067695 0.027470 0.171187 \n",
Slides added
2020-05-21 16:20:12 +02:00
"0 0.031216 0.028970 0.021179 0.050489 0.019185 0.123856 \n",
Codes added
2020-05-21 13:42:50 +02:00
"0 0.000463 0.000644 0.000189 0.000752 0.000168 0.001677 \n",
"\n",
" LAUC HR Reco in test Test coverage Shannon Gini \n",
"0 0.555546 0.765642 1.000000 0.038961 3.159079 0.987317 \n",
"0 0.515501 0.437964 1.000000 0.033911 2.836513 0.991139 \n",
"0 0.509546 0.384942 1.000000 0.025974 2.711772 0.992003 \n",
Slides added
2020-05-21 16:20:12 +02:00
"0 0.506812 0.322375 0.987805 0.184704 5.103172 0.906873 \n",
"0 0.496424 0.009544 0.600530 0.005051 1.803126 0.996380 "
Codes added
2020-05-21 13:42:50 +02:00
]
},
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 21,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dir_path=\"Recommendations generated/ml-100k/\"\n",
"super_reactions=[4,5]\n",
"test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None)\n",
"\n",
"ev.evaluate_all(test, dir_path, super_reactions)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.9"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Reference in New Issue Copy Permalink