s406917/workshops_recommender_systems
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
45e88a01aa
workshops_recommender_systems/P2. Evaluation.ipynb

1815 lines
58 KiB
Plaintext
Raw Normal View History

Codes added
2020-05-21 13:42:50 +02:00
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Prepare test set"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 1,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {
"slideshow": {
"slide_type": "-"
}
},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import scipy.sparse as sparse\n",
"from collections import defaultdict\n",
"from itertools import chain\n",
"import random\n",
"from tqdm import tqdm\n",
"\n",
"# In evaluation we do not load train set - it is not needed\n",
"test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None)\n",
"test.columns=['user', 'item', 'rating', 'timestamp']\n",
"\n",
"test['user_code'] = test['user'].astype(\"category\").cat.codes\n",
"test['item_code'] = test['item'].astype(\"category\").cat.codes\n",
"\n",
"user_code_id = dict(enumerate(test['user'].astype(\"category\").cat.categories))\n",
"user_id_code = dict((v, k) for k, v in user_code_id.items())\n",
"item_code_id = dict(enumerate(test['item'].astype(\"category\").cat.categories))\n",
"item_id_code = dict((v, k) for k, v in item_code_id.items())\n",
"\n",
"test_ui = sparse.csr_matrix((test['rating'], (test['user_code'], test['item_code'])))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Estimations metrics"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 2,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [],
"source": [
"estimations_df=pd.read_csv('Recommendations generated/ml-100k/Ready_Baseline_estimations.csv', header=None)\n",
"estimations_df.columns=['user', 'item' ,'score']\n",
"\n",
"estimations_df['user_code']=[user_id_code[user] for user in estimations_df['user']]\n",
"estimations_df['item_code']=[item_id_code[item] for item in estimations_df['item']]\n",
"estimations=sparse.csr_matrix((estimations_df['score'], (estimations_df['user_code'], estimations_df['item_code'])), shape=test_ui.shape)"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 3,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [],
"source": [
"def estimations_metrics(test_ui, estimations):\n",
" result=[]\n",
"\n",
" RMSE=(np.sum((estimations.data-test_ui.data)**2)/estimations.nnz)**(1/2)\n",
" result.append(['RMSE', RMSE])\n",
"\n",
" MAE=np.sum(abs(estimations.data-test_ui.data))/estimations.nnz\n",
" result.append(['MAE', MAE])\n",
" \n",
" df_result=(pd.DataFrame(list(zip(*result))[1])).T\n",
" df_result.columns=list(zip(*result))[0]\n",
" return df_result"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 4,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" RMSE MAE\n",
"0 0.949459 0.752487"
]
},
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 4,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
Slides added
2020-05-21 16:20:12 +02:00
"# in case of error (in the laboratories) you might have to switch to the other version of pandas\n",
"# try !pip3 install pandas=='1.0.3' (or pip if you use python 2) and restart the kernel\n",
"\n",
Codes added
2020-05-21 13:42:50 +02:00
"estimations_metrics(test_ui, estimations)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Ranking metrics"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 5,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[663, 475, 62, ..., 472, 269, 503],\n",
" [ 48, 313, 475, ..., 591, 175, 466],\n",
" [351, 313, 475, ..., 591, 175, 466],\n",
" ...,\n",
" [259, 313, 475, ..., 11, 591, 175],\n",
" [ 33, 313, 475, ..., 11, 591, 175],\n",
" [ 77, 313, 475, ..., 11, 591, 175]])"
]
},
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 5,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import numpy as np\n",
"reco = np.loadtxt('Recommendations generated/ml-100k/Ready_Baseline_reco.csv', delimiter=',')\n",
"# Let's ignore scores - they are not used in evaluation: \n",
"users=reco[:,:1]\n",
"items=reco[:,1::2]\n",
"# Let's use inner ids instead of real ones\n",
"users=np.vectorize(lambda x: user_id_code.setdefault(x, -1))(users)\n",
"items=np.vectorize(lambda x: item_id_code.setdefault(x, -1))(items) # maybe items we recommend are not in test set\n",
"# Let's put them into one array\n",
"reco=np.concatenate((users, items), axis=1)\n",
"reco"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 6,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [],
"source": [
"def ranking_metrics(test_ui, reco, super_reactions=[], topK=10):\n",
" \n",
" nb_items=test_ui.shape[1]\n",
" relevant_users, super_relevant_users, prec, rec, F_1, F_05, prec_super, rec_super, ndcg, mAP, MRR, LAUC, HR=\\\n",
" 0,0,0,0,0,0,0,0,0,0,0,0,0\n",
" \n",
" cg = (1.0 / np.log2(np.arange(2, topK + 2)))\n",
" cg_sum = np.cumsum(cg)\n",
" \n",
" for (nb_user, user) in tqdm(enumerate(reco[:,0])):\n",
" u_rated_items=test_ui.indices[test_ui.indptr[user]:test_ui.indptr[user+1]]\n",
" nb_u_rated_items=len(u_rated_items)\n",
" if nb_u_rated_items>0: # skip users with no items in test set (still possible that there will be no super items)\n",
" relevant_users+=1\n",
" \n",
" u_super_items=u_rated_items[np.vectorize(lambda x: x in super_reactions)\\\n",
" (test_ui.data[test_ui.indptr[user]:test_ui.indptr[user+1]])]\n",
" # more natural seems u_super_items=[item for item in u_rated_items if test_ui[user,item] in super_reactions]\n",
" # but accesing test_ui[user,item] is expensive -we should avoid doing it\n",
" if len(u_super_items)>0:\n",
" super_relevant_users+=1\n",
" \n",
" user_successes=np.zeros(topK)\n",
" nb_user_successes=0\n",
" user_super_successes=np.zeros(topK)\n",
" nb_user_super_successes=0\n",
" \n",
" # evaluation\n",
" for (item_position,item) in enumerate(reco[nb_user,1:topK+1]):\n",
" if item in u_rated_items:\n",
" user_successes[item_position]=1\n",
" nb_user_successes+=1\n",
" if item in u_super_items:\n",
" user_super_successes[item_position]=1\n",
" nb_user_super_successes+=1\n",
" \n",
" prec_u=nb_user_successes/topK \n",
" prec+=prec_u\n",
" \n",
" rec_u=nb_user_successes/nb_u_rated_items\n",
" rec+=rec_u\n",
" \n",
" F_1+=2*(prec_u*rec_u)/(prec_u+rec_u) if prec_u+rec_u>0 else 0\n",
" F_05+=(0.5**2+1)*(prec_u*rec_u)/(0.5**2*prec_u+rec_u) if prec_u+rec_u>0 else 0\n",
" \n",
" prec_super+=nb_user_super_successes/topK\n",
" rec_super+=nb_user_super_successes/max(len(u_super_items),1) # to set 0 if no super items\n",
" ndcg+=np.dot(user_successes,cg)/cg_sum[min(topK, nb_u_rated_items)-1]\n",
" \n",
" cumsum_successes=np.cumsum(user_successes)\n",
" mAP+=np.dot(cumsum_successes/np.arange(1,topK+1), user_successes)/min(topK, nb_u_rated_items)\n",
" MRR+=1/(user_successes.nonzero()[0][0]+1) if user_successes.nonzero()[0].size>0 else 0\n",
" LAUC+=(np.dot(cumsum_successes, 1-user_successes)+\\\n",
" (nb_user_successes+nb_u_rated_items)/2*((nb_items-nb_u_rated_items)-(topK-nb_user_successes)))/\\\n",
" ((nb_items-nb_u_rated_items)*nb_u_rated_items)\n",
" \n",
" HR+=nb_user_successes>0\n",
" \n",
" \n",
" result=[]\n",
" result.append(('precision', prec/relevant_users))\n",
" result.append(('recall', rec/relevant_users))\n",
" result.append(('F_1', F_1/relevant_users))\n",
" result.append(('F_05', F_05/relevant_users))\n",
" result.append(('precision_super', prec_super/super_relevant_users))\n",
" result.append(('recall_super', rec_super/super_relevant_users))\n",
" result.append(('NDCG', ndcg/relevant_users))\n",
" result.append(('mAP', mAP/relevant_users))\n",
" result.append(('MRR', MRR/relevant_users))\n",
" result.append(('LAUC', LAUC/relevant_users))\n",
" result.append(('HR', HR/relevant_users))\n",
"\n",
" df_result=(pd.DataFrame(list(zip(*result))[1])).T\n",
" df_result.columns=list(zip(*result))[0]\n",
" return df_result"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 7,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
Changes in slides
2020-05-21 22:53:34 +02:00
"943it [00:00, 6311.05it/s]\n"
Codes added
2020-05-21 13:42:50 +02:00
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.09141</td>\n",
" <td>0.037652</td>\n",
" <td>0.04603</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" precision recall F_1 F_05 precision_super recall_super \\\n",
"0 0.09141 0.037652 0.04603 0.061286 0.079614 0.056463 \n",
"\n",
" NDCG mAP MRR LAUC HR \n",
"0 0.095957 0.043178 0.198193 0.515501 0.437964 "
]
},
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 7,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ranking_metrics(test_ui, reco, super_reactions=[4,5], topK=10)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Diversity metrics"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"def diversity_metrics(test_ui, reco, topK=10):\n",
" \n",
" frequencies=defaultdict(int)\n",
" \n",
" # let's assign 0 to all items in test set\n",
" for item in list(set(test_ui.indices)):\n",
" frequencies[item]=0\n",
" \n",
" # counting frequencies\n",
" for item in reco[:,1:].flat:\n",
" frequencies[item]+=1\n",
" \n",
" nb_reco_outside_test=frequencies[-1]\n",
" del frequencies[-1]\n",
" \n",
" frequencies=np.array(list(frequencies.values()))\n",
" \n",
" nb_rec_items=len(frequencies[frequencies>0])\n",
" nb_reco_inside_test=np.sum(frequencies)\n",
" \n",
" frequencies=frequencies/np.sum(frequencies)\n",
" frequencies=np.sort(frequencies)\n",
" \n",
" with np.errstate(divide='ignore'): # let's put zeros put items with 0 frequency and ignore division warning\n",
" log_frequencies=np.nan_to_num(np.log(frequencies), posinf=0, neginf=0)\n",
" \n",
" result=[]\n",
" result.append(('Reco in test', nb_reco_inside_test/(nb_reco_inside_test+nb_reco_outside_test)))\n",
" result.append(('Test coverage', nb_rec_items/test_ui.shape[1]))\n",
" result.append(('Shannon', -np.dot(frequencies, log_frequencies)))\n",
" result.append(('Gini', np.dot(frequencies, np.arange(1-len(frequencies), len(frequencies), 2))/(len(frequencies)-1)))\n",
" \n",
" df_result=(pd.DataFrame(list(zip(*result))[1])).T\n",
" df_result.columns=list(zip(*result))[0]\n",
" return df_result"
]
},
{
"cell_type": "code",
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 9,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1.0</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Reco in test Test coverage Shannon Gini\n",
"0 1.0 0.033911 2.836513 0.991139"
]
},
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 9,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
Slides added
2020-05-21 16:20:12 +02:00
"# in case of errors try !pip3 install numpy==1.18.4 (or pip if you use python 2) and restart the kernel\n",
"\n",
Codes added
2020-05-21 13:42:50 +02:00
"import evaluation_measures as ev\n",
"import imp\n",
"imp.reload(ev)\n",
"\n",
"x=diversity_metrics(test_ui, reco, topK=10)\n",
"x"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# To be used in other notebooks"
]
},
{
"cell_type": "code",
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 10,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
Changes in slides
2020-05-21 22:53:34 +02:00
"943it [00:00, 7479.62it/s]\n"
Codes added
2020-05-21 13:42:50 +02:00
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" <td>0.09141</td>\n",
" <td>0.037652</td>\n",
" <td>0.04603</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
" <td>1.0</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" RMSE MAE precision recall F_1 F_05 \\\n",
"0 0.949459 0.752487 0.09141 0.037652 0.04603 0.061286 \n",
"\n",
" precision_super recall_super NDCG mAP MRR LAUC \\\n",
"0 0.079614 0.056463 0.095957 0.043178 0.198193 0.515501 \n",
"\n",
" HR Reco in test Test coverage Shannon Gini \n",
"0 0.437964 1.0 0.033911 2.836513 0.991139 "
]
},
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 10,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import evaluation_measures as ev\n",
"import imp\n",
"imp.reload(ev)\n",
"\n",
"estimations_df=pd.read_csv('Recommendations generated/ml-100k/Ready_Baseline_estimations.csv', header=None)\n",
"reco=np.loadtxt('Recommendations generated/ml-100k/Ready_Baseline_reco.csv', delimiter=',')\n",
"\n",
"ev.evaluate(test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None),\n",
" estimations_df=estimations_df, \n",
" reco=reco,\n",
" super_reactions=[4,5])\n",
"#also you can just type ev.evaluate_all(estimations_df, reco) - I put above values as default"
]
},
{
"cell_type": "code",
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 11,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
Changes in slides
2020-05-21 22:53:34 +02:00
"943it [00:00, 7637.21it/s]\n",
"943it [00:00, 6445.83it/s]\n",
"943it [00:00, 7342.27it/s]\n",
"943it [00:00, 7577.08it/s]\n",
"943it [00:00, 7295.64it/s]\n"
Codes added
2020-05-21 13:42:50 +02:00
]
}
],
"source": [
"import evaluation_measures as ev\n",
"import imp\n",
"imp.reload(ev)\n",
"\n",
"dir_path=\"Recommendations generated/ml-100k/\"\n",
"super_reactions=[4,5]\n",
"test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None)\n",
"\n",
"df=ev.evaluate_all(test, dir_path, super_reactions)\n",
"#also you can just type ev.evaluate_all() - I put above values as default"
]
},
{
"cell_type": "code",
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 12,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_TopPop</td>\n",
" <td>2.508258</td>\n",
" <td>2.217909</td>\n",
" <td>0.188865</td>\n",
" <td>0.116919</td>\n",
" <td>0.118732</td>\n",
" <td>0.141584</td>\n",
" <td>0.130472</td>\n",
" <td>0.137473</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Baseline</td>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" <td>0.091410</td>\n",
" <td>0.037652</td>\n",
" <td>0.046030</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_GlobalAvg</td>\n",
" <td>1.125760</td>\n",
" <td>0.943534</td>\n",
" <td>0.061188</td>\n",
" <td>0.025968</td>\n",
" <td>0.031383</td>\n",
" <td>0.041343</td>\n",
" <td>0.040558</td>\n",
" <td>0.032107</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Random</td>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <td>1.517787</td>\n",
" <td>1.217953</td>\n",
" <td>0.047826</td>\n",
" <td>0.017861</td>\n",
" <td>0.022711</td>\n",
" <td>0.031080</td>\n",
" <td>0.028219</td>\n",
" <td>0.016982</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineUI</td>\n",
" <td>0.967585</td>\n",
" <td>0.762740</td>\n",
" <td>0.000954</td>\n",
" <td>0.000170</td>\n",
" <td>0.000278</td>\n",
" <td>0.000463</td>\n",
" <td>0.000644</td>\n",
" <td>0.000189</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
Slides added
2020-05-21 16:20:12 +02:00
" Model RMSE MAE precision recall F_1 \\\n",
"0 Self_TopPop 2.508258 2.217909 0.188865 0.116919 0.118732 \n",
"0 Ready_Baseline 0.949459 0.752487 0.091410 0.037652 0.046030 \n",
"0 Self_GlobalAvg 1.125760 0.943534 0.061188 0.025968 0.031383 \n",
Changes in slides
2020-05-21 22:53:34 +02:00
"0 Ready_Random 1.517787 1.217953 0.047826 0.017861 0.022711 \n",
Slides added
2020-05-21 16:20:12 +02:00
"0 Self_BaselineUI 0.967585 0.762740 0.000954 0.000170 0.000278 \n",
Codes added
2020-05-21 13:42:50 +02:00
"\n",
" F_05 precision_super recall_super \n",
"0 0.141584 0.130472 0.137473 \n",
"0 0.061286 0.079614 0.056463 \n",
"0 0.041343 0.040558 0.032107 \n",
Changes in slides
2020-05-21 22:53:34 +02:00
"0 0.031080 0.028219 0.016982 \n",
Slides added
2020-05-21 16:20:12 +02:00
"0 0.000463 0.000644 0.000189 "
Codes added
2020-05-21 13:42:50 +02:00
]
},
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 12,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.iloc[:,:9]"
]
},
{
"cell_type": "code",
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 13,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_TopPop</td>\n",
" <td>0.214651</td>\n",
" <td>0.111707</td>\n",
" <td>0.400939</td>\n",
" <td>0.555546</td>\n",
" <td>0.765642</td>\n",
" <td>1.000000</td>\n",
" <td>0.038961</td>\n",
" <td>3.159079</td>\n",
" <td>0.987317</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Baseline</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
" <td>1.000000</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_GlobalAvg</td>\n",
" <td>0.067695</td>\n",
" <td>0.027470</td>\n",
" <td>0.171187</td>\n",
" <td>0.509546</td>\n",
" <td>0.384942</td>\n",
" <td>1.000000</td>\n",
" <td>0.025974</td>\n",
" <td>2.711772</td>\n",
" <td>0.992003</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Random</td>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <td>0.051154</td>\n",
" <td>0.019551</td>\n",
" <td>0.125693</td>\n",
" <td>0.505448</td>\n",
" <td>0.318134</td>\n",
" <td>0.986426</td>\n",
" <td>0.186869</td>\n",
" <td>5.091730</td>\n",
" <td>0.908288</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineUI</td>\n",
" <td>0.000752</td>\n",
" <td>0.000168</td>\n",
" <td>0.001677</td>\n",
" <td>0.496424</td>\n",
" <td>0.009544</td>\n",
" <td>0.600530</td>\n",
" <td>0.005051</td>\n",
" <td>1.803126</td>\n",
" <td>0.996380</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
Slides added
2020-05-21 16:20:12 +02:00
" Model NDCG mAP MRR LAUC HR \\\n",
"0 Self_TopPop 0.214651 0.111707 0.400939 0.555546 0.765642 \n",
"0 Ready_Baseline 0.095957 0.043178 0.198193 0.515501 0.437964 \n",
"0 Self_GlobalAvg 0.067695 0.027470 0.171187 0.509546 0.384942 \n",
Changes in slides
2020-05-21 22:53:34 +02:00
"0 Ready_Random 0.051154 0.019551 0.125693 0.505448 0.318134 \n",
Slides added
2020-05-21 16:20:12 +02:00
"0 Self_BaselineUI 0.000752 0.000168 0.001677 0.496424 0.009544 \n",
Codes added
2020-05-21 13:42:50 +02:00
"\n",
" Reco in test Test coverage Shannon Gini \n",
"0 1.000000 0.038961 3.159079 0.987317 \n",
"0 1.000000 0.033911 2.836513 0.991139 \n",
"0 1.000000 0.025974 2.711772 0.992003 \n",
Changes in slides
2020-05-21 22:53:34 +02:00
"0 0.986426 0.186869 5.091730 0.908288 \n",
Slides added
2020-05-21 16:20:12 +02:00
"0 0.600530 0.005051 1.803126 0.996380 "
Codes added
2020-05-21 13:42:50 +02:00
]
},
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 13,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.iloc[:,np.append(0,np.arange(9, df.shape[1]))]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Check metrics on toy dataset"
]
},
{
"cell_type": "code",
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 14,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
Changes in slides
2020-05-21 22:53:34 +02:00
"3it [00:00, 2446.61it/s]\n"
Codes added
2020-05-21 13:42:50 +02:00
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineUI</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>1.612452</td>\n",
" <td>1.4</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>0.444444</td>\n",
" <td>0.888889</td>\n",
" <td>0.555556</td>\n",
" <td>0.478632</td>\n",
" <td>0.333333</td>\n",
" <td>0.75</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>0.676907</td>\n",
" <td>0.574074</td>\n",
" <td>0.611111</td>\n",
" <td>0.638889</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>1.0</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>0.888889</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>0.8</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>1.386294</td>\n",
" <td>0.25</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
Slides added
2020-05-21 16:20:12 +02:00
" Model RMSE MAE precision recall F_1 F_05 \\\n",
"0 Self_BaselineUI 1.612452 1.4 0.444444 0.888889 0.555556 0.478632 \n",
Codes added
2020-05-21 13:42:50 +02:00
"\n",
Slides added
2020-05-21 16:20:12 +02:00
" precision_super recall_super NDCG mAP MRR LAUC HR \\\n",
"0 0.333333 0.75 0.676907 0.574074 0.611111 0.638889 1.0 \n",
Codes added
2020-05-21 13:42:50 +02:00
"\n",
Slides added
2020-05-21 16:20:12 +02:00
" Reco in test Test coverage Shannon Gini \n",
"0 0.888889 0.8 1.386294 0.25 "
Codes added
2020-05-21 13:42:50 +02:00
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Training data:\n"
]
},
{
"data": {
"text/plain": [
"matrix([[3, 4, 0, 0, 5, 0, 0, 4],\n",
" [0, 1, 2, 3, 0, 0, 0, 0],\n",
" [0, 0, 0, 5, 0, 3, 4, 0]], dtype=int64)"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Test data:\n"
]
},
{
"data": {
"text/plain": [
"matrix([[0, 0, 0, 0, 0, 0, 3, 0],\n",
" [0, 0, 0, 0, 5, 0, 0, 0],\n",
" [5, 0, 4, 0, 0, 0, 0, 2]], dtype=int64)"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Recommendations:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>0</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" <th>3</th>\n",
" <th>4</th>\n",
" <th>5</th>\n",
" <th>6</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0</td>\n",
" <td>30</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>5.0</td>\n",
" <td>20</td>\n",
" <td>4.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>60</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>4.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>10</td>\n",
" <td>40</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>3.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>60</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>2.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>70</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>2.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>20</td>\n",
" <td>40</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>5.0</td>\n",
" <td>20</td>\n",
" <td>4.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>70</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>4.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
Slides added
2020-05-21 16:20:12 +02:00
" 0 1 2 3 4 5 6\n",
"0 0 30 5.0 20 4.0 60 4.0\n",
"1 10 40 3.0 60 2.0 70 2.0\n",
"2 20 40 5.0 20 4.0 70 4.0"
Codes added
2020-05-21 13:42:50 +02:00
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Estimations:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>user</th>\n",
" <th>item</th>\n",
" <th>est_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0</td>\n",
" <td>60</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>4.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>10</td>\n",
" <td>40</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>3.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>20</td>\n",
" <td>0</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>3.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>20</td>\n",
" <td>20</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>4.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>20</td>\n",
" <td>70</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>4.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" user item est_score\n",
Slides added
2020-05-21 16:20:12 +02:00
"0 0 60 4.0\n",
"1 10 40 3.0\n",
"2 20 0 3.0\n",
"3 20 20 4.0\n",
"4 20 70 4.0"
Codes added
2020-05-21 13:42:50 +02:00
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import evaluation_measures as ev\n",
"import imp\n",
"import helpers\n",
"imp.reload(ev)\n",
"\n",
"dir_path=\"Recommendations generated/toy-example/\"\n",
"super_reactions=[4,5]\n",
"test=pd.read_csv('./Datasets/toy-example/test.csv', sep='\\t', header=None)\n",
"\n",
"display(ev.evaluate_all(test, dir_path, super_reactions, topK=3))\n",
"#also you can just type ev.evaluate_all() - I put above values as default\n",
"\n",
"toy_train_read=pd.read_csv('./Datasets/toy-example/train.csv', sep='\\t', header=None, names=['user', 'item', 'rating', 'timestamp'])\n",
"toy_test_read=pd.read_csv('./Datasets/toy-example/test.csv', sep='\\t', header=None, names=['user', 'item', 'rating', 'timestamp'])\n",
"reco=pd.read_csv('Recommendations generated/toy-example/Self_BaselineUI_reco.csv', header=None)\n",
"estimations=pd.read_csv('Recommendations generated/toy-example/Self_BaselineUI_estimations.csv', names=['user', 'item', 'est_score'])\n",
"toy_train_ui, toy_test_ui, toy_user_code_id, toy_user_id_code, \\\n",
"toy_item_code_id, toy_item_id_code = helpers.data_to_csr(toy_train_read, toy_test_read)\n",
"\n",
"print('Training data:')\n",
"display(toy_train_ui.todense())\n",
"\n",
"print('Test data:')\n",
"display(toy_test_ui.todense())\n",
"\n",
"print('Recommendations:')\n",
"display(reco)\n",
"\n",
"print('Estimations:')\n",
"display(estimations)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Sample recommendations"
]
},
{
"cell_type": "code",
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 15,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Here is what user rated high:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>user</th>\n",
" <th>rating</th>\n",
" <th>title</th>\n",
" <th>genres</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>7917</th>\n",
" <td>100</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>5</td>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <td>As Good As It Gets (1997)</td>\n",
" <td>Comedy, Drama</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>40494</th>\n",
" <td>100</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>5</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>Titanic (1997)</td>\n",
" <td>Action, Drama, Romance</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>23127</th>\n",
" <td>100</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>5</td>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <td>Apt Pupil (1998)</td>\n",
" <td>Drama, Thriller</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>45163</th>\n",
" <td>100</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>4</td>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <td>Tomorrow Never Dies (1997)</td>\n",
" <td>Action, Romance, Thriller</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>28194</th>\n",
" <td>100</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>4</td>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <td>Conspiracy Theory (1997)</td>\n",
" <td>Action, Mystery, Romance, Thriller</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>46108</th>\n",
" <td>100</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>4</td>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <td>L.A. Confidential (1997)</td>\n",
" <td>Crime, Film-Noir, Mystery, Thriller</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>43730</th>\n",
" <td>100</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>4</td>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <td>Amistad (1997)</td>\n",
" <td>Drama</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>42366</th>\n",
" <td>100</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>4</td>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <td>Postman, The (1997)</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>Drama</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>37000</th>\n",
" <td>100</td>\n",
" <td>4</td>\n",
" <td>Contact (1997)</td>\n",
" <td>Drama, Sci-Fi</td>\n",
" </tr>\n",
" <tr>\n",
" <th>66556</th>\n",
" <td>100</td>\n",
" <td>4</td>\n",
" <td>Apostle, The (1997)</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>Drama</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>67009</th>\n",
" <td>100</td>\n",
" <td>4</td>\n",
" <td>Big Bang Theory, The (1994)</td>\n",
" <td>Crime</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>67332</th>\n",
" <td>100</td>\n",
" <td>4</td>\n",
" <td>Good Will Hunting (1997)</td>\n",
" <td>Drama</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>29564</th>\n",
" <td>100</td>\n",
" <td>4</td>\n",
" <td>Kundun (1997)</td>\n",
" <td>Drama</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>26441</th>\n",
" <td>100</td>\n",
" <td>4</td>\n",
" <td>Air Force One (1997)</td>\n",
" <td>Action, Thriller</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>69735</th>\n",
" <td>100</td>\n",
" <td>4</td>\n",
" <td>Wag the Dog (1997)</td>\n",
" <td>Comedy, Drama</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
Slides added
2020-05-21 16:20:12 +02:00
" user rating title \\\n",
Changes in slides
2020-05-21 22:53:34 +02:00
"7917 100 5 As Good As It Gets (1997) \n",
"40494 100 5 Titanic (1997) \n",
"23127 100 5 Apt Pupil (1998) \n",
"45163 100 4 Tomorrow Never Dies (1997) \n",
"28194 100 4 Conspiracy Theory (1997) \n",
"46108 100 4 L.A. Confidential (1997) \n",
"43730 100 4 Amistad (1997) \n",
"42366 100 4 Postman, The (1997) \n",
"37000 100 4 Contact (1997) \n",
"66556 100 4 Apostle, The (1997) \n",
"67009 100 4 Big Bang Theory, The (1994) \n",
"67332 100 4 Good Will Hunting (1997) \n",
"29564 100 4 Kundun (1997) \n",
"26441 100 4 Air Force One (1997) \n",
"69735 100 4 Wag the Dog (1997) \n",
Codes added
2020-05-21 13:42:50 +02:00
"\n",
Slides added
2020-05-21 16:20:12 +02:00
" genres \n",
Changes in slides
2020-05-21 22:53:34 +02:00
"7917 Comedy, Drama \n",
"40494 Action, Drama, Romance \n",
"23127 Drama, Thriller \n",
"45163 Action, Romance, Thriller \n",
"28194 Action, Mystery, Romance, Thriller \n",
"46108 Crime, Film-Noir, Mystery, Thriller \n",
"43730 Drama \n",
"42366 Drama \n",
"37000 Drama, Sci-Fi \n",
"66556 Drama \n",
"67009 Crime \n",
"67332 Drama \n",
"29564 Drama \n",
"26441 Action, Thriller \n",
"69735 Comedy, Drama "
Codes added
2020-05-21 13:42:50 +02:00
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Here is what we recommend:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>user</th>\n",
" <th>rec_nb</th>\n",
" <th>title</th>\n",
" <th>genres</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>98</th>\n",
" <td>100.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>1</td>\n",
" <td>Great Day in Harlem, A (1994)</td>\n",
" <td>Documentary</td>\n",
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>1041</th>\n",
" <td>100.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>2</td>\n",
" <td>Tough and Deadly (1995)</td>\n",
" <td>Action, Drama, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>1983</th>\n",
" <td>100.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>3</td>\n",
" <td>Aiqing wansui (1994)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>2925</th>\n",
" <td>100.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>4</td>\n",
" <td>Delta of Venus (1994)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>3867</th>\n",
" <td>100.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>5</td>\n",
" <td>Someone Else's America (1995)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>4809</th>\n",
" <td>100.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>6</td>\n",
" <td>Saint of Fort Washington, The (1993)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>5750</th>\n",
" <td>100.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>7</td>\n",
" <td>Celestial Clockwork (1994)</td>\n",
" <td>Comedy</td>\n",
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>6693</th>\n",
" <td>100.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>8</td>\n",
" <td>Some Mother's Son (1996)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>8587</th>\n",
" <td>100.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>9</td>\n",
" <td>Maya Lin: A Strong Clear Vision (1994)</td>\n",
" <td>Documentary</td>\n",
" </tr>\n",
" <tr>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <th>7633</th>\n",
" <td>100.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>10</td>\n",
" <td>Prefontaine (1997)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" user rec_nb title \\\n",
Changes in slides
2020-05-21 22:53:34 +02:00
"98 100.0 1 Great Day in Harlem, A (1994) \n",
"1041 100.0 2 Tough and Deadly (1995) \n",
"1983 100.0 3 Aiqing wansui (1994) \n",
"2925 100.0 4 Delta of Venus (1994) \n",
"3867 100.0 5 Someone Else's America (1995) \n",
"4809 100.0 6 Saint of Fort Washington, The (1993) \n",
"5750 100.0 7 Celestial Clockwork (1994) \n",
"6693 100.0 8 Some Mother's Son (1996) \n",
"8587 100.0 9 Maya Lin: A Strong Clear Vision (1994) \n",
"7633 100.0 10 Prefontaine (1997) \n",
Codes added
2020-05-21 13:42:50 +02:00
"\n",
" genres \n",
Changes in slides
2020-05-21 22:53:34 +02:00
"98 Documentary \n",
"1041 Action, Drama, Thriller \n",
"1983 Drama \n",
"2925 Drama \n",
"3867 Drama \n",
"4809 Drama \n",
"5750 Comedy \n",
"6693 Drama \n",
"8587 Documentary \n",
"7633 Drama "
Codes added
2020-05-21 13:42:50 +02:00
]
},
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 15,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"train=pd.read_csv('./Datasets/ml-100k/train.csv', sep='\\t', header=None, names=['user', 'item', 'rating', 'timestamp'])\n",
"items=pd.read_csv('./Datasets/ml-100k/movies.csv')\n",
"\n",
"user=random.choice(list(set(train['user'])))\n",
"\n",
"train_content=pd.merge(train, items, left_on='item', right_on='id')\n",
"\n",
"print('Here is what user rated high:')\n",
"display(train_content[train_content['user']==user][['user', 'rating', 'title', 'genres']]\\\n",
" .sort_values(by='rating', ascending=False)[:15])\n",
"\n",
"reco = np.loadtxt('Recommendations generated/ml-100k/Self_BaselineUI_reco.csv', delimiter=',')\n",
"items=pd.read_csv('./Datasets/ml-100k/movies.csv')\n",
"\n",
"# Let's ignore scores - they are not used in evaluation: \n",
"reco_users=reco[:,:1]\n",
"reco_items=reco[:,1::2]\n",
"# Let's put them into one array\n",
"reco=np.concatenate((reco_users, reco_items), axis=1)\n",
"\n",
"# Let's rebuild it user-item dataframe\n",
"recommended=[]\n",
"for row in reco:\n",
" for rec_nb, entry in enumerate(row[1:]):\n",
" recommended.append((row[0], rec_nb+1, entry))\n",
"recommended=pd.DataFrame(recommended, columns=['user','rec_nb', 'item'])\n",
"\n",
"recommended_content=pd.merge(recommended, items, left_on='item', right_on='id')\n",
"\n",
"print('Here is what we recommend:')\n",
"recommended_content[recommended_content['user']==user][['user', 'rec_nb', 'title', 'genres']].sort_values(by='rec_nb')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# project task 3: implement some other evaluation measure"
]
},
{
"cell_type": "code",
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 16,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [],
"source": [
"# it may be your idea, modification of what we have already implemented \n",
"# (for example Hit2 rate which would count as a success users whoreceived at least 2 relevant recommendations) \n",
"# or something well-known\n",
"# expected output: modification of evaluation_measures.py such that evaluate_all will also display your measure"
]
},
{
"cell_type": "code",
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 17,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
Changes in slides
2020-05-21 22:53:34 +02:00
"943it [00:00, 6211.85it/s]\n",
"943it [00:00, 7728.29it/s]\n",
"943it [00:00, 7446.93it/s]\n",
"943it [00:00, 6379.07it/s]\n",
"943it [00:00, 6994.80it/s]\n"
Codes added
2020-05-21 13:42:50 +02:00
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_TopPop</td>\n",
" <td>2.508258</td>\n",
" <td>2.217909</td>\n",
" <td>0.188865</td>\n",
" <td>0.116919</td>\n",
" <td>0.118732</td>\n",
" <td>0.141584</td>\n",
" <td>0.130472</td>\n",
" <td>0.137473</td>\n",
" <td>0.214651</td>\n",
" <td>0.111707</td>\n",
" <td>0.400939</td>\n",
" <td>0.555546</td>\n",
" <td>0.765642</td>\n",
" <td>1.000000</td>\n",
" <td>0.038961</td>\n",
" <td>3.159079</td>\n",
" <td>0.987317</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Baseline</td>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" <td>0.091410</td>\n",
" <td>0.037652</td>\n",
" <td>0.046030</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
" <td>1.000000</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_GlobalAvg</td>\n",
" <td>1.125760</td>\n",
" <td>0.943534</td>\n",
" <td>0.061188</td>\n",
" <td>0.025968</td>\n",
" <td>0.031383</td>\n",
" <td>0.041343</td>\n",
" <td>0.040558</td>\n",
" <td>0.032107</td>\n",
" <td>0.067695</td>\n",
" <td>0.027470</td>\n",
" <td>0.171187</td>\n",
" <td>0.509546</td>\n",
" <td>0.384942</td>\n",
" <td>1.000000</td>\n",
" <td>0.025974</td>\n",
" <td>2.711772</td>\n",
" <td>0.992003</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Random</td>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <td>1.517787</td>\n",
" <td>1.217953</td>\n",
" <td>0.047826</td>\n",
" <td>0.017861</td>\n",
" <td>0.022711</td>\n",
" <td>0.031080</td>\n",
" <td>0.028219</td>\n",
" <td>0.016982</td>\n",
" <td>0.051154</td>\n",
" <td>0.019551</td>\n",
" <td>0.125693</td>\n",
" <td>0.505448</td>\n",
" <td>0.318134</td>\n",
" <td>0.986426</td>\n",
" <td>0.186869</td>\n",
" <td>5.091730</td>\n",
" <td>0.908288</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineUI</td>\n",
" <td>0.967585</td>\n",
" <td>0.762740</td>\n",
" <td>0.000954</td>\n",
" <td>0.000170</td>\n",
" <td>0.000278</td>\n",
" <td>0.000463</td>\n",
" <td>0.000644</td>\n",
" <td>0.000189</td>\n",
" <td>0.000752</td>\n",
" <td>0.000168</td>\n",
" <td>0.001677</td>\n",
" <td>0.496424</td>\n",
" <td>0.009544</td>\n",
" <td>0.600530</td>\n",
" <td>0.005051</td>\n",
" <td>1.803126</td>\n",
" <td>0.996380</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
Slides added
2020-05-21 16:20:12 +02:00
" Model RMSE MAE precision recall F_1 \\\n",
"0 Self_TopPop 2.508258 2.217909 0.188865 0.116919 0.118732 \n",
"0 Ready_Baseline 0.949459 0.752487 0.091410 0.037652 0.046030 \n",
"0 Self_GlobalAvg 1.125760 0.943534 0.061188 0.025968 0.031383 \n",
Changes in slides
2020-05-21 22:53:34 +02:00
"0 Ready_Random 1.517787 1.217953 0.047826 0.017861 0.022711 \n",
Slides added
2020-05-21 16:20:12 +02:00
"0 Self_BaselineUI 0.967585 0.762740 0.000954 0.000170 0.000278 \n",
Codes added
2020-05-21 13:42:50 +02:00
"\n",
" F_05 precision_super recall_super NDCG mAP MRR \\\n",
"0 0.141584 0.130472 0.137473 0.214651 0.111707 0.400939 \n",
"0 0.061286 0.079614 0.056463 0.095957 0.043178 0.198193 \n",
"0 0.041343 0.040558 0.032107 0.067695 0.027470 0.171187 \n",
Changes in slides
2020-05-21 22:53:34 +02:00
"0 0.031080 0.028219 0.016982 0.051154 0.019551 0.125693 \n",
Codes added
2020-05-21 13:42:50 +02:00
"0 0.000463 0.000644 0.000189 0.000752 0.000168 0.001677 \n",
"\n",
" LAUC HR Reco in test Test coverage Shannon Gini \n",
"0 0.555546 0.765642 1.000000 0.038961 3.159079 0.987317 \n",
"0 0.515501 0.437964 1.000000 0.033911 2.836513 0.991139 \n",
"0 0.509546 0.384942 1.000000 0.025974 2.711772 0.992003 \n",
Changes in slides
2020-05-21 22:53:34 +02:00
"0 0.505448 0.318134 0.986426 0.186869 5.091730 0.908288 \n",
Slides added
2020-05-21 16:20:12 +02:00
"0 0.496424 0.009544 0.600530 0.005051 1.803126 0.996380 "
Codes added
2020-05-21 13:42:50 +02:00
]
},
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 17,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dir_path=\"Recommendations generated/ml-100k/\"\n",
"super_reactions=[4,5]\n",
"test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None)\n",
"\n",
"ev.evaluate_all(test, dir_path, super_reactions)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.9"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Reference in New Issue Copy Permalink