s406917/workshops_recommender_systems
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
c331c3100c
workshops_recommender_systems/P2. Evaluation.ipynb

1977 lines
65 KiB
Plaintext
Raw Normal View History

Codes added
2020-05-21 13:42:50 +02:00
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Prepare test set"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 1,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {
"slideshow": {
"slide_type": "-"
}
},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import scipy.sparse as sparse\n",
"from collections import defaultdict\n",
"from itertools import chain\n",
"import random\n",
"from tqdm import tqdm\n",
"\n",
"# In evaluation we do not load train set - it is not needed\n",
"test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None)\n",
"test.columns=['user', 'item', 'rating', 'timestamp']\n",
"\n",
"test['user_code'] = test['user'].astype(\"category\").cat.codes\n",
"test['item_code'] = test['item'].astype(\"category\").cat.codes\n",
"\n",
"user_code_id = dict(enumerate(test['user'].astype(\"category\").cat.categories))\n",
"user_id_code = dict((v, k) for k, v in user_code_id.items())\n",
"item_code_id = dict(enumerate(test['item'].astype(\"category\").cat.categories))\n",
"item_id_code = dict((v, k) for k, v in item_code_id.items())\n",
"\n",
"test_ui = sparse.csr_matrix((test['rating'], (test['user_code'], test['item_code'])))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Estimations metrics"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 2,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [],
"source": [
"estimations_df=pd.read_csv('Recommendations generated/ml-100k/Ready_Baseline_estimations.csv', header=None)\n",
"estimations_df.columns=['user', 'item' ,'score']\n",
"\n",
"estimations_df['user_code']=[user_id_code[user] for user in estimations_df['user']]\n",
"estimations_df['item_code']=[item_id_code[item] for item in estimations_df['item']]\n",
"estimations=sparse.csr_matrix((estimations_df['score'], (estimations_df['user_code'], estimations_df['item_code'])), shape=test_ui.shape)"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 3,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [],
"source": [
"def estimations_metrics(test_ui, estimations):\n",
" result=[]\n",
"\n",
" RMSE=(np.sum((estimations.data-test_ui.data)**2)/estimations.nnz)**(1/2)\n",
" result.append(['RMSE', RMSE])\n",
"\n",
" MAE=np.sum(abs(estimations.data-test_ui.data))/estimations.nnz\n",
" result.append(['MAE', MAE])\n",
" \n",
" df_result=(pd.DataFrame(list(zip(*result))[1])).T\n",
" df_result.columns=list(zip(*result))[0]\n",
" return df_result"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 4,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" RMSE MAE\n",
"0 0.949459 0.752487"
]
},
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 4,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
Slides added
2020-05-21 16:20:12 +02:00
"# in case of error (in the laboratories) you might have to switch to the other version of pandas\n",
"# try !pip3 install pandas=='1.0.3' (or pip if you use python 2) and restart the kernel\n",
"\n",
Codes added
2020-05-21 13:42:50 +02:00
"estimations_metrics(test_ui, estimations)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Ranking metrics"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 5,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[663, 475, 62, ..., 472, 269, 503],\n",
" [ 48, 313, 475, ..., 591, 175, 466],\n",
" [351, 313, 475, ..., 591, 175, 466],\n",
" ...,\n",
" [259, 313, 475, ..., 11, 591, 175],\n",
" [ 33, 313, 475, ..., 11, 591, 175],\n",
" [ 77, 313, 475, ..., 11, 591, 175]])"
]
},
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 5,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import numpy as np\n",
"reco = np.loadtxt('Recommendations generated/ml-100k/Ready_Baseline_reco.csv', delimiter=',')\n",
"# Let's ignore scores - they are not used in evaluation: \n",
"users=reco[:,:1]\n",
"items=reco[:,1::2]\n",
"# Let's use inner ids instead of real ones\n",
"users=np.vectorize(lambda x: user_id_code.setdefault(x, -1))(users)\n",
"items=np.vectorize(lambda x: item_id_code.setdefault(x, -1))(items) # maybe items we recommend are not in test set\n",
"# Let's put them into one array\n",
"reco=np.concatenate((users, items), axis=1)\n",
"reco"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 6,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [],
"source": [
"def ranking_metrics(test_ui, reco, super_reactions=[], topK=10):\n",
" \n",
" nb_items=test_ui.shape[1]\n",
" relevant_users, super_relevant_users, prec, rec, F_1, F_05, prec_super, rec_super, ndcg, mAP, MRR, LAUC, HR=\\\n",
" 0,0,0,0,0,0,0,0,0,0,0,0,0\n",
" \n",
" cg = (1.0 / np.log2(np.arange(2, topK + 2)))\n",
" cg_sum = np.cumsum(cg)\n",
" \n",
" for (nb_user, user) in tqdm(enumerate(reco[:,0])):\n",
" u_rated_items=test_ui.indices[test_ui.indptr[user]:test_ui.indptr[user+1]]\n",
" nb_u_rated_items=len(u_rated_items)\n",
" if nb_u_rated_items>0: # skip users with no items in test set (still possible that there will be no super items)\n",
" relevant_users+=1\n",
" \n",
" u_super_items=u_rated_items[np.vectorize(lambda x: x in super_reactions)\\\n",
" (test_ui.data[test_ui.indptr[user]:test_ui.indptr[user+1]])]\n",
" # more natural seems u_super_items=[item for item in u_rated_items if test_ui[user,item] in super_reactions]\n",
" # but accesing test_ui[user,item] is expensive -we should avoid doing it\n",
" if len(u_super_items)>0:\n",
" super_relevant_users+=1\n",
" \n",
" user_successes=np.zeros(topK)\n",
" nb_user_successes=0\n",
" user_super_successes=np.zeros(topK)\n",
" nb_user_super_successes=0\n",
" \n",
" # evaluation\n",
" for (item_position,item) in enumerate(reco[nb_user,1:topK+1]):\n",
" if item in u_rated_items:\n",
" user_successes[item_position]=1\n",
" nb_user_successes+=1\n",
" if item in u_super_items:\n",
" user_super_successes[item_position]=1\n",
" nb_user_super_successes+=1\n",
" \n",
" prec_u=nb_user_successes/topK \n",
" prec+=prec_u\n",
" \n",
" rec_u=nb_user_successes/nb_u_rated_items\n",
" rec+=rec_u\n",
" \n",
" F_1+=2*(prec_u*rec_u)/(prec_u+rec_u) if prec_u+rec_u>0 else 0\n",
" F_05+=(0.5**2+1)*(prec_u*rec_u)/(0.5**2*prec_u+rec_u) if prec_u+rec_u>0 else 0\n",
" \n",
" prec_super+=nb_user_super_successes/topK\n",
" rec_super+=nb_user_super_successes/max(len(u_super_items),1) # to set 0 if no super items\n",
" ndcg+=np.dot(user_successes,cg)/cg_sum[min(topK, nb_u_rated_items)-1]\n",
" \n",
" cumsum_successes=np.cumsum(user_successes)\n",
" mAP+=np.dot(cumsum_successes/np.arange(1,topK+1), user_successes)/min(topK, nb_u_rated_items)\n",
" MRR+=1/(user_successes.nonzero()[0][0]+1) if user_successes.nonzero()[0].size>0 else 0\n",
" LAUC+=(np.dot(cumsum_successes, 1-user_successes)+\\\n",
" (nb_user_successes+nb_u_rated_items)/2*((nb_items-nb_u_rated_items)-(topK-nb_user_successes)))/\\\n",
" ((nb_items-nb_u_rated_items)*nb_u_rated_items)\n",
" \n",
" HR+=nb_user_successes>0\n",
" \n",
" \n",
" result=[]\n",
" result.append(('precision', prec/relevant_users))\n",
" result.append(('recall', rec/relevant_users))\n",
" result.append(('F_1', F_1/relevant_users))\n",
" result.append(('F_05', F_05/relevant_users))\n",
" result.append(('precision_super', prec_super/super_relevant_users))\n",
" result.append(('recall_super', rec_super/super_relevant_users))\n",
" result.append(('NDCG', ndcg/relevant_users))\n",
" result.append(('mAP', mAP/relevant_users))\n",
" result.append(('MRR', MRR/relevant_users))\n",
" result.append(('LAUC', LAUC/relevant_users))\n",
" result.append(('HR', HR/relevant_users))\n",
"\n",
" df_result=(pd.DataFrame(list(zip(*result))[1])).T\n",
" df_result.columns=list(zip(*result))[0]\n",
" return df_result"
]
},
{
"cell_type": "code",
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 7,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
projects done
2020-06-08 17:39:37 +02:00
"943it [00:00, 3056.45it/s]\n"
Codes added
2020-05-21 13:42:50 +02:00
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.09141</td>\n",
" <td>0.037652</td>\n",
" <td>0.04603</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" precision recall F_1 F_05 precision_super recall_super \\\n",
"0 0.09141 0.037652 0.04603 0.061286 0.079614 0.056463 \n",
"\n",
" NDCG mAP MRR LAUC HR \n",
"0 0.095957 0.043178 0.198193 0.515501 0.437964 "
]
},
Slides added
2020-05-21 16:20:12 +02:00
"execution_count": 7,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ranking_metrics(test_ui, reco, super_reactions=[4,5], topK=10)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Diversity metrics"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"def diversity_metrics(test_ui, reco, topK=10):\n",
" \n",
" frequencies=defaultdict(int)\n",
" \n",
" # let's assign 0 to all items in test set\n",
" for item in list(set(test_ui.indices)):\n",
" frequencies[item]=0\n",
" \n",
" # counting frequencies\n",
" for item in reco[:,1:].flat:\n",
" frequencies[item]+=1\n",
" \n",
" nb_reco_outside_test=frequencies[-1]\n",
" del frequencies[-1]\n",
" \n",
" frequencies=np.array(list(frequencies.values()))\n",
" \n",
" nb_rec_items=len(frequencies[frequencies>0])\n",
" nb_reco_inside_test=np.sum(frequencies)\n",
" \n",
" frequencies=frequencies/np.sum(frequencies)\n",
" frequencies=np.sort(frequencies)\n",
" \n",
" with np.errstate(divide='ignore'): # let's put zeros put items with 0 frequency and ignore division warning\n",
" log_frequencies=np.nan_to_num(np.log(frequencies), posinf=0, neginf=0)\n",
" \n",
" result=[]\n",
" result.append(('Reco in test', nb_reco_inside_test/(nb_reco_inside_test+nb_reco_outside_test)))\n",
" result.append(('Test coverage', nb_rec_items/test_ui.shape[1]))\n",
" result.append(('Shannon', -np.dot(frequencies, log_frequencies)))\n",
" result.append(('Gini', np.dot(frequencies, np.arange(1-len(frequencies), len(frequencies), 2))/(len(frequencies)-1)))\n",
" \n",
" df_result=(pd.DataFrame(list(zip(*result))[1])).T\n",
" df_result.columns=list(zip(*result))[0]\n",
" return df_result"
]
},
{
"cell_type": "code",
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 9,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1.0</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Reco in test Test coverage Shannon Gini\n",
"0 1.0 0.033911 2.836513 0.991139"
]
},
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 9,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
Slides added
2020-05-21 16:20:12 +02:00
"# in case of errors try !pip3 install numpy==1.18.4 (or pip if you use python 2) and restart the kernel\n",
"\n",
Codes added
2020-05-21 13:42:50 +02:00
"import evaluation_measures as ev\n",
"import imp\n",
"imp.reload(ev)\n",
"\n",
"x=diversity_metrics(test_ui, reco, topK=10)\n",
"x"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# To be used in other notebooks"
]
},
{
"cell_type": "code",
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 10,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
projects done
2020-06-08 17:39:37 +02:00
"943it [00:00, 2872.87it/s]\n"
Codes added
2020-05-21 13:42:50 +02:00
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" <td>0.09141</td>\n",
" <td>0.037652</td>\n",
" <td>0.04603</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
" <td>1.0</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" RMSE MAE precision recall F_1 F_05 \\\n",
"0 0.949459 0.752487 0.09141 0.037652 0.04603 0.061286 \n",
"\n",
" precision_super recall_super NDCG mAP MRR LAUC \\\n",
"0 0.079614 0.056463 0.095957 0.043178 0.198193 0.515501 \n",
"\n",
" HR Reco in test Test coverage Shannon Gini \n",
"0 0.437964 1.0 0.033911 2.836513 0.991139 "
]
},
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 10,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import evaluation_measures as ev\n",
"import imp\n",
"imp.reload(ev)\n",
"\n",
"estimations_df=pd.read_csv('Recommendations generated/ml-100k/Ready_Baseline_estimations.csv', header=None)\n",
"reco=np.loadtxt('Recommendations generated/ml-100k/Ready_Baseline_reco.csv', delimiter=',')\n",
"\n",
"ev.evaluate(test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None),\n",
" estimations_df=estimations_df, \n",
" reco=reco,\n",
" super_reactions=[4,5])\n",
"#also you can just type ev.evaluate_all(estimations_df, reco) - I put above values as default"
]
},
{
"cell_type": "code",
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 11,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
projects done
2020-06-08 17:39:37 +02:00
"943it [00:00, 3486.43it/s]\n",
"943it [00:00, 4513.34it/s]\n",
"943it [00:00, 3772.94it/s]\n",
"943it [00:00, 2882.13it/s]\n",
"943it [00:00, 2333.14it/s]\n",
"943it [00:00, 2868.64it/s]\n",
"943it [00:00, 3744.73it/s]\n"
Codes added
2020-05-21 13:42:50 +02:00
]
}
],
"source": [
"import evaluation_measures as ev\n",
"import imp\n",
"imp.reload(ev)\n",
"\n",
"dir_path=\"Recommendations generated/ml-100k/\"\n",
"super_reactions=[4,5]\n",
"test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None)\n",
"\n",
"df=ev.evaluate_all(test, dir_path, super_reactions)\n",
"#also you can just type ev.evaluate_all() - I put above values as default"
]
},
{
"cell_type": "code",
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 12,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_TopPop</td>\n",
" <td>2.508258</td>\n",
" <td>2.217909</td>\n",
" <td>0.188865</td>\n",
" <td>0.116919</td>\n",
" <td>0.118732</td>\n",
" <td>0.141584</td>\n",
" <td>0.130472</td>\n",
" <td>0.137473</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Baseline</td>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" <td>0.091410</td>\n",
" <td>0.037652</td>\n",
" <td>0.046030</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_GlobalAvg</td>\n",
" <td>1.125760</td>\n",
" <td>0.943534</td>\n",
" <td>0.061188</td>\n",
" <td>0.025968</td>\n",
" <td>0.031383</td>\n",
" <td>0.041343</td>\n",
" <td>0.040558</td>\n",
" <td>0.032107</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Random</td>\n",
projects done
2020-06-08 17:39:37 +02:00
" <td>1.517593</td>\n",
" <td>1.220181</td>\n",
" <td>0.046023</td>\n",
" <td>0.019038</td>\n",
" <td>0.023118</td>\n",
" <td>0.030734</td>\n",
" <td>0.029292</td>\n",
" <td>0.021639</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_TopRated</td>\n",
" <td>2.508258</td>\n",
" <td>2.217909</td>\n",
" <td>0.000954</td>\n",
" <td>0.000188</td>\n",
" <td>0.000298</td>\n",
" <td>0.000481</td>\n",
" <td>0.000644</td>\n",
" <td>0.000223</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineIU</td>\n",
" <td>0.958136</td>\n",
" <td>0.754051</td>\n",
" <td>0.000954</td>\n",
" <td>0.000188</td>\n",
" <td>0.000298</td>\n",
" <td>0.000481</td>\n",
" <td>0.000644</td>\n",
" <td>0.000223</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineUI</td>\n",
" <td>0.967585</td>\n",
" <td>0.762740</td>\n",
" <td>0.000954</td>\n",
" <td>0.000170</td>\n",
" <td>0.000278</td>\n",
" <td>0.000463</td>\n",
" <td>0.000644</td>\n",
" <td>0.000189</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
Slides added
2020-05-21 16:20:12 +02:00
" Model RMSE MAE precision recall F_1 \\\n",
"0 Self_TopPop 2.508258 2.217909 0.188865 0.116919 0.118732 \n",
"0 Ready_Baseline 0.949459 0.752487 0.091410 0.037652 0.046030 \n",
"0 Self_GlobalAvg 1.125760 0.943534 0.061188 0.025968 0.031383 \n",
projects done
2020-06-08 17:39:37 +02:00
"0 Ready_Random 1.517593 1.220181 0.046023 0.019038 0.023118 \n",
"0 Self_TopRated 2.508258 2.217909 0.000954 0.000188 0.000298 \n",
"0 Self_BaselineIU 0.958136 0.754051 0.000954 0.000188 0.000298 \n",
Slides added
2020-05-21 16:20:12 +02:00
"0 Self_BaselineUI 0.967585 0.762740 0.000954 0.000170 0.000278 \n",
Codes added
2020-05-21 13:42:50 +02:00
"\n",
" F_05 precision_super recall_super \n",
"0 0.141584 0.130472 0.137473 \n",
"0 0.061286 0.079614 0.056463 \n",
"0 0.041343 0.040558 0.032107 \n",
projects done
2020-06-08 17:39:37 +02:00
"0 0.030734 0.029292 0.021639 \n",
"0 0.000481 0.000644 0.000223 \n",
"0 0.000481 0.000644 0.000223 \n",
Slides added
2020-05-21 16:20:12 +02:00
"0 0.000463 0.000644 0.000189 "
Codes added
2020-05-21 13:42:50 +02:00
]
},
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 12,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.iloc[:,:9]"
]
},
{
"cell_type": "code",
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 13,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_TopPop</td>\n",
" <td>0.214651</td>\n",
" <td>0.111707</td>\n",
" <td>0.400939</td>\n",
" <td>0.555546</td>\n",
" <td>0.765642</td>\n",
" <td>1.000000</td>\n",
" <td>0.038961</td>\n",
" <td>3.159079</td>\n",
" <td>0.987317</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Baseline</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
" <td>1.000000</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_GlobalAvg</td>\n",
" <td>0.067695</td>\n",
" <td>0.027470</td>\n",
" <td>0.171187</td>\n",
" <td>0.509546</td>\n",
" <td>0.384942</td>\n",
" <td>1.000000</td>\n",
" <td>0.025974</td>\n",
" <td>2.711772</td>\n",
" <td>0.992003</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Random</td>\n",
projects done
2020-06-08 17:39:37 +02:00
" <td>0.050818</td>\n",
" <td>0.019958</td>\n",
" <td>0.126646</td>\n",
" <td>0.506031</td>\n",
" <td>0.305408</td>\n",
" <td>0.988547</td>\n",
" <td>0.174603</td>\n",
" <td>5.082383</td>\n",
" <td>0.908434</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_TopRated</td>\n",
" <td>0.001043</td>\n",
" <td>0.000335</td>\n",
" <td>0.003348</td>\n",
" <td>0.496433</td>\n",
" <td>0.009544</td>\n",
" <td>0.699046</td>\n",
" <td>0.005051</td>\n",
" <td>1.945910</td>\n",
" <td>0.995669</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineIU</td>\n",
" <td>0.001043</td>\n",
" <td>0.000335</td>\n",
" <td>0.003348</td>\n",
" <td>0.496433</td>\n",
" <td>0.009544</td>\n",
" <td>0.699046</td>\n",
" <td>0.005051</td>\n",
" <td>1.945910</td>\n",
" <td>0.995669</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineUI</td>\n",
" <td>0.000752</td>\n",
" <td>0.000168</td>\n",
" <td>0.001677</td>\n",
" <td>0.496424</td>\n",
" <td>0.009544</td>\n",
" <td>0.600530</td>\n",
" <td>0.005051</td>\n",
" <td>1.803126</td>\n",
" <td>0.996380</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
Slides added
2020-05-21 16:20:12 +02:00
" Model NDCG mAP MRR LAUC HR \\\n",
"0 Self_TopPop 0.214651 0.111707 0.400939 0.555546 0.765642 \n",
"0 Ready_Baseline 0.095957 0.043178 0.198193 0.515501 0.437964 \n",
"0 Self_GlobalAvg 0.067695 0.027470 0.171187 0.509546 0.384942 \n",
projects done
2020-06-08 17:39:37 +02:00
"0 Ready_Random 0.050818 0.019958 0.126646 0.506031 0.305408 \n",
"0 Self_TopRated 0.001043 0.000335 0.003348 0.496433 0.009544 \n",
"0 Self_BaselineIU 0.001043 0.000335 0.003348 0.496433 0.009544 \n",
Slides added
2020-05-21 16:20:12 +02:00
"0 Self_BaselineUI 0.000752 0.000168 0.001677 0.496424 0.009544 \n",
Codes added
2020-05-21 13:42:50 +02:00
"\n",
" Reco in test Test coverage Shannon Gini \n",
"0 1.000000 0.038961 3.159079 0.987317 \n",
"0 1.000000 0.033911 2.836513 0.991139 \n",
"0 1.000000 0.025974 2.711772 0.992003 \n",
projects done
2020-06-08 17:39:37 +02:00
"0 0.988547 0.174603 5.082383 0.908434 \n",
"0 0.699046 0.005051 1.945910 0.995669 \n",
"0 0.699046 0.005051 1.945910 0.995669 \n",
Slides added
2020-05-21 16:20:12 +02:00
"0 0.600530 0.005051 1.803126 0.996380 "
Codes added
2020-05-21 13:42:50 +02:00
]
},
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 13,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.iloc[:,np.append(0,np.arange(9, df.shape[1]))]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Check metrics on toy dataset"
]
},
{
"cell_type": "code",
projects done
2020-06-08 17:39:37 +02:00
"execution_count": 17,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
projects done
2020-06-08 17:39:37 +02:00
"3it [00:00, 3003.08it/s]\n",
"3it [00:00, 1975.65it/s]\n"
Codes added
2020-05-21 13:42:50 +02:00
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>HR2</th>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineUI</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>1.612452</td>\n",
projects done
2020-06-08 17:39:37 +02:00
" <td>1.400</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>0.444444</td>\n",
" <td>0.888889</td>\n",
" <td>0.555556</td>\n",
" <td>0.478632</td>\n",
" <td>0.333333</td>\n",
" <td>0.75</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>0.676907</td>\n",
" <td>0.574074</td>\n",
" <td>0.611111</td>\n",
" <td>0.638889</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>1.0</td>\n",
projects done
2020-06-08 17:39:37 +02:00
" <td>0.333333</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>0.888889</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>0.8</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>1.386294</td>\n",
projects done
2020-06-08 17:39:37 +02:00
" <td>0.250000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineIU</td>\n",
" <td>1.648337</td>\n",
" <td>1.575</td>\n",
" <td>0.444444</td>\n",
" <td>0.888889</td>\n",
" <td>0.555556</td>\n",
" <td>0.478632</td>\n",
" <td>0.333333</td>\n",
" <td>0.75</td>\n",
" <td>0.720550</td>\n",
" <td>0.629630</td>\n",
" <td>0.666667</td>\n",
" <td>0.722222</td>\n",
" <td>1.0</td>\n",
" <td>0.333333</td>\n",
" <td>0.777778</td>\n",
" <td>0.8</td>\n",
" <td>1.351784</td>\n",
" <td>0.357143</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
projects done
2020-06-08 17:39:37 +02:00
" Model RMSE MAE precision recall F_1 F_05 \\\n",
"0 Self_BaselineUI 1.612452 1.400 0.444444 0.888889 0.555556 0.478632 \n",
"0 Self_BaselineIU 1.648337 1.575 0.444444 0.888889 0.555556 0.478632 \n",
Codes added
2020-05-21 13:42:50 +02:00
"\n",
Slides added
2020-05-21 16:20:12 +02:00
" precision_super recall_super NDCG mAP MRR LAUC HR \\\n",
"0 0.333333 0.75 0.676907 0.574074 0.611111 0.638889 1.0 \n",
projects done
2020-06-08 17:39:37 +02:00
"0 0.333333 0.75 0.720550 0.629630 0.666667 0.722222 1.0 \n",
Codes added
2020-05-21 13:42:50 +02:00
"\n",
projects done
2020-06-08 17:39:37 +02:00
" HR2 Reco in test Test coverage Shannon Gini \n",
"0 0.333333 0.888889 0.8 1.386294 0.250000 \n",
"0 0.333333 0.777778 0.8 1.351784 0.357143 "
Codes added
2020-05-21 13:42:50 +02:00
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Training data:\n"
]
},
{
"data": {
"text/plain": [
"matrix([[3, 4, 0, 0, 5, 0, 0, 4],\n",
" [0, 1, 2, 3, 0, 0, 0, 0],\n",
" [0, 0, 0, 5, 0, 3, 4, 0]], dtype=int64)"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Test data:\n"
]
},
{
"data": {
"text/plain": [
"matrix([[0, 0, 0, 0, 0, 0, 3, 0],\n",
" [0, 0, 0, 0, 5, 0, 0, 0],\n",
" [5, 0, 4, 0, 0, 0, 0, 2]], dtype=int64)"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Recommendations:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>0</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" <th>3</th>\n",
" <th>4</th>\n",
" <th>5</th>\n",
" <th>6</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0</td>\n",
" <td>30</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>5.0</td>\n",
" <td>20</td>\n",
" <td>4.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>60</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>4.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>10</td>\n",
" <td>40</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>3.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>60</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>2.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>70</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>2.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>20</td>\n",
" <td>40</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>5.0</td>\n",
" <td>20</td>\n",
" <td>4.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>70</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>4.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
Slides added
2020-05-21 16:20:12 +02:00
" 0 1 2 3 4 5 6\n",
"0 0 30 5.0 20 4.0 60 4.0\n",
"1 10 40 3.0 60 2.0 70 2.0\n",
"2 20 40 5.0 20 4.0 70 4.0"
Codes added
2020-05-21 13:42:50 +02:00
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Estimations:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>user</th>\n",
" <th>item</th>\n",
" <th>est_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0</td>\n",
" <td>60</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>4.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>10</td>\n",
" <td>40</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>3.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>20</td>\n",
" <td>0</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>3.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>20</td>\n",
" <td>20</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>4.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>20</td>\n",
" <td>70</td>\n",
Slides added
2020-05-21 16:20:12 +02:00
" <td>4.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" user item est_score\n",
Slides added
2020-05-21 16:20:12 +02:00
"0 0 60 4.0\n",
"1 10 40 3.0\n",
"2 20 0 3.0\n",
"3 20 20 4.0\n",
"4 20 70 4.0"
Codes added
2020-05-21 13:42:50 +02:00
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import evaluation_measures as ev\n",
"import imp\n",
"import helpers\n",
"imp.reload(ev)\n",
"\n",
"dir_path=\"Recommendations generated/toy-example/\"\n",
"super_reactions=[4,5]\n",
"test=pd.read_csv('./Datasets/toy-example/test.csv', sep='\\t', header=None)\n",
"\n",
"display(ev.evaluate_all(test, dir_path, super_reactions, topK=3))\n",
"#also you can just type ev.evaluate_all() - I put above values as default\n",
"\n",
"toy_train_read=pd.read_csv('./Datasets/toy-example/train.csv', sep='\\t', header=None, names=['user', 'item', 'rating', 'timestamp'])\n",
"toy_test_read=pd.read_csv('./Datasets/toy-example/test.csv', sep='\\t', header=None, names=['user', 'item', 'rating', 'timestamp'])\n",
"reco=pd.read_csv('Recommendations generated/toy-example/Self_BaselineUI_reco.csv', header=None)\n",
"estimations=pd.read_csv('Recommendations generated/toy-example/Self_BaselineUI_estimations.csv', names=['user', 'item', 'est_score'])\n",
"toy_train_ui, toy_test_ui, toy_user_code_id, toy_user_id_code, \\\n",
"toy_item_code_id, toy_item_id_code = helpers.data_to_csr(toy_train_read, toy_test_read)\n",
"\n",
"print('Training data:')\n",
"display(toy_train_ui.todense())\n",
"\n",
"print('Test data:')\n",
"display(toy_test_ui.todense())\n",
"\n",
"print('Recommendations:')\n",
"display(reco)\n",
"\n",
"print('Estimations:')\n",
"display(estimations)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Sample recommendations"
]
},
{
"cell_type": "code",
projects done
2020-06-08 17:39:37 +02:00
"execution_count": 18,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Here is what user rated high:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>user</th>\n",
" <th>rating</th>\n",
" <th>title</th>\n",
" <th>genres</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>404</th>\n",
" <td>395</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>5</td>\n",
projects done
2020-06-08 17:39:37 +02:00
" <td>Toy Story (1995)</td>\n",
" <td>Animation, Children's, Comedy</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>14150</th>\n",
" <td>395</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>5</td>\n",
projects done
2020-06-08 17:39:37 +02:00
" <td>Powder (1995)</td>\n",
" <td>Drama</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>64079</th>\n",
" <td>395</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>5</td>\n",
projects done
2020-06-08 17:39:37 +02:00
" <td>Alien: Resurrection (1997)</td>\n",
" <td>Action, Horror, Sci-Fi</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>43644</th>\n",
" <td>395</td>\n",
" <td>5</td>\n",
" <td>Amistad (1997)</td>\n",
" <td>Drama</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>40896</th>\n",
" <td>395</td>\n",
" <td>5</td>\n",
" <td>Miracle on 34th Street (1994)</td>\n",
" <td>Drama</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>36088</th>\n",
" <td>395</td>\n",
" <td>5</td>\n",
" <td>Silence of the Lambs, The (1991)</td>\n",
" <td>Drama, Thriller</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>31848</th>\n",
" <td>395</td>\n",
" <td>5</td>\n",
" <td>Shawshank Redemption, The (1994)</td>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" <td>Drama</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>27211</th>\n",
" <td>395</td>\n",
" <td>5</td>\n",
" <td>Empire Strikes Back, The (1980)</td>\n",
" <td>Action, Adventure, Drama, Romance, Sci-Fi, War</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>21870</th>\n",
" <td>395</td>\n",
" <td>5</td>\n",
" <td>Men in Black (1997)</td>\n",
" <td>Action, Adventure, Comedy, Sci-Fi</td>\n",
Changes in slides
2020-05-21 22:53:34 +02:00
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>20257</th>\n",
" <td>395</td>\n",
" <td>5</td>\n",
" <td>Return of the Jedi (1983)</td>\n",
" <td>Action, Adventure, Romance, Sci-Fi, War</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>16682</th>\n",
" <td>395</td>\n",
" <td>5</td>\n",
" <td>Blues Brothers, The (1980)</td>\n",
" <td>Action, Comedy, Musical</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>15962</th>\n",
" <td>395</td>\n",
" <td>5</td>\n",
" <td>Godfather, The (1972)</td>\n",
" <td>Action, Crime, Drama</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>15416</th>\n",
" <td>395</td>\n",
" <td>5</td>\n",
" <td>Blade Runner (1982)</td>\n",
" <td>Film-Noir, Sci-Fi</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>15047</th>\n",
" <td>395</td>\n",
" <td>5</td>\n",
" <td>Indiana Jones and the Last Crusade (1989)</td>\n",
" <td>Action, Adventure</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>23025</th>\n",
" <td>395</td>\n",
" <td>5</td>\n",
" <td>Apt Pupil (1998)</td>\n",
" <td>Drama, Thriller</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
projects done
2020-06-08 17:39:37 +02:00
" user rating title \\\n",
"404 395 5 Toy Story (1995) \n",
"14150 395 5 Powder (1995) \n",
"64079 395 5 Alien: Resurrection (1997) \n",
"43644 395 5 Amistad (1997) \n",
"40896 395 5 Miracle on 34th Street (1994) \n",
"36088 395 5 Silence of the Lambs, The (1991) \n",
"31848 395 5 Shawshank Redemption, The (1994) \n",
"27211 395 5 Empire Strikes Back, The (1980) \n",
"21870 395 5 Men in Black (1997) \n",
"20257 395 5 Return of the Jedi (1983) \n",
"16682 395 5 Blues Brothers, The (1980) \n",
"15962 395 5 Godfather, The (1972) \n",
"15416 395 5 Blade Runner (1982) \n",
"15047 395 5 Indiana Jones and the Last Crusade (1989) \n",
"23025 395 5 Apt Pupil (1998) \n",
Codes added
2020-05-21 13:42:50 +02:00
"\n",
projects done
2020-06-08 17:39:37 +02:00
" genres \n",
"404 Animation, Children's, Comedy \n",
"14150 Drama \n",
"64079 Action, Horror, Sci-Fi \n",
"43644 Drama \n",
"40896 Drama \n",
"36088 Drama, Thriller \n",
"31848 Drama \n",
"27211 Action, Adventure, Drama, Romance, Sci-Fi, War \n",
"21870 Action, Adventure, Comedy, Sci-Fi \n",
"20257 Action, Adventure, Romance, Sci-Fi, War \n",
"16682 Action, Comedy, Musical \n",
"15962 Action, Crime, Drama \n",
"15416 Film-Noir, Sci-Fi \n",
"15047 Action, Adventure \n",
"23025 Drama, Thriller "
Codes added
2020-05-21 13:42:50 +02:00
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Here is what we recommend:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>user</th>\n",
" <th>rec_nb</th>\n",
" <th>title</th>\n",
" <th>genres</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>393</th>\n",
" <td>395.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>1</td>\n",
" <td>Great Day in Harlem, A (1994)</td>\n",
" <td>Documentary</td>\n",
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>1335</th>\n",
" <td>395.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>2</td>\n",
" <td>Tough and Deadly (1995)</td>\n",
" <td>Action, Drama, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>2277</th>\n",
" <td>395.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>3</td>\n",
" <td>Aiqing wansui (1994)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>3219</th>\n",
" <td>395.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>4</td>\n",
" <td>Delta of Venus (1994)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>4162</th>\n",
" <td>395.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>5</td>\n",
" <td>Someone Else's America (1995)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>5103</th>\n",
" <td>395.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>6</td>\n",
" <td>Saint of Fort Washington, The (1993)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>6045</th>\n",
" <td>395.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>7</td>\n",
" <td>Celestial Clockwork (1994)</td>\n",
" <td>Comedy</td>\n",
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>6988</th>\n",
" <td>395.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>8</td>\n",
" <td>Some Mother's Son (1996)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>8882</th>\n",
" <td>395.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>9</td>\n",
" <td>Maya Lin: A Strong Clear Vision (1994)</td>\n",
" <td>Documentary</td>\n",
" </tr>\n",
" <tr>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>7928</th>\n",
" <td>395.0</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>10</td>\n",
" <td>Prefontaine (1997)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" user rec_nb title \\\n",
projects done
2020-06-08 17:39:37 +02:00
"393 395.0 1 Great Day in Harlem, A (1994) \n",
"1335 395.0 2 Tough and Deadly (1995) \n",
"2277 395.0 3 Aiqing wansui (1994) \n",
"3219 395.0 4 Delta of Venus (1994) \n",
"4162 395.0 5 Someone Else's America (1995) \n",
"5103 395.0 6 Saint of Fort Washington, The (1993) \n",
"6045 395.0 7 Celestial Clockwork (1994) \n",
"6988 395.0 8 Some Mother's Son (1996) \n",
"8882 395.0 9 Maya Lin: A Strong Clear Vision (1994) \n",
"7928 395.0 10 Prefontaine (1997) \n",
Codes added
2020-05-21 13:42:50 +02:00
"\n",
" genres \n",
projects done
2020-06-08 17:39:37 +02:00
"393 Documentary \n",
"1335 Action, Drama, Thriller \n",
"2277 Drama \n",
"3219 Drama \n",
"4162 Drama \n",
"5103 Drama \n",
"6045 Comedy \n",
"6988 Drama \n",
"8882 Documentary \n",
"7928 Drama "
Codes added
2020-05-21 13:42:50 +02:00
]
},
projects done
2020-06-08 17:39:37 +02:00
"execution_count": 18,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"train=pd.read_csv('./Datasets/ml-100k/train.csv', sep='\\t', header=None, names=['user', 'item', 'rating', 'timestamp'])\n",
"items=pd.read_csv('./Datasets/ml-100k/movies.csv')\n",
"\n",
"user=random.choice(list(set(train['user'])))\n",
"\n",
"train_content=pd.merge(train, items, left_on='item', right_on='id')\n",
"\n",
"print('Here is what user rated high:')\n",
"display(train_content[train_content['user']==user][['user', 'rating', 'title', 'genres']]\\\n",
" .sort_values(by='rating', ascending=False)[:15])\n",
"\n",
"reco = np.loadtxt('Recommendations generated/ml-100k/Self_BaselineUI_reco.csv', delimiter=',')\n",
"items=pd.read_csv('./Datasets/ml-100k/movies.csv')\n",
"\n",
"# Let's ignore scores - they are not used in evaluation: \n",
"reco_users=reco[:,:1]\n",
"reco_items=reco[:,1::2]\n",
"# Let's put them into one array\n",
"reco=np.concatenate((reco_users, reco_items), axis=1)\n",
"\n",
"# Let's rebuild it user-item dataframe\n",
"recommended=[]\n",
"for row in reco:\n",
" for rec_nb, entry in enumerate(row[1:]):\n",
" recommended.append((row[0], rec_nb+1, entry))\n",
"recommended=pd.DataFrame(recommended, columns=['user','rec_nb', 'item'])\n",
"\n",
"recommended_content=pd.merge(recommended, items, left_on='item', right_on='id')\n",
"\n",
"print('Here is what we recommend:')\n",
"recommended_content[recommended_content['user']==user][['user', 'rec_nb', 'title', 'genres']].sort_values(by='rec_nb')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# project task 3: implement some other evaluation measure"
]
},
{
"cell_type": "code",
Changes in slides
2020-05-21 22:53:34 +02:00
"execution_count": 16,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [],
"source": [
"# it may be your idea, modification of what we have already implemented \n",
"# (for example Hit2 rate which would count as a success users whoreceived at least 2 relevant recommendations) \n",
"# or something well-known\n",
"# expected output: modification of evaluation_measures.py such that evaluate_all will also display your measure"
]
},
{
"cell_type": "code",
projects done
2020-06-08 17:39:37 +02:00
"execution_count": 19,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
projects done
2020-06-08 17:39:37 +02:00
"943it [00:00, 6823.12it/s]\n",
"943it [00:00, 7031.46it/s]\n",
"943it [00:00, 6757.46it/s]\n",
"943it [00:00, 4266.43it/s]\n",
"943it [00:00, 6333.51it/s]\n",
"943it [00:00, 7252.84it/s]\n",
"943it [00:00, 5832.99it/s]\n"
Codes added
2020-05-21 13:42:50 +02:00
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
projects done
2020-06-08 17:39:37 +02:00
" <th>HR2</th>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_TopPop</td>\n",
" <td>2.508258</td>\n",
" <td>2.217909</td>\n",
" <td>0.188865</td>\n",
" <td>0.116919</td>\n",
" <td>0.118732</td>\n",
" <td>0.141584</td>\n",
" <td>0.130472</td>\n",
" <td>0.137473</td>\n",
" <td>0.214651</td>\n",
" <td>0.111707</td>\n",
" <td>0.400939</td>\n",
" <td>0.555546</td>\n",
" <td>0.765642</td>\n",
projects done
2020-06-08 17:39:37 +02:00
" <td>0.492047</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>1.000000</td>\n",
" <td>0.038961</td>\n",
" <td>3.159079</td>\n",
" <td>0.987317</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Baseline</td>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" <td>0.091410</td>\n",
" <td>0.037652</td>\n",
" <td>0.046030</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
projects done
2020-06-08 17:39:37 +02:00
" <td>0.239661</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>1.000000</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_GlobalAvg</td>\n",
" <td>1.125760</td>\n",
" <td>0.943534</td>\n",
" <td>0.061188</td>\n",
" <td>0.025968</td>\n",
" <td>0.031383</td>\n",
" <td>0.041343</td>\n",
" <td>0.040558</td>\n",
" <td>0.032107</td>\n",
" <td>0.067695</td>\n",
" <td>0.027470</td>\n",
" <td>0.171187</td>\n",
" <td>0.509546</td>\n",
" <td>0.384942</td>\n",
projects done
2020-06-08 17:39:37 +02:00
" <td>0.142100</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>1.000000</td>\n",
" <td>0.025974</td>\n",
" <td>2.711772</td>\n",
" <td>0.992003</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Random</td>\n",
projects done
2020-06-08 17:39:37 +02:00
" <td>1.517593</td>\n",
" <td>1.220181</td>\n",
" <td>0.046023</td>\n",
" <td>0.019038</td>\n",
" <td>0.023118</td>\n",
" <td>0.030734</td>\n",
" <td>0.029292</td>\n",
" <td>0.021639</td>\n",
" <td>0.050818</td>\n",
" <td>0.019958</td>\n",
" <td>0.126646</td>\n",
" <td>0.506031</td>\n",
" <td>0.305408</td>\n",
" <td>0.111347</td>\n",
" <td>0.988547</td>\n",
" <td>0.174603</td>\n",
" <td>5.082383</td>\n",
" <td>0.908434</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_TopRated</td>\n",
" <td>2.508258</td>\n",
" <td>2.217909</td>\n",
" <td>0.000954</td>\n",
" <td>0.000188</td>\n",
" <td>0.000298</td>\n",
" <td>0.000481</td>\n",
" <td>0.000644</td>\n",
" <td>0.000223</td>\n",
" <td>0.001043</td>\n",
" <td>0.000335</td>\n",
" <td>0.003348</td>\n",
" <td>0.496433</td>\n",
" <td>0.009544</td>\n",
" <td>0.000000</td>\n",
" <td>0.699046</td>\n",
" <td>0.005051</td>\n",
" <td>1.945910</td>\n",
" <td>0.995669</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineIU</td>\n",
" <td>0.958136</td>\n",
" <td>0.754051</td>\n",
" <td>0.000954</td>\n",
" <td>0.000188</td>\n",
" <td>0.000298</td>\n",
" <td>0.000481</td>\n",
" <td>0.000644</td>\n",
" <td>0.000223</td>\n",
" <td>0.001043</td>\n",
" <td>0.000335</td>\n",
" <td>0.003348</td>\n",
" <td>0.496433</td>\n",
" <td>0.009544</td>\n",
" <td>0.000000</td>\n",
" <td>0.699046</td>\n",
" <td>0.005051</td>\n",
" <td>1.945910</td>\n",
" <td>0.995669</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineUI</td>\n",
" <td>0.967585</td>\n",
" <td>0.762740</td>\n",
" <td>0.000954</td>\n",
" <td>0.000170</td>\n",
" <td>0.000278</td>\n",
" <td>0.000463</td>\n",
" <td>0.000644</td>\n",
" <td>0.000189</td>\n",
" <td>0.000752</td>\n",
" <td>0.000168</td>\n",
" <td>0.001677</td>\n",
" <td>0.496424</td>\n",
" <td>0.009544</td>\n",
projects done
2020-06-08 17:39:37 +02:00
" <td>0.000000</td>\n",
Codes added
2020-05-21 13:42:50 +02:00
" <td>0.600530</td>\n",
" <td>0.005051</td>\n",
" <td>1.803126</td>\n",
" <td>0.996380</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
Slides added
2020-05-21 16:20:12 +02:00
" Model RMSE MAE precision recall F_1 \\\n",
"0 Self_TopPop 2.508258 2.217909 0.188865 0.116919 0.118732 \n",
"0 Ready_Baseline 0.949459 0.752487 0.091410 0.037652 0.046030 \n",
"0 Self_GlobalAvg 1.125760 0.943534 0.061188 0.025968 0.031383 \n",
projects done
2020-06-08 17:39:37 +02:00
"0 Ready_Random 1.517593 1.220181 0.046023 0.019038 0.023118 \n",
"0 Self_TopRated 2.508258 2.217909 0.000954 0.000188 0.000298 \n",
"0 Self_BaselineIU 0.958136 0.754051 0.000954 0.000188 0.000298 \n",
Slides added
2020-05-21 16:20:12 +02:00
"0 Self_BaselineUI 0.967585 0.762740 0.000954 0.000170 0.000278 \n",
Codes added
2020-05-21 13:42:50 +02:00
"\n",
" F_05 precision_super recall_super NDCG mAP MRR \\\n",
"0 0.141584 0.130472 0.137473 0.214651 0.111707 0.400939 \n",
"0 0.061286 0.079614 0.056463 0.095957 0.043178 0.198193 \n",
"0 0.041343 0.040558 0.032107 0.067695 0.027470 0.171187 \n",
projects done
2020-06-08 17:39:37 +02:00
"0 0.030734 0.029292 0.021639 0.050818 0.019958 0.126646 \n",
"0 0.000481 0.000644 0.000223 0.001043 0.000335 0.003348 \n",
"0 0.000481 0.000644 0.000223 0.001043 0.000335 0.003348 \n",
Codes added
2020-05-21 13:42:50 +02:00
"0 0.000463 0.000644 0.000189 0.000752 0.000168 0.001677 \n",
"\n",
projects done
2020-06-08 17:39:37 +02:00
" LAUC HR HR2 Reco in test Test coverage Shannon \\\n",
"0 0.555546 0.765642 0.492047 1.000000 0.038961 3.159079 \n",
"0 0.515501 0.437964 0.239661 1.000000 0.033911 2.836513 \n",
"0 0.509546 0.384942 0.142100 1.000000 0.025974 2.711772 \n",
"0 0.506031 0.305408 0.111347 0.988547 0.174603 5.082383 \n",
"0 0.496433 0.009544 0.000000 0.699046 0.005051 1.945910 \n",
"0 0.496433 0.009544 0.000000 0.699046 0.005051 1.945910 \n",
"0 0.496424 0.009544 0.000000 0.600530 0.005051 1.803126 \n",
"\n",
" Gini \n",
"0 0.987317 \n",
"0 0.991139 \n",
"0 0.992003 \n",
"0 0.908434 \n",
"0 0.995669 \n",
"0 0.995669 \n",
"0 0.996380 "
Codes added
2020-05-21 13:42:50 +02:00
]
},
projects done
2020-06-08 17:39:37 +02:00
"execution_count": 19,
Codes added
2020-05-21 13:42:50 +02:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dir_path=\"Recommendations generated/ml-100k/\"\n",
"super_reactions=[4,5]\n",
"test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None)\n",
"\n",
"ev.evaluate_all(test, dir_path, super_reactions)"
]
projects done
2020-06-08 17:39:37 +02:00
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
Codes added
2020-05-21 13:42:50 +02:00
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
projects done
2020-06-08 17:39:37 +02:00
"version": "3.7.5"
Codes added
2020-05-21 13:42:50 +02:00
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Reference in New Issue Copy Permalink