s413941/Uczenie_maszynowe_Systemy_Rekomendacyjne
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
master
Uczenie_maszynowe_Systemy_R.../P2. Evaluation.ipynb

2123 lines
76 KiB
Plaintext
Raw Permalink Normal View History

UMSR
2020-06-14 22:23:50 +02:00
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.9"
},
"colab": {
"name": "P2. Evaluation.ipynb",
"provenance": [],
"collapsed_sections": [],
"toc_visible": true
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "XsGH6668KkEb",
"colab_type": "text"
},
"source": [
"### Prepare test set"
]
},
{
"cell_type": "code",
"metadata": {
"id": "6i5evQN1KkEc",
"colab_type": "code",
"colab": {}
},
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import scipy.sparse as sparse\n",
"from collections import defaultdict\n",
"from itertools import chain\n",
"import random\n",
"from tqdm import tqdm\n",
"\n",
"# In evaluation we do not load train set - it is not needed\n",
"test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None)\n",
"test.columns=['user', 'item', 'rating', 'timestamp']\n",
"\n",
"test['user_code'] = test['user'].astype(\"category\").cat.codes\n",
"test['item_code'] = test['item'].astype(\"category\").cat.codes\n",
"\n",
"user_code_id = dict(enumerate(test['user'].astype(\"category\").cat.categories))\n",
"user_id_code = dict((v, k) for k, v in user_code_id.items())\n",
"item_code_id = dict(enumerate(test['item'].astype(\"category\").cat.categories))\n",
"item_id_code = dict((v, k) for k, v in item_code_id.items())\n",
"\n",
"test_ui = sparse.csr_matrix((test['rating'], (test['user_code'], test['item_code'])))"
],
"execution_count": 0,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "o1f-bErIK3Xx",
"colab_type": "code",
"colab": {}
},
"source": [
"import os\n",
"if not os.path.exists('./Recommendations generated/'):\n",
" os.mkdir('./Recommendations generated/')\n",
" os.mkdir('./Recommendations generated/ml-100k/')\n",
" os.mkdir('./Recommendations generated/toy-example/')"
],
"execution_count": 0,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "e-h95H_wKkEg",
"colab_type": "text"
},
"source": [
"### Estimations metrics"
]
},
{
"cell_type": "code",
"metadata": {
"id": "Pg63ftwyKkEg",
"colab_type": "code",
"colab": {}
},
"source": [
"estimations_df=pd.read_csv('Recommendations generated/ml-100k/Ready_Baseline_estimations.csv', header=None)\n",
"estimations_df.columns=['user', 'item' ,'score']\n",
"\n",
"estimations_df['user_code']=[user_id_code[user] for user in estimations_df['user']]\n",
"estimations_df['item_code']=[item_id_code[item] for item in estimations_df['item']]\n",
"estimations=sparse.csr_matrix((estimations_df['score'], (estimations_df['user_code'], estimations_df['item_code'])), shape=test_ui.shape)"
],
"execution_count": 0,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "g4QKDwfQKkEj",
"colab_type": "code",
"colab": {}
},
"source": [
"def estimations_metrics(test_ui, estimations):\n",
" result=[]\n",
"\n",
" RMSE=(np.sum((estimations.data-test_ui.data)**2)/estimations.nnz)**(1/2)\n",
" result.append(['RMSE', RMSE])\n",
"\n",
" MAE=np.sum(abs(estimations.data-test_ui.data))/estimations.nnz\n",
" result.append(['MAE', MAE])\n",
" \n",
" df_result=(pd.DataFrame(list(zip(*result))[1])).T\n",
" df_result.columns=list(zip(*result))[0]\n",
" return df_result"
],
"execution_count": 0,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "LTDMTTiiKkEl",
"colab_type": "code",
"outputId": "7ebd1465-3dc6-404d-9666-67897e4d2750",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 80
}
},
"source": [
"# in case of error (in the laboratories) you might have to switch to the other version of pandas\n",
"# try !pip3 install pandas=='1.0.3' (or pip if you use python 2) and restart the kernel\n",
"\n",
"estimations_metrics(test_ui, estimations)"
],
"execution_count": 0,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" RMSE MAE\n",
"0 0.949459 0.752487"
]
},
"metadata": {
"tags": []
},
"execution_count": 6
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "rSgzlQ-iKkEp",
"colab_type": "text"
},
"source": [
"### Ranking metrics"
]
},
{
"cell_type": "code",
"metadata": {
"id": "BX6YbfwgKkEp",
"colab_type": "code",
"outputId": "be746705-2a4d-4e53-f616-a3f4876ce68b",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 136
}
},
"source": [
"import numpy as np\n",
"reco = np.loadtxt('Recommendations generated/ml-100k/Ready_Baseline_reco.csv', delimiter=',')\n",
"# Let's ignore scores - they are not used in evaluation: \n",
"users=reco[:,:1]\n",
"items=reco[:,1::2]\n",
"# Let's use inner ids instead of real ones\n",
"users=np.vectorize(lambda x: user_id_code.setdefault(x, -1))(users)\n",
"items=np.vectorize(lambda x: item_id_code.setdefault(x, -1))(items) # maybe items we recommend are not in test set\n",
"# Let's put them into one array\n",
"reco=np.concatenate((users, items), axis=1)\n",
"reco"
],
"execution_count": 0,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"array([[663, 475, 62, ..., 472, 269, 503],\n",
" [ 48, 313, 475, ..., 591, 175, 466],\n",
" [351, 313, 475, ..., 591, 175, 466],\n",
" ...,\n",
" [259, 313, 475, ..., 11, 591, 175],\n",
" [ 33, 313, 475, ..., 11, 591, 175],\n",
" [ 77, 313, 475, ..., 11, 591, 175]])"
]
},
"metadata": {
"tags": []
},
"execution_count": 7
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "NjVfzif8KkEs",
"colab_type": "code",
"colab": {}
},
"source": [
"def ranking_metrics(test_ui, reco, super_reactions=[], topK=10):\n",
" \n",
" nb_items=test_ui.shape[1]\n",
" relevant_users, super_relevant_users, prec, rec, F_1, F_05, prec_super, rec_super, ndcg, mAP, MRR, LAUC, HR=\\\n",
" 0,0,0,0,0,0,0,0,0,0,0,0,0\n",
" \n",
" cg = (1.0 / np.log2(np.arange(2, topK + 2)))\n",
" cg_sum = np.cumsum(cg)\n",
" \n",
" for (nb_user, user) in tqdm(enumerate(reco[:,0])):\n",
" u_rated_items=test_ui.indices[test_ui.indptr[user]:test_ui.indptr[user+1]]\n",
" nb_u_rated_items=len(u_rated_items)\n",
" if nb_u_rated_items>0: # skip users with no items in test set (still possible that there will be no super items)\n",
" relevant_users+=1\n",
" \n",
" u_super_items=u_rated_items[np.vectorize(lambda x: x in super_reactions)\\\n",
" (test_ui.data[test_ui.indptr[user]:test_ui.indptr[user+1]])]\n",
" # more natural seems u_super_items=[item for item in u_rated_items if test_ui[user,item] in super_reactions]\n",
" # but accesing test_ui[user,item] is expensive -we should avoid doing it\n",
" if len(u_super_items)>0:\n",
" super_relevant_users+=1\n",
" \n",
" user_successes=np.zeros(topK)\n",
" nb_user_successes=0\n",
" user_super_successes=np.zeros(topK)\n",
" nb_user_super_successes=0\n",
" \n",
" # evaluation\n",
" for (item_position,item) in enumerate(reco[nb_user,1:topK+1]):\n",
" if item in u_rated_items:\n",
" user_successes[item_position]=1\n",
" nb_user_successes+=1\n",
" if item in u_super_items:\n",
" user_super_successes[item_position]=1\n",
" nb_user_super_successes+=1\n",
" \n",
" prec_u=nb_user_successes/topK \n",
" prec+=prec_u\n",
" \n",
" rec_u=nb_user_successes/nb_u_rated_items\n",
" rec+=rec_u\n",
" \n",
" F_1+=2*(prec_u*rec_u)/(prec_u+rec_u) if prec_u+rec_u>0 else 0\n",
" F_05+=(0.5**2+1)*(prec_u*rec_u)/(0.5**2*prec_u+rec_u) if prec_u+rec_u>0 else 0\n",
" \n",
" prec_super+=nb_user_super_successes/topK\n",
" rec_super+=nb_user_super_successes/max(len(u_super_items),1) # to set 0 if no super items\n",
" ndcg+=np.dot(user_successes,cg)/cg_sum[min(topK, nb_u_rated_items)-1]\n",
" \n",
" cumsum_successes=np.cumsum(user_successes)\n",
" mAP+=np.dot(cumsum_successes/np.arange(1,topK+1), user_successes)/min(topK, nb_u_rated_items)\n",
" MRR+=1/(user_successes.nonzero()[0][0]+1) if user_successes.nonzero()[0].size>0 else 0\n",
" LAUC+=(np.dot(cumsum_successes, 1-user_successes)+\\\n",
" (nb_user_successes+nb_u_rated_items)/2*((nb_items-nb_u_rated_items)-(topK-nb_user_successes)))/\\\n",
" ((nb_items-nb_u_rated_items)*nb_u_rated_items)\n",
" \n",
" HR+=nb_user_successes>0\n",
" \n",
" \n",
" result=[]\n",
" result.append(('precision', prec/relevant_users))\n",
" result.append(('recall', rec/relevant_users))\n",
" result.append(('F_1', F_1/relevant_users))\n",
" result.append(('F_05', F_05/relevant_users))\n",
" result.append(('precision_super', prec_super/super_relevant_users))\n",
" result.append(('recall_super', rec_super/super_relevant_users))\n",
" result.append(('NDCG', ndcg/relevant_users))\n",
" result.append(('mAP', mAP/relevant_users))\n",
" result.append(('MRR', MRR/relevant_users))\n",
" result.append(('LAUC', LAUC/relevant_users))\n",
" result.append(('HR', HR/relevant_users))\n",
"\n",
" df_result=(pd.DataFrame(list(zip(*result))[1])).T\n",
" df_result.columns=list(zip(*result))[0]\n",
" return df_result"
],
"execution_count": 0,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "xspx_TyWKkEv",
"colab_type": "code",
"outputId": "41145c03-763d-4584-e7d4-f5ec16a5507f",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 97
}
},
"source": [
"ranking_metrics(test_ui, reco, super_reactions=[4,5], topK=10)"
],
"execution_count": 0,
"outputs": [
{
"output_type": "stream",
"text": [
"943it [00:00, 6981.89it/s]\n"
],
"name": "stderr"
},
{
"output_type": "execute_result",
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.09141</td>\n",
" <td>0.037652</td>\n",
" <td>0.04603</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" precision recall F_1 ... MRR LAUC HR\n",
"0 0.09141 0.037652 0.04603 ... 0.198193 0.515501 0.437964\n",
"\n",
"[1 rows x 11 columns]"
]
},
"metadata": {
"tags": []
},
"execution_count": 9
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "sIV-r9MWKkEy",
"colab_type": "text"
},
"source": [
"### Diversity metrics"
]
},
{
"cell_type": "code",
"metadata": {
"id": "XOtNDMiRKkEy",
"colab_type": "code",
"colab": {}
},
"source": [
"def diversity_metrics(test_ui, reco, topK=10):\n",
" \n",
" frequencies=defaultdict(int)\n",
" \n",
" # let's assign 0 to all items in test set\n",
" for item in list(set(test_ui.indices)):\n",
" frequencies[item]=0\n",
" \n",
" # counting frequencies\n",
" for item in reco[:,1:].flat:\n",
" frequencies[item]+=1\n",
" \n",
" nb_reco_outside_test=frequencies[-1]\n",
" del frequencies[-1]\n",
" \n",
" frequencies=np.array(list(frequencies.values()))\n",
" \n",
" nb_rec_items=len(frequencies[frequencies>0])\n",
" nb_reco_inside_test=np.sum(frequencies)\n",
" \n",
" frequencies=frequencies/np.sum(frequencies)\n",
" frequencies=np.sort(frequencies)\n",
" \n",
" with np.errstate(divide='ignore'): # let's put zeros put items with 0 frequency and ignore division warning\n",
" log_frequencies=np.nan_to_num(np.log(frequencies), posinf=0, neginf=0)\n",
" \n",
" result=[]\n",
" result.append(('Reco in test', nb_reco_inside_test/(nb_reco_inside_test+nb_reco_outside_test)))\n",
" result.append(('Test coverage', nb_rec_items/test_ui.shape[1]))\n",
" result.append(('Shannon', -np.dot(frequencies, log_frequencies)))\n",
" result.append(('Gini', np.dot(frequencies, np.arange(1-len(frequencies), len(frequencies), 2))/(len(frequencies)-1)))\n",
" \n",
" df_result=(pd.DataFrame(list(zip(*result))[1])).T\n",
" df_result.columns=list(zip(*result))[0]\n",
" return df_result"
],
"execution_count": 0,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "LmWMC0beKkE0",
"colab_type": "code",
"outputId": "99e50bfb-b99d-4e54-b34e-9726d421f3e0",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 80
}
},
"source": [
"# in case of errors try !pip3 install numpy==1.18.4 (or pip if you use python 2) and restart the kernel\n",
"\n",
"import evaluation_measures as ev\n",
"import imp\n",
"imp.reload(ev)\n",
"\n",
"x=diversity_metrics(test_ui, reco, topK=10)\n",
"x"
],
"execution_count": 0,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1.0</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Reco in test Test coverage Shannon Gini\n",
"0 1.0 0.033911 2.836513 0.991139"
]
},
"metadata": {
"tags": []
},
"execution_count": 13
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "KWtRicX0KkE3",
"colab_type": "text"
},
"source": [
"# To be used in other notebooks"
]
},
{
"cell_type": "code",
"metadata": {
"id": "1ozOsZWMKkE3",
"colab_type": "code",
"outputId": "79b554c9-ec53-4565-f55d-c8343876cb67",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 114
}
},
"source": [
"import evaluation_measures as ev\n",
"import imp\n",
"imp.reload(ev)\n",
"\n",
"estimations_df=pd.read_csv('Recommendations generated/ml-100k/Ready_Baseline_estimations.csv', header=None)\n",
"reco=np.loadtxt('Recommendations generated/ml-100k/Ready_Baseline_reco.csv', delimiter=',')\n",
"\n",
"ev.evaluate(test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None),\n",
" estimations_df=estimations_df, \n",
" reco=reco,\n",
" super_reactions=[4,5])\n",
"#also you can just type ev.evaluate_all(estimations_df, reco) - I put above values as default"
],
"execution_count": 0,
"outputs": [
{
"output_type": "stream",
"text": [
"943it [00:00, 6721.23it/s]\n"
],
"name": "stderr"
},
{
"output_type": "execute_result",
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" <th>HR2</th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" <td>0.09141</td>\n",
" <td>0.037652</td>\n",
" <td>0.04603</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
" <td>0.239661</td>\n",
" <td>1.0</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" RMSE MAE precision ... Test coverage Shannon Gini\n",
"0 0.949459 0.752487 0.09141 ... 0.033911 2.836513 0.991139\n",
"\n",
"[1 rows x 18 columns]"
]
},
"metadata": {
"tags": []
},
"execution_count": 14
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "i1qKPl8LKkE6",
"colab_type": "code",
"outputId": "5f167923-0533-45a7-9536-5cc2e162f79e",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 34
}
},
"source": [
"import evaluation_measures as ev\n",
"import imp\n",
"imp.reload(ev)\n",
"\n",
"dir_path=\"Recommendations generated/ml-100k/\"\n",
"super_reactions=[4,5]\n",
"test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None)\n",
"\n",
"df=ev.evaluate_all(test, dir_path, super_reactions)\n",
"#also you can just type ev.evaluate_all() - I put above values as default"
],
"execution_count": 0,
"outputs": [
{
"output_type": "stream",
"text": [
"943it [00:00, 7140.87it/s]\n"
],
"name": "stderr"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "ADUVNxYnKkE9",
"colab_type": "code",
"outputId": "2d7a1008-8eed-425a-a3a0-4602fe5d5b32",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 80
}
},
"source": [
"df.iloc[:,:9]"
],
"execution_count": 0,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Baseline</td>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" <td>0.09141</td>\n",
" <td>0.037652</td>\n",
" <td>0.04603</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Model RMSE MAE ... F_05 precision_super recall_super\n",
"0 Ready_Baseline 0.949459 0.752487 ... 0.061286 0.079614 0.056463\n",
"\n",
"[1 rows x 9 columns]"
]
},
"metadata": {
"tags": []
},
"execution_count": 16
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "QmTqLVNLKkFA",
"colab_type": "code",
"outputId": "eba7e18f-d73e-4108-a5f5-3fca175db010",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 80
}
},
"source": [
"df.iloc[:,np.append(0,np.arange(9, df.shape[1]))]"
],
"execution_count": 0,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" <th>HR2</th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Baseline</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
" <td>0.239661</td>\n",
" <td>1.0</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Model NDCG mAP ... Test coverage Shannon Gini\n",
"0 Ready_Baseline 0.095957 0.043178 ... 0.033911 2.836513 0.991139\n",
"\n",
"[1 rows x 11 columns]"
]
},
"metadata": {
"tags": []
},
"execution_count": 17
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "ADNCFV1fOzER",
"colab_type": "code",
"outputId": "bd3b8f9b-57e6-4b67-af7c-d7abea1c02b0",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 289
}
},
"source": [
"pip install surprise"
],
"execution_count": 0,
"outputs": [
{
"output_type": "stream",
"text": [
"Collecting surprise\n",
" Downloading https://files.pythonhosted.org/packages/61/de/e5cba8682201fcf9c3719a6fdda95693468ed061945493dea2dd37c5618b/surprise-0.1-py2.py3-none-any.whl\n",
"Collecting scikit-surprise\n",
"\u001b[?25l Downloading https://files.pythonhosted.org/packages/f5/da/b5700d96495fb4f092be497f02492768a3d96a3f4fa2ae7dea46d4081cfa/scikit-surprise-1.1.0.tar.gz (6.4MB)\n",
"\u001b[K |████████████████████████████████| 6.5MB 2.6MB/s \n",
"\u001b[?25hRequirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise->surprise) (0.15.1)\n",
"Requirement already satisfied: numpy>=1.11.2 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise->surprise) (1.18.5)\n",
"Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise->surprise) (1.4.1)\n",
"Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise->surprise) (1.12.0)\n",
"Building wheels for collected packages: scikit-surprise\n",
" Building wheel for scikit-surprise (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
" Created wheel for scikit-surprise: filename=scikit_surprise-1.1.0-cp36-cp36m-linux_x86_64.whl size=1675370 sha256=b9c28146ba19d464e7357de16052301c4261e1e492c668f6ec5ec59683796e4f\n",
" Stored in directory: /root/.cache/pip/wheels/cc/fa/8c/16c93fccce688ae1bde7d979ff102f7bee980d9cfeb8641bcf\n",
"Successfully built scikit-surprise\n",
"Installing collected packages: scikit-surprise, surprise\n",
"Successfully installed scikit-surprise-1.1.0 surprise-0.1\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "r7NacKLTKkFD",
"colab_type": "text"
},
"source": [
"# Check metrics on toy dataset"
]
},
{
"cell_type": "code",
"metadata": {
"id": "ijxxLsVNKkFD",
"colab_type": "code",
"outputId": "0844bd14-84f0-45e1-e07d-1265a2b98b33",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 1000
}
},
"source": [
"import evaluation_measures as ev\n",
"import imp\n",
"import helpers\n",
"imp.reload(ev)\n",
"\n",
"dir_path=\"Recommendations generated/toy-example/\"\n",
"super_reactions=[4,5]\n",
"test=pd.read_csv('./Datasets/toy-example/test.csv', sep='\\t', header=None)\n",
"\n",
"display(ev.evaluate_all(test, dir_path, super_reactions, topK=3))\n",
"#also you can just type ev.evaluate_all() - I put above values as default\n",
"\n",
"toy_train_read=pd.read_csv('./Datasets/toy-example/train.csv', sep='\\t', header=None, names=['user', 'item', 'rating', 'timestamp'])\n",
"toy_test_read=pd.read_csv('./Datasets/toy-example/test.csv', sep='\\t', header=None, names=['user', 'item', 'rating', 'timestamp'])\n",
"reco=pd.read_csv('Recommendations generated/toy-example/Self_BaselineUI_reco.csv', header=None)\n",
"estimations=pd.read_csv('Recommendations generated/toy-example/Self_BaselineUI_estimations.csv', names=['user', 'item', 'est_score'])\n",
"toy_train_ui, toy_test_ui, toy_user_code_id, toy_user_id_code, \\\n",
"toy_item_code_id, toy_item_id_code = helpers.data_to_csr(toy_train_read, toy_test_read)\n",
"\n",
"print('Training data:')\n",
"display(toy_train_ui.todense())\n",
"\n",
"print('Test data:')\n",
"display(toy_test_ui.todense())\n",
"\n",
"print('Recommendations:')\n",
"display(reco)\n",
"\n",
"print('Estimations:')\n",
"display(estimations)"
],
"execution_count": 0,
"outputs": [
{
"output_type": "stream",
"text": [
"943it [00:00, 12096.61it/s]\n"
],
"name": "stderr"
},
{
"output_type": "display_data",
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" <th>HR2</th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineUI</td>\n",
" <td>0.967585</td>\n",
" <td>0.76274</td>\n",
" <td>0.000353</td>\n",
" <td>0.000044</td>\n",
" <td>0.000079</td>\n",
" <td>0.000147</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.000314</td>\n",
" <td>0.000177</td>\n",
" <td>0.00053</td>\n",
" <td>0.498923</td>\n",
" <td>0.00106</td>\n",
" <td>0.0</td>\n",
" <td>0.60053</td>\n",
" <td>0.005051</td>\n",
" <td>1.803126</td>\n",
" <td>0.99638</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Model RMSE MAE ... Test coverage Shannon Gini\n",
"0 Self_BaselineUI 0.967585 0.76274 ... 0.005051 1.803126 0.99638\n",
"\n",
"[1 rows x 19 columns]"
]
},
"metadata": {
"tags": []
}
},
{
"output_type": "stream",
"text": [
"Training data:\n"
],
"name": "stdout"
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"matrix([[5, 3, 4, ..., 0, 0, 0],\n",
" [0, 0, 0, ..., 0, 0, 0],\n",
" [0, 0, 0, ..., 0, 0, 0],\n",
" ...,\n",
" [5, 0, 0, ..., 0, 0, 0],\n",
" [0, 0, 0, ..., 0, 0, 0],\n",
" [0, 0, 0, ..., 0, 0, 0]], dtype=int64)"
]
},
"metadata": {
"tags": []
}
},
{
"output_type": "stream",
"text": [
"Test data:\n"
],
"name": "stdout"
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"matrix([[0, 0, 0, ..., 0, 0, 0],\n",
" [4, 0, 0, ..., 0, 0, 0],\n",
" [0, 0, 0, ..., 0, 0, 0],\n",
" ...,\n",
" [0, 0, 0, ..., 0, 0, 0],\n",
" [0, 0, 0, ..., 0, 0, 0],\n",
" [0, 5, 0, ..., 0, 0, 0]], dtype=int64)"
]
},
"metadata": {
"tags": []
}
},
{
"output_type": "stream",
"text": [
"Recommendations:\n"
],
"name": "stdout"
},
{
"output_type": "display_data",
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>0</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" <th>3</th>\n",
" <th>4</th>\n",
" <th>5</th>\n",
" <th>6</th>\n",
" <th>7</th>\n",
" <th>8</th>\n",
" <th>9</th>\n",
" <th>10</th>\n",
" <th>11</th>\n",
" <th>12</th>\n",
" <th>13</th>\n",
" <th>14</th>\n",
" <th>15</th>\n",
" <th>16</th>\n",
" <th>17</th>\n",
" <th>18</th>\n",
" <th>19</th>\n",
" <th>20</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1</td>\n",
" <td>814</td>\n",
" <td>5.449584</td>\n",
" <td>1491</td>\n",
" <td>5.232561</td>\n",
" <td>1536</td>\n",
" <td>5.188667</td>\n",
" <td>1306</td>\n",
" <td>5.169081</td>\n",
" <td>1599</td>\n",
" <td>4.989625</td>\n",
" <td>1467</td>\n",
" <td>4.931335</td>\n",
" <td>1080</td>\n",
" <td>4.859644</td>\n",
" <td>1642</td>\n",
" <td>4.857250</td>\n",
" <td>1189</td>\n",
" <td>4.782040</td>\n",
" <td>1500</td>\n",
" <td>4.742546</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2</td>\n",
" <td>814</td>\n",
" <td>5.580813</td>\n",
" <td>1491</td>\n",
" <td>5.363789</td>\n",
" <td>1536</td>\n",
" <td>5.319895</td>\n",
" <td>1306</td>\n",
" <td>5.300310</td>\n",
" <td>1599</td>\n",
" <td>5.120854</td>\n",
" <td>1467</td>\n",
" <td>5.062563</td>\n",
" <td>1080</td>\n",
" <td>4.990872</td>\n",
" <td>1642</td>\n",
" <td>4.988479</td>\n",
" <td>119</td>\n",
" <td>4.921909</td>\n",
" <td>1189</td>\n",
" <td>4.913268</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>3</td>\n",
" <td>814</td>\n",
" <td>4.615469</td>\n",
" <td>1491</td>\n",
" <td>4.398445</td>\n",
" <td>1536</td>\n",
" <td>4.354551</td>\n",
" <td>1306</td>\n",
" <td>4.334965</td>\n",
" <td>1599</td>\n",
" <td>4.155510</td>\n",
" <td>1467</td>\n",
" <td>4.097219</td>\n",
" <td>1080</td>\n",
" <td>4.025528</td>\n",
" <td>1642</td>\n",
" <td>4.023134</td>\n",
" <td>119</td>\n",
" <td>3.956565</td>\n",
" <td>1189</td>\n",
" <td>3.947924</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>4</td>\n",
" <td>814</td>\n",
" <td>6.244539</td>\n",
" <td>1491</td>\n",
" <td>6.027515</td>\n",
" <td>1536</td>\n",
" <td>5.983621</td>\n",
" <td>1306</td>\n",
" <td>5.964035</td>\n",
" <td>1599</td>\n",
" <td>5.784579</td>\n",
" <td>1467</td>\n",
" <td>5.726289</td>\n",
" <td>1080</td>\n",
" <td>5.654598</td>\n",
" <td>1642</td>\n",
" <td>5.652204</td>\n",
" <td>119</td>\n",
" <td>5.585635</td>\n",
" <td>1189</td>\n",
" <td>5.576994</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>5</td>\n",
" <td>814</td>\n",
" <td>4.738510</td>\n",
" <td>1491</td>\n",
" <td>4.521486</td>\n",
" <td>1536</td>\n",
" <td>4.477592</td>\n",
" <td>1306</td>\n",
" <td>4.458007</td>\n",
" <td>1599</td>\n",
" <td>4.278551</td>\n",
" <td>1467</td>\n",
" <td>4.220260</td>\n",
" <td>1080</td>\n",
" <td>4.148569</td>\n",
" <td>1642</td>\n",
" <td>4.146176</td>\n",
" <td>119</td>\n",
" <td>4.079607</td>\n",
" <td>1189</td>\n",
" <td>4.070965</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>938</th>\n",
" <td>939</td>\n",
" <td>814</td>\n",
" <td>6.236644</td>\n",
" <td>1491</td>\n",
" <td>6.019620</td>\n",
" <td>1536</td>\n",
" <td>5.975726</td>\n",
" <td>1306</td>\n",
" <td>5.956140</td>\n",
" <td>1599</td>\n",
" <td>5.776685</td>\n",
" <td>1467</td>\n",
" <td>5.718394</td>\n",
" <td>1080</td>\n",
" <td>5.646703</td>\n",
" <td>1642</td>\n",
" <td>5.644310</td>\n",
" <td>119</td>\n",
" <td>5.577740</td>\n",
" <td>1189</td>\n",
" <td>5.569099</td>\n",
" </tr>\n",
" <tr>\n",
" <th>939</th>\n",
" <td>940</td>\n",
" <td>814</td>\n",
" <td>5.462440</td>\n",
" <td>1491</td>\n",
" <td>5.245416</td>\n",
" <td>1536</td>\n",
" <td>5.201522</td>\n",
" <td>1306</td>\n",
" <td>5.181936</td>\n",
" <td>1599</td>\n",
" <td>5.002481</td>\n",
" <td>1467</td>\n",
" <td>4.944190</td>\n",
" <td>1080</td>\n",
" <td>4.872499</td>\n",
" <td>1642</td>\n",
" <td>4.870106</td>\n",
" <td>119</td>\n",
" <td>4.803536</td>\n",
" <td>1189</td>\n",
" <td>4.794895</td>\n",
" </tr>\n",
" <tr>\n",
" <th>940</th>\n",
" <td>941</td>\n",
" <td>814</td>\n",
" <td>5.894539</td>\n",
" <td>1491</td>\n",
" <td>5.677515</td>\n",
" <td>1536</td>\n",
" <td>5.633621</td>\n",
" <td>1306</td>\n",
" <td>5.614035</td>\n",
" <td>1599</td>\n",
" <td>5.434579</td>\n",
" <td>1467</td>\n",
" <td>5.376289</td>\n",
" <td>1080</td>\n",
" <td>5.304598</td>\n",
" <td>1642</td>\n",
" <td>5.302204</td>\n",
" <td>119</td>\n",
" <td>5.235635</td>\n",
" <td>1189</td>\n",
" <td>5.226994</td>\n",
" </tr>\n",
" <tr>\n",
" <th>941</th>\n",
" <td>942</td>\n",
" <td>814</td>\n",
" <td>6.164380</td>\n",
" <td>1491</td>\n",
" <td>5.947356</td>\n",
" <td>1536</td>\n",
" <td>5.903462</td>\n",
" <td>1306</td>\n",
" <td>5.883876</td>\n",
" <td>1599</td>\n",
" <td>5.704421</td>\n",
" <td>1467</td>\n",
" <td>5.646130</td>\n",
" <td>1080</td>\n",
" <td>5.574439</td>\n",
" <td>1642</td>\n",
" <td>5.572046</td>\n",
" <td>119</td>\n",
" <td>5.505476</td>\n",
" <td>1189</td>\n",
" <td>5.496835</td>\n",
" </tr>\n",
" <tr>\n",
" <th>942</th>\n",
" <td>943</td>\n",
" <td>814</td>\n",
" <td>5.284783</td>\n",
" <td>1491</td>\n",
" <td>5.067759</td>\n",
" <td>1536</td>\n",
" <td>5.023865</td>\n",
" <td>1306</td>\n",
" <td>5.004279</td>\n",
" <td>1599</td>\n",
" <td>4.824823</td>\n",
" <td>1467</td>\n",
" <td>4.766533</td>\n",
" <td>1080</td>\n",
" <td>4.694842</td>\n",
" <td>1642</td>\n",
" <td>4.692448</td>\n",
" <td>119</td>\n",
" <td>4.625879</td>\n",
" <td>1189</td>\n",
" <td>4.617238</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>943 rows × 21 columns</p>\n",
"</div>"
],
"text/plain": [
" 0 1 2 3 ... 17 18 19 20\n",
"0 1 814 5.449584 1491 ... 1189 4.782040 1500 4.742546\n",
"1 2 814 5.580813 1491 ... 119 4.921909 1189 4.913268\n",
"2 3 814 4.615469 1491 ... 119 3.956565 1189 3.947924\n",
"3 4 814 6.244539 1491 ... 119 5.585635 1189 5.576994\n",
"4 5 814 4.738510 1491 ... 119 4.079607 1189 4.070965\n",
".. ... ... ... ... ... ... ... ... ...\n",
"938 939 814 6.236644 1491 ... 119 5.577740 1189 5.569099\n",
"939 940 814 5.462440 1491 ... 119 4.803536 1189 4.794895\n",
"940 941 814 5.894539 1491 ... 119 5.235635 1189 5.226994\n",
"941 942 814 6.164380 1491 ... 119 5.505476 1189 5.496835\n",
"942 943 814 5.284783 1491 ... 119 4.625879 1189 4.617238\n",
"\n",
"[943 rows x 21 columns]"
]
},
"metadata": {
"tags": []
}
},
{
"output_type": "stream",
"text": [
"Estimations:\n"
],
"name": "stdout"
},
{
"output_type": "display_data",
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>user</th>\n",
" <th>item</th>\n",
" <th>est_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1</td>\n",
" <td>5</td>\n",
" <td>3.457161</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1</td>\n",
" <td>10</td>\n",
" <td>3.798540</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>1</td>\n",
" <td>25</td>\n",
" <td>3.435415</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>1</td>\n",
" <td>32</td>\n",
" <td>3.732018</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>1</td>\n",
" <td>33</td>\n",
" <td>3.531991</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19995</th>\n",
" <td>943</td>\n",
" <td>928</td>\n",
" <td>2.907189</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19996</th>\n",
" <td>943</td>\n",
" <td>1067</td>\n",
" <td>3.485929</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19997</th>\n",
" <td>943</td>\n",
" <td>1074</td>\n",
" <td>2.861988</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19998</th>\n",
" <td>943</td>\n",
" <td>1188</td>\n",
" <td>2.727428</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19999</th>\n",
" <td>943</td>\n",
" <td>1228</td>\n",
" <td>2.568442</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>20000 rows × 3 columns</p>\n",
"</div>"
],
"text/plain": [
" user item est_score\n",
"0 1 5 3.457161\n",
"1 1 10 3.798540\n",
"2 1 25 3.435415\n",
"3 1 32 3.732018\n",
"4 1 33 3.531991\n",
"... ... ... ...\n",
"19995 943 928 2.907189\n",
"19996 943 1067 3.485929\n",
"19997 943 1074 2.861988\n",
"19998 943 1188 2.727428\n",
"19999 943 1228 2.568442\n",
"\n",
"[20000 rows x 3 columns]"
]
},
"metadata": {
"tags": []
}
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "QSCe2WVVKkFG",
"colab_type": "text"
},
"source": [
"# Sample recommendations"
]
},
{
"cell_type": "code",
"metadata": {
"id": "uT8s_WQOKkFG",
"colab_type": "code",
"outputId": "13361cd0-f54a-4923-9303-422e501d7543",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 890
}
},
"source": [
"train=pd.read_csv('./Datasets/ml-100k/train.csv', sep='\\t', header=None, names=['user', 'item', 'rating', 'timestamp'])\n",
"items=pd.read_csv('./Datasets/ml-100k/movies.csv')\n",
"\n",
"user=random.choice(list(set(train['user'])))\n",
"\n",
"train_content=pd.merge(train, items, left_on='item', right_on='id')\n",
"\n",
"print('Here is what user rated high:')\n",
"display(train_content[train_content['user']==user][['user', 'rating', 'title', 'genres']]\\\n",
" .sort_values(by='rating', ascending=False)[:15])\n",
"\n",
"reco = np.loadtxt('Recommendations generated/ml-100k/Self_BaselineUI_reco.csv', delimiter=',')\n",
"items=pd.read_csv('./Datasets/ml-100k/movies.csv')\n",
"\n",
"# Let's ignore scores - they are not used in evaluation: \n",
"reco_users=reco[:,:1]\n",
"reco_items=reco[:,1::2]\n",
"# Let's put them into one array\n",
"reco=np.concatenate((reco_users, reco_items), axis=1)\n",
"\n",
"# Let's rebuild it user-item dataframe\n",
"recommended=[]\n",
"for row in reco:\n",
" for rec_nb, entry in enumerate(row[1:]):\n",
" recommended.append((row[0], rec_nb+1, entry))\n",
"recommended=pd.DataFrame(recommended, columns=['user','rec_nb', 'item'])\n",
"\n",
"recommended_content=pd.merge(recommended, items, left_on='item', right_on='id')\n",
"\n",
"print('Here is what we recommend:')\n",
"recommended_content[recommended_content['user']==user][['user', 'rec_nb', 'title', 'genres']].sort_values(by='rec_nb')"
],
"execution_count": 0,
"outputs": [
{
"output_type": "stream",
"text": [
"Here is what user rated high:\n"
],
"name": "stdout"
},
{
"output_type": "display_data",
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>user</th>\n",
" <th>rating</th>\n",
" <th>title</th>\n",
" <th>genres</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>5529</th>\n",
" <td>181</td>\n",
" <td>5</td>\n",
" <td>Jerry Maguire (1996)</td>\n",
" <td>Drama, Romance</td>\n",
" </tr>\n",
" <tr>\n",
" <th>38687</th>\n",
" <td>181</td>\n",
" <td>4</td>\n",
" <td>First Wives Club, The (1996)</td>\n",
" <td>Comedy</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19390</th>\n",
" <td>181</td>\n",
" <td>4</td>\n",
" <td>Scream (1996)</td>\n",
" <td>Horror, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
" <th>63720</th>\n",
" <td>181</td>\n",
" <td>4</td>\n",
" <td>Mirror Has Two Faces, The (1996)</td>\n",
" <td>Comedy, Romance</td>\n",
" </tr>\n",
" <tr>\n",
" <th>57287</th>\n",
" <td>181</td>\n",
" <td>4</td>\n",
" <td>Evita (1996)</td>\n",
" <td>Drama, Musical</td>\n",
" </tr>\n",
" <tr>\n",
" <th>57106</th>\n",
" <td>181</td>\n",
" <td>4</td>\n",
" <td>Up Close and Personal (1996)</td>\n",
" <td>Drama, Romance</td>\n",
" </tr>\n",
" <tr>\n",
" <th>24776</th>\n",
" <td>181</td>\n",
" <td>4</td>\n",
" <td>Gattaca (1997)</td>\n",
" <td>Drama, Sci-Fi, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11778</th>\n",
" <td>181</td>\n",
" <td>4</td>\n",
" <td>Mission: Impossible (1996)</td>\n",
" <td>Action, Adventure, Mystery</td>\n",
" </tr>\n",
" <tr>\n",
" <th>68469</th>\n",
" <td>181</td>\n",
" <td>4</td>\n",
" <td>Eye for an Eye (1996)</td>\n",
" <td>Drama, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25908</th>\n",
" <td>181</td>\n",
" <td>4</td>\n",
" <td>Dead Man Walking (1995)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9811</th>\n",
" <td>181</td>\n",
" <td>4</td>\n",
" <td>Time to Kill, A (1996)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
" <th>21011</th>\n",
" <td>181</td>\n",
" <td>4</td>\n",
" <td>I Know What You Did Last Summer (1997)</td>\n",
" <td>Horror, Mystery, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
" <th>33572</th>\n",
" <td>181</td>\n",
" <td>4</td>\n",
" <td>Independence Day (ID4) (1996)</td>\n",
" <td>Action, Sci-Fi, War</td>\n",
" </tr>\n",
" <tr>\n",
" <th>44631</th>\n",
" <td>181</td>\n",
" <td>4</td>\n",
" <td>Ransom (1996)</td>\n",
" <td>Drama, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
" <th>46726</th>\n",
" <td>181</td>\n",
" <td>4</td>\n",
" <td>Hunchback of Notre Dame, The (1996)</td>\n",
" <td>Animation, Children's, Musical</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" user ... genres\n",
"5529 181 ... Drama, Romance\n",
"38687 181 ... Comedy\n",
"19390 181 ... Horror, Thriller\n",
"63720 181 ... Comedy, Romance\n",
"57287 181 ... Drama, Musical\n",
"57106 181 ... Drama, Romance\n",
"24776 181 ... Drama, Sci-Fi, Thriller\n",
"11778 181 ... Action, Adventure, Mystery\n",
"68469 181 ... Drama, Thriller\n",
"25908 181 ... Drama\n",
"9811 181 ... Drama\n",
"21011 181 ... Horror, Mystery, Thriller\n",
"33572 181 ... Action, Sci-Fi, War\n",
"44631 181 ... Drama, Thriller\n",
"46726 181 ... Animation, Children's, Musical\n",
"\n",
"[15 rows x 4 columns]"
]
},
"metadata": {
"tags": []
}
},
{
"output_type": "stream",
"text": [
"Here is what we recommend:\n"
],
"name": "stdout"
},
{
"output_type": "execute_result",
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>user</th>\n",
" <th>rec_nb</th>\n",
" <th>title</th>\n",
" <th>genres</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>179</th>\n",
" <td>181.0</td>\n",
" <td>1</td>\n",
" <td>Great Day in Harlem, A (1994)</td>\n",
" <td>Documentary</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1122</th>\n",
" <td>181.0</td>\n",
" <td>2</td>\n",
" <td>Tough and Deadly (1995)</td>\n",
" <td>Action, Drama, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2064</th>\n",
" <td>181.0</td>\n",
" <td>3</td>\n",
" <td>Aiqing wansui (1994)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3005</th>\n",
" <td>181.0</td>\n",
" <td>4</td>\n",
" <td>Delta of Venus (1994)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3948</th>\n",
" <td>181.0</td>\n",
" <td>5</td>\n",
" <td>Someone Else's America (1995)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4890</th>\n",
" <td>181.0</td>\n",
" <td>6</td>\n",
" <td>Saint of Fort Washington, The (1993)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5831</th>\n",
" <td>181.0</td>\n",
" <td>7</td>\n",
" <td>Celestial Clockwork (1994)</td>\n",
" <td>Comedy</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6774</th>\n",
" <td>181.0</td>\n",
" <td>8</td>\n",
" <td>Some Mother's Son (1996)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8668</th>\n",
" <td>181.0</td>\n",
" <td>9</td>\n",
" <td>Maya Lin: A Strong Clear Vision (1994)</td>\n",
" <td>Documentary</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7714</th>\n",
" <td>181.0</td>\n",
" <td>10</td>\n",
" <td>Prefontaine (1997)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" user ... genres\n",
"179 181.0 ... Documentary\n",
"1122 181.0 ... Action, Drama, Thriller\n",
"2064 181.0 ... Drama\n",
"3005 181.0 ... Drama\n",
"3948 181.0 ... Drama\n",
"4890 181.0 ... Drama\n",
"5831 181.0 ... Comedy\n",
"6774 181.0 ... Drama\n",
"8668 181.0 ... Documentary\n",
"7714 181.0 ... Drama\n",
"\n",
"[10 rows x 4 columns]"
]
},
"metadata": {
"tags": []
},
"execution_count": 24
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "gFsye0sAKkFQ",
"colab_type": "text"
},
"source": [
"# project task 3: implement some other evaluation measure"
]
},
{
"cell_type": "code",
"metadata": {
"id": "3bhpq4zOKkFR",
"colab_type": "code",
"colab": {}
},
"source": [
"# it may be your idea, modification of what we have already implemented \n",
"# (for example Hit2 rate which would count as a success users whoreceived at least 2 relevant recommendations) \n",
"# or something well-known\n",
"# expected output: modification of evaluation_measures.py such that evaluate_all will also display your measure"
],
"execution_count": 0,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "8s81gSB3KkFU",
"colab_type": "code",
"outputId": "720e0c84-ef77-4c24-9ed9-df7697e9b98a",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 199
}
},
"source": [
"dir_path=\"Recommendations generated/ml-100k/\"\n",
"super_reactions=[4,5]\n",
"test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None)\n",
"\n",
"ev.evaluate_all(test, dir_path, super_reactions)"
],
"execution_count": 0,
"outputs": [
{
"output_type": "stream",
"text": [
"943it [00:00, 6725.92it/s]\n",
"943it [00:00, 6395.43it/s]\n"
],
"name": "stderr"
},
{
"output_type": "execute_result",
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" <th>HR2</th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Baseline</td>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" <td>0.091410</td>\n",
" <td>0.037652</td>\n",
" <td>0.046030</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
" <td>0.239661</td>\n",
" <td>1.00000</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineUI</td>\n",
" <td>0.967585</td>\n",
" <td>0.762740</td>\n",
" <td>0.000954</td>\n",
" <td>0.000170</td>\n",
" <td>0.000278</td>\n",
" <td>0.000463</td>\n",
" <td>0.000644</td>\n",
" <td>0.000189</td>\n",
" <td>0.000752</td>\n",
" <td>0.000168</td>\n",
" <td>0.001677</td>\n",
" <td>0.496424</td>\n",
" <td>0.009544</td>\n",
" <td>0.000000</td>\n",
" <td>0.60053</td>\n",
" <td>0.005051</td>\n",
" <td>1.803126</td>\n",
" <td>0.996380</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Model RMSE MAE ... Test coverage Shannon Gini\n",
"0 Ready_Baseline 0.949459 0.752487 ... 0.033911 2.836513 0.991139\n",
"0 Self_BaselineUI 0.967585 0.762740 ... 0.005051 1.803126 0.996380\n",
"\n",
"[2 rows x 19 columns]"
]
},
"metadata": {
"tags": []
},
"execution_count": 25
}
]
}
]
}
Reference in New Issue Copy Permalink