2123 lines
76 KiB
Plaintext
2123 lines
76 KiB
Plaintext
{
|
||
"nbformat": 4,
|
||
"nbformat_minor": 0,
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "Python 3",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.6.9"
|
||
},
|
||
"colab": {
|
||
"name": "P2. Evaluation.ipynb",
|
||
"provenance": [],
|
||
"collapsed_sections": [],
|
||
"toc_visible": true
|
||
}
|
||
},
|
||
"cells": [
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"id": "XsGH6668KkEb",
|
||
"colab_type": "text"
|
||
},
|
||
"source": [
|
||
"### Prepare test set"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"metadata": {
|
||
"id": "6i5evQN1KkEc",
|
||
"colab_type": "code",
|
||
"colab": {}
|
||
},
|
||
"source": [
|
||
"import pandas as pd\n",
|
||
"import numpy as np\n",
|
||
"import scipy.sparse as sparse\n",
|
||
"from collections import defaultdict\n",
|
||
"from itertools import chain\n",
|
||
"import random\n",
|
||
"from tqdm import tqdm\n",
|
||
"\n",
|
||
"# In evaluation we do not load train set - it is not needed\n",
|
||
"test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None)\n",
|
||
"test.columns=['user', 'item', 'rating', 'timestamp']\n",
|
||
"\n",
|
||
"test['user_code'] = test['user'].astype(\"category\").cat.codes\n",
|
||
"test['item_code'] = test['item'].astype(\"category\").cat.codes\n",
|
||
"\n",
|
||
"user_code_id = dict(enumerate(test['user'].astype(\"category\").cat.categories))\n",
|
||
"user_id_code = dict((v, k) for k, v in user_code_id.items())\n",
|
||
"item_code_id = dict(enumerate(test['item'].astype(\"category\").cat.categories))\n",
|
||
"item_id_code = dict((v, k) for k, v in item_code_id.items())\n",
|
||
"\n",
|
||
"test_ui = sparse.csr_matrix((test['rating'], (test['user_code'], test['item_code'])))"
|
||
],
|
||
"execution_count": 0,
|
||
"outputs": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"metadata": {
|
||
"id": "o1f-bErIK3Xx",
|
||
"colab_type": "code",
|
||
"colab": {}
|
||
},
|
||
"source": [
|
||
"import os\n",
|
||
"if not os.path.exists('./Recommendations generated/'):\n",
|
||
" os.mkdir('./Recommendations generated/')\n",
|
||
" os.mkdir('./Recommendations generated/ml-100k/')\n",
|
||
" os.mkdir('./Recommendations generated/toy-example/')"
|
||
],
|
||
"execution_count": 0,
|
||
"outputs": []
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"id": "e-h95H_wKkEg",
|
||
"colab_type": "text"
|
||
},
|
||
"source": [
|
||
"### Estimations metrics"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"metadata": {
|
||
"id": "Pg63ftwyKkEg",
|
||
"colab_type": "code",
|
||
"colab": {}
|
||
},
|
||
"source": [
|
||
"estimations_df=pd.read_csv('Recommendations generated/ml-100k/Ready_Baseline_estimations.csv', header=None)\n",
|
||
"estimations_df.columns=['user', 'item' ,'score']\n",
|
||
"\n",
|
||
"estimations_df['user_code']=[user_id_code[user] for user in estimations_df['user']]\n",
|
||
"estimations_df['item_code']=[item_id_code[item] for item in estimations_df['item']]\n",
|
||
"estimations=sparse.csr_matrix((estimations_df['score'], (estimations_df['user_code'], estimations_df['item_code'])), shape=test_ui.shape)"
|
||
],
|
||
"execution_count": 0,
|
||
"outputs": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"metadata": {
|
||
"id": "g4QKDwfQKkEj",
|
||
"colab_type": "code",
|
||
"colab": {}
|
||
},
|
||
"source": [
|
||
"def estimations_metrics(test_ui, estimations):\n",
|
||
" result=[]\n",
|
||
"\n",
|
||
" RMSE=(np.sum((estimations.data-test_ui.data)**2)/estimations.nnz)**(1/2)\n",
|
||
" result.append(['RMSE', RMSE])\n",
|
||
"\n",
|
||
" MAE=np.sum(abs(estimations.data-test_ui.data))/estimations.nnz\n",
|
||
" result.append(['MAE', MAE])\n",
|
||
" \n",
|
||
" df_result=(pd.DataFrame(list(zip(*result))[1])).T\n",
|
||
" df_result.columns=list(zip(*result))[0]\n",
|
||
" return df_result"
|
||
],
|
||
"execution_count": 0,
|
||
"outputs": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"metadata": {
|
||
"id": "LTDMTTiiKkEl",
|
||
"colab_type": "code",
|
||
"outputId": "7ebd1465-3dc6-404d-9666-67897e4d2750",
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 80
|
||
}
|
||
},
|
||
"source": [
|
||
"# in case of error (in the laboratories) you might have to switch to the other version of pandas\n",
|
||
"# try !pip3 install pandas=='1.0.3' (or pip if you use python 2) and restart the kernel\n",
|
||
"\n",
|
||
"estimations_metrics(test_ui, estimations)"
|
||
],
|
||
"execution_count": 0,
|
||
"outputs": [
|
||
{
|
||
"output_type": "execute_result",
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>RMSE</th>\n",
|
||
" <th>MAE</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>0.949459</td>\n",
|
||
" <td>0.752487</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" RMSE MAE\n",
|
||
"0 0.949459 0.752487"
|
||
]
|
||
},
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"execution_count": 6
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"id": "rSgzlQ-iKkEp",
|
||
"colab_type": "text"
|
||
},
|
||
"source": [
|
||
"### Ranking metrics"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"metadata": {
|
||
"id": "BX6YbfwgKkEp",
|
||
"colab_type": "code",
|
||
"outputId": "be746705-2a4d-4e53-f616-a3f4876ce68b",
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 136
|
||
}
|
||
},
|
||
"source": [
|
||
"import numpy as np\n",
|
||
"reco = np.loadtxt('Recommendations generated/ml-100k/Ready_Baseline_reco.csv', delimiter=',')\n",
|
||
"# Let's ignore scores - they are not used in evaluation: \n",
|
||
"users=reco[:,:1]\n",
|
||
"items=reco[:,1::2]\n",
|
||
"# Let's use inner ids instead of real ones\n",
|
||
"users=np.vectorize(lambda x: user_id_code.setdefault(x, -1))(users)\n",
|
||
"items=np.vectorize(lambda x: item_id_code.setdefault(x, -1))(items) # maybe items we recommend are not in test set\n",
|
||
"# Let's put them into one array\n",
|
||
"reco=np.concatenate((users, items), axis=1)\n",
|
||
"reco"
|
||
],
|
||
"execution_count": 0,
|
||
"outputs": [
|
||
{
|
||
"output_type": "execute_result",
|
||
"data": {
|
||
"text/plain": [
|
||
"array([[663, 475, 62, ..., 472, 269, 503],\n",
|
||
" [ 48, 313, 475, ..., 591, 175, 466],\n",
|
||
" [351, 313, 475, ..., 591, 175, 466],\n",
|
||
" ...,\n",
|
||
" [259, 313, 475, ..., 11, 591, 175],\n",
|
||
" [ 33, 313, 475, ..., 11, 591, 175],\n",
|
||
" [ 77, 313, 475, ..., 11, 591, 175]])"
|
||
]
|
||
},
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"execution_count": 7
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"metadata": {
|
||
"id": "NjVfzif8KkEs",
|
||
"colab_type": "code",
|
||
"colab": {}
|
||
},
|
||
"source": [
|
||
"def ranking_metrics(test_ui, reco, super_reactions=[], topK=10):\n",
|
||
" \n",
|
||
" nb_items=test_ui.shape[1]\n",
|
||
" relevant_users, super_relevant_users, prec, rec, F_1, F_05, prec_super, rec_super, ndcg, mAP, MRR, LAUC, HR=\\\n",
|
||
" 0,0,0,0,0,0,0,0,0,0,0,0,0\n",
|
||
" \n",
|
||
" cg = (1.0 / np.log2(np.arange(2, topK + 2)))\n",
|
||
" cg_sum = np.cumsum(cg)\n",
|
||
" \n",
|
||
" for (nb_user, user) in tqdm(enumerate(reco[:,0])):\n",
|
||
" u_rated_items=test_ui.indices[test_ui.indptr[user]:test_ui.indptr[user+1]]\n",
|
||
" nb_u_rated_items=len(u_rated_items)\n",
|
||
" if nb_u_rated_items>0: # skip users with no items in test set (still possible that there will be no super items)\n",
|
||
" relevant_users+=1\n",
|
||
" \n",
|
||
" u_super_items=u_rated_items[np.vectorize(lambda x: x in super_reactions)\\\n",
|
||
" (test_ui.data[test_ui.indptr[user]:test_ui.indptr[user+1]])]\n",
|
||
" # more natural seems u_super_items=[item for item in u_rated_items if test_ui[user,item] in super_reactions]\n",
|
||
" # but accesing test_ui[user,item] is expensive -we should avoid doing it\n",
|
||
" if len(u_super_items)>0:\n",
|
||
" super_relevant_users+=1\n",
|
||
" \n",
|
||
" user_successes=np.zeros(topK)\n",
|
||
" nb_user_successes=0\n",
|
||
" user_super_successes=np.zeros(topK)\n",
|
||
" nb_user_super_successes=0\n",
|
||
" \n",
|
||
" # evaluation\n",
|
||
" for (item_position,item) in enumerate(reco[nb_user,1:topK+1]):\n",
|
||
" if item in u_rated_items:\n",
|
||
" user_successes[item_position]=1\n",
|
||
" nb_user_successes+=1\n",
|
||
" if item in u_super_items:\n",
|
||
" user_super_successes[item_position]=1\n",
|
||
" nb_user_super_successes+=1\n",
|
||
" \n",
|
||
" prec_u=nb_user_successes/topK \n",
|
||
" prec+=prec_u\n",
|
||
" \n",
|
||
" rec_u=nb_user_successes/nb_u_rated_items\n",
|
||
" rec+=rec_u\n",
|
||
" \n",
|
||
" F_1+=2*(prec_u*rec_u)/(prec_u+rec_u) if prec_u+rec_u>0 else 0\n",
|
||
" F_05+=(0.5**2+1)*(prec_u*rec_u)/(0.5**2*prec_u+rec_u) if prec_u+rec_u>0 else 0\n",
|
||
" \n",
|
||
" prec_super+=nb_user_super_successes/topK\n",
|
||
" rec_super+=nb_user_super_successes/max(len(u_super_items),1) # to set 0 if no super items\n",
|
||
" ndcg+=np.dot(user_successes,cg)/cg_sum[min(topK, nb_u_rated_items)-1]\n",
|
||
" \n",
|
||
" cumsum_successes=np.cumsum(user_successes)\n",
|
||
" mAP+=np.dot(cumsum_successes/np.arange(1,topK+1), user_successes)/min(topK, nb_u_rated_items)\n",
|
||
" MRR+=1/(user_successes.nonzero()[0][0]+1) if user_successes.nonzero()[0].size>0 else 0\n",
|
||
" LAUC+=(np.dot(cumsum_successes, 1-user_successes)+\\\n",
|
||
" (nb_user_successes+nb_u_rated_items)/2*((nb_items-nb_u_rated_items)-(topK-nb_user_successes)))/\\\n",
|
||
" ((nb_items-nb_u_rated_items)*nb_u_rated_items)\n",
|
||
" \n",
|
||
" HR+=nb_user_successes>0\n",
|
||
" \n",
|
||
" \n",
|
||
" result=[]\n",
|
||
" result.append(('precision', prec/relevant_users))\n",
|
||
" result.append(('recall', rec/relevant_users))\n",
|
||
" result.append(('F_1', F_1/relevant_users))\n",
|
||
" result.append(('F_05', F_05/relevant_users))\n",
|
||
" result.append(('precision_super', prec_super/super_relevant_users))\n",
|
||
" result.append(('recall_super', rec_super/super_relevant_users))\n",
|
||
" result.append(('NDCG', ndcg/relevant_users))\n",
|
||
" result.append(('mAP', mAP/relevant_users))\n",
|
||
" result.append(('MRR', MRR/relevant_users))\n",
|
||
" result.append(('LAUC', LAUC/relevant_users))\n",
|
||
" result.append(('HR', HR/relevant_users))\n",
|
||
"\n",
|
||
" df_result=(pd.DataFrame(list(zip(*result))[1])).T\n",
|
||
" df_result.columns=list(zip(*result))[0]\n",
|
||
" return df_result"
|
||
],
|
||
"execution_count": 0,
|
||
"outputs": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"metadata": {
|
||
"id": "xspx_TyWKkEv",
|
||
"colab_type": "code",
|
||
"outputId": "41145c03-763d-4584-e7d4-f5ec16a5507f",
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 97
|
||
}
|
||
},
|
||
"source": [
|
||
"ranking_metrics(test_ui, reco, super_reactions=[4,5], topK=10)"
|
||
],
|
||
"execution_count": 0,
|
||
"outputs": [
|
||
{
|
||
"output_type": "stream",
|
||
"text": [
|
||
"943it [00:00, 6981.89it/s]\n"
|
||
],
|
||
"name": "stderr"
|
||
},
|
||
{
|
||
"output_type": "execute_result",
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>precision</th>\n",
|
||
" <th>recall</th>\n",
|
||
" <th>F_1</th>\n",
|
||
" <th>F_05</th>\n",
|
||
" <th>precision_super</th>\n",
|
||
" <th>recall_super</th>\n",
|
||
" <th>NDCG</th>\n",
|
||
" <th>mAP</th>\n",
|
||
" <th>MRR</th>\n",
|
||
" <th>LAUC</th>\n",
|
||
" <th>HR</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>0.09141</td>\n",
|
||
" <td>0.037652</td>\n",
|
||
" <td>0.04603</td>\n",
|
||
" <td>0.061286</td>\n",
|
||
" <td>0.079614</td>\n",
|
||
" <td>0.056463</td>\n",
|
||
" <td>0.095957</td>\n",
|
||
" <td>0.043178</td>\n",
|
||
" <td>0.198193</td>\n",
|
||
" <td>0.515501</td>\n",
|
||
" <td>0.437964</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" precision recall F_1 ... MRR LAUC HR\n",
|
||
"0 0.09141 0.037652 0.04603 ... 0.198193 0.515501 0.437964\n",
|
||
"\n",
|
||
"[1 rows x 11 columns]"
|
||
]
|
||
},
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"execution_count": 9
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"id": "sIV-r9MWKkEy",
|
||
"colab_type": "text"
|
||
},
|
||
"source": [
|
||
"### Diversity metrics"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"metadata": {
|
||
"id": "XOtNDMiRKkEy",
|
||
"colab_type": "code",
|
||
"colab": {}
|
||
},
|
||
"source": [
|
||
"def diversity_metrics(test_ui, reco, topK=10):\n",
|
||
" \n",
|
||
" frequencies=defaultdict(int)\n",
|
||
" \n",
|
||
" # let's assign 0 to all items in test set\n",
|
||
" for item in list(set(test_ui.indices)):\n",
|
||
" frequencies[item]=0\n",
|
||
" \n",
|
||
" # counting frequencies\n",
|
||
" for item in reco[:,1:].flat:\n",
|
||
" frequencies[item]+=1\n",
|
||
" \n",
|
||
" nb_reco_outside_test=frequencies[-1]\n",
|
||
" del frequencies[-1]\n",
|
||
" \n",
|
||
" frequencies=np.array(list(frequencies.values()))\n",
|
||
" \n",
|
||
" nb_rec_items=len(frequencies[frequencies>0])\n",
|
||
" nb_reco_inside_test=np.sum(frequencies)\n",
|
||
" \n",
|
||
" frequencies=frequencies/np.sum(frequencies)\n",
|
||
" frequencies=np.sort(frequencies)\n",
|
||
" \n",
|
||
" with np.errstate(divide='ignore'): # let's put zeros put items with 0 frequency and ignore division warning\n",
|
||
" log_frequencies=np.nan_to_num(np.log(frequencies), posinf=0, neginf=0)\n",
|
||
" \n",
|
||
" result=[]\n",
|
||
" result.append(('Reco in test', nb_reco_inside_test/(nb_reco_inside_test+nb_reco_outside_test)))\n",
|
||
" result.append(('Test coverage', nb_rec_items/test_ui.shape[1]))\n",
|
||
" result.append(('Shannon', -np.dot(frequencies, log_frequencies)))\n",
|
||
" result.append(('Gini', np.dot(frequencies, np.arange(1-len(frequencies), len(frequencies), 2))/(len(frequencies)-1)))\n",
|
||
" \n",
|
||
" df_result=(pd.DataFrame(list(zip(*result))[1])).T\n",
|
||
" df_result.columns=list(zip(*result))[0]\n",
|
||
" return df_result"
|
||
],
|
||
"execution_count": 0,
|
||
"outputs": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"metadata": {
|
||
"id": "LmWMC0beKkE0",
|
||
"colab_type": "code",
|
||
"outputId": "99e50bfb-b99d-4e54-b34e-9726d421f3e0",
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 80
|
||
}
|
||
},
|
||
"source": [
|
||
"# in case of errors try !pip3 install numpy==1.18.4 (or pip if you use python 2) and restart the kernel\n",
|
||
"\n",
|
||
"import evaluation_measures as ev\n",
|
||
"import imp\n",
|
||
"imp.reload(ev)\n",
|
||
"\n",
|
||
"x=diversity_metrics(test_ui, reco, topK=10)\n",
|
||
"x"
|
||
],
|
||
"execution_count": 0,
|
||
"outputs": [
|
||
{
|
||
"output_type": "execute_result",
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>Reco in test</th>\n",
|
||
" <th>Test coverage</th>\n",
|
||
" <th>Shannon</th>\n",
|
||
" <th>Gini</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>0.033911</td>\n",
|
||
" <td>2.836513</td>\n",
|
||
" <td>0.991139</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" Reco in test Test coverage Shannon Gini\n",
|
||
"0 1.0 0.033911 2.836513 0.991139"
|
||
]
|
||
},
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"execution_count": 13
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"id": "KWtRicX0KkE3",
|
||
"colab_type": "text"
|
||
},
|
||
"source": [
|
||
"# To be used in other notebooks"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"metadata": {
|
||
"id": "1ozOsZWMKkE3",
|
||
"colab_type": "code",
|
||
"outputId": "79b554c9-ec53-4565-f55d-c8343876cb67",
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 114
|
||
}
|
||
},
|
||
"source": [
|
||
"import evaluation_measures as ev\n",
|
||
"import imp\n",
|
||
"imp.reload(ev)\n",
|
||
"\n",
|
||
"estimations_df=pd.read_csv('Recommendations generated/ml-100k/Ready_Baseline_estimations.csv', header=None)\n",
|
||
"reco=np.loadtxt('Recommendations generated/ml-100k/Ready_Baseline_reco.csv', delimiter=',')\n",
|
||
"\n",
|
||
"ev.evaluate(test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None),\n",
|
||
" estimations_df=estimations_df, \n",
|
||
" reco=reco,\n",
|
||
" super_reactions=[4,5])\n",
|
||
"#also you can just type ev.evaluate_all(estimations_df, reco) - I put above values as default"
|
||
],
|
||
"execution_count": 0,
|
||
"outputs": [
|
||
{
|
||
"output_type": "stream",
|
||
"text": [
|
||
"943it [00:00, 6721.23it/s]\n"
|
||
],
|
||
"name": "stderr"
|
||
},
|
||
{
|
||
"output_type": "execute_result",
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>RMSE</th>\n",
|
||
" <th>MAE</th>\n",
|
||
" <th>precision</th>\n",
|
||
" <th>recall</th>\n",
|
||
" <th>F_1</th>\n",
|
||
" <th>F_05</th>\n",
|
||
" <th>precision_super</th>\n",
|
||
" <th>recall_super</th>\n",
|
||
" <th>NDCG</th>\n",
|
||
" <th>mAP</th>\n",
|
||
" <th>MRR</th>\n",
|
||
" <th>LAUC</th>\n",
|
||
" <th>HR</th>\n",
|
||
" <th>HR2</th>\n",
|
||
" <th>Reco in test</th>\n",
|
||
" <th>Test coverage</th>\n",
|
||
" <th>Shannon</th>\n",
|
||
" <th>Gini</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>0.949459</td>\n",
|
||
" <td>0.752487</td>\n",
|
||
" <td>0.09141</td>\n",
|
||
" <td>0.037652</td>\n",
|
||
" <td>0.04603</td>\n",
|
||
" <td>0.061286</td>\n",
|
||
" <td>0.079614</td>\n",
|
||
" <td>0.056463</td>\n",
|
||
" <td>0.095957</td>\n",
|
||
" <td>0.043178</td>\n",
|
||
" <td>0.198193</td>\n",
|
||
" <td>0.515501</td>\n",
|
||
" <td>0.437964</td>\n",
|
||
" <td>0.239661</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>0.033911</td>\n",
|
||
" <td>2.836513</td>\n",
|
||
" <td>0.991139</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" RMSE MAE precision ... Test coverage Shannon Gini\n",
|
||
"0 0.949459 0.752487 0.09141 ... 0.033911 2.836513 0.991139\n",
|
||
"\n",
|
||
"[1 rows x 18 columns]"
|
||
]
|
||
},
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"execution_count": 14
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"metadata": {
|
||
"id": "i1qKPl8LKkE6",
|
||
"colab_type": "code",
|
||
"outputId": "5f167923-0533-45a7-9536-5cc2e162f79e",
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 34
|
||
}
|
||
},
|
||
"source": [
|
||
"import evaluation_measures as ev\n",
|
||
"import imp\n",
|
||
"imp.reload(ev)\n",
|
||
"\n",
|
||
"dir_path=\"Recommendations generated/ml-100k/\"\n",
|
||
"super_reactions=[4,5]\n",
|
||
"test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None)\n",
|
||
"\n",
|
||
"df=ev.evaluate_all(test, dir_path, super_reactions)\n",
|
||
"#also you can just type ev.evaluate_all() - I put above values as default"
|
||
],
|
||
"execution_count": 0,
|
||
"outputs": [
|
||
{
|
||
"output_type": "stream",
|
||
"text": [
|
||
"943it [00:00, 7140.87it/s]\n"
|
||
],
|
||
"name": "stderr"
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"metadata": {
|
||
"id": "ADUVNxYnKkE9",
|
||
"colab_type": "code",
|
||
"outputId": "2d7a1008-8eed-425a-a3a0-4602fe5d5b32",
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 80
|
||
}
|
||
},
|
||
"source": [
|
||
"df.iloc[:,:9]"
|
||
],
|
||
"execution_count": 0,
|
||
"outputs": [
|
||
{
|
||
"output_type": "execute_result",
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>Model</th>\n",
|
||
" <th>RMSE</th>\n",
|
||
" <th>MAE</th>\n",
|
||
" <th>precision</th>\n",
|
||
" <th>recall</th>\n",
|
||
" <th>F_1</th>\n",
|
||
" <th>F_05</th>\n",
|
||
" <th>precision_super</th>\n",
|
||
" <th>recall_super</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>Ready_Baseline</td>\n",
|
||
" <td>0.949459</td>\n",
|
||
" <td>0.752487</td>\n",
|
||
" <td>0.09141</td>\n",
|
||
" <td>0.037652</td>\n",
|
||
" <td>0.04603</td>\n",
|
||
" <td>0.061286</td>\n",
|
||
" <td>0.079614</td>\n",
|
||
" <td>0.056463</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" Model RMSE MAE ... F_05 precision_super recall_super\n",
|
||
"0 Ready_Baseline 0.949459 0.752487 ... 0.061286 0.079614 0.056463\n",
|
||
"\n",
|
||
"[1 rows x 9 columns]"
|
||
]
|
||
},
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"execution_count": 16
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"metadata": {
|
||
"id": "QmTqLVNLKkFA",
|
||
"colab_type": "code",
|
||
"outputId": "eba7e18f-d73e-4108-a5f5-3fca175db010",
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 80
|
||
}
|
||
},
|
||
"source": [
|
||
"df.iloc[:,np.append(0,np.arange(9, df.shape[1]))]"
|
||
],
|
||
"execution_count": 0,
|
||
"outputs": [
|
||
{
|
||
"output_type": "execute_result",
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>Model</th>\n",
|
||
" <th>NDCG</th>\n",
|
||
" <th>mAP</th>\n",
|
||
" <th>MRR</th>\n",
|
||
" <th>LAUC</th>\n",
|
||
" <th>HR</th>\n",
|
||
" <th>HR2</th>\n",
|
||
" <th>Reco in test</th>\n",
|
||
" <th>Test coverage</th>\n",
|
||
" <th>Shannon</th>\n",
|
||
" <th>Gini</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>Ready_Baseline</td>\n",
|
||
" <td>0.095957</td>\n",
|
||
" <td>0.043178</td>\n",
|
||
" <td>0.198193</td>\n",
|
||
" <td>0.515501</td>\n",
|
||
" <td>0.437964</td>\n",
|
||
" <td>0.239661</td>\n",
|
||
" <td>1.0</td>\n",
|
||
" <td>0.033911</td>\n",
|
||
" <td>2.836513</td>\n",
|
||
" <td>0.991139</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" Model NDCG mAP ... Test coverage Shannon Gini\n",
|
||
"0 Ready_Baseline 0.095957 0.043178 ... 0.033911 2.836513 0.991139\n",
|
||
"\n",
|
||
"[1 rows x 11 columns]"
|
||
]
|
||
},
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"execution_count": 17
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"metadata": {
|
||
"id": "ADNCFV1fOzER",
|
||
"colab_type": "code",
|
||
"outputId": "bd3b8f9b-57e6-4b67-af7c-d7abea1c02b0",
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 289
|
||
}
|
||
},
|
||
"source": [
|
||
"pip install surprise"
|
||
],
|
||
"execution_count": 0,
|
||
"outputs": [
|
||
{
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Collecting surprise\n",
|
||
" Downloading https://files.pythonhosted.org/packages/61/de/e5cba8682201fcf9c3719a6fdda95693468ed061945493dea2dd37c5618b/surprise-0.1-py2.py3-none-any.whl\n",
|
||
"Collecting scikit-surprise\n",
|
||
"\u001b[?25l Downloading https://files.pythonhosted.org/packages/f5/da/b5700d96495fb4f092be497f02492768a3d96a3f4fa2ae7dea46d4081cfa/scikit-surprise-1.1.0.tar.gz (6.4MB)\n",
|
||
"\u001b[K |████████████████████████████████| 6.5MB 2.6MB/s \n",
|
||
"\u001b[?25hRequirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise->surprise) (0.15.1)\n",
|
||
"Requirement already satisfied: numpy>=1.11.2 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise->surprise) (1.18.5)\n",
|
||
"Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise->surprise) (1.4.1)\n",
|
||
"Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise->surprise) (1.12.0)\n",
|
||
"Building wheels for collected packages: scikit-surprise\n",
|
||
" Building wheel for scikit-surprise (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
|
||
" Created wheel for scikit-surprise: filename=scikit_surprise-1.1.0-cp36-cp36m-linux_x86_64.whl size=1675370 sha256=b9c28146ba19d464e7357de16052301c4261e1e492c668f6ec5ec59683796e4f\n",
|
||
" Stored in directory: /root/.cache/pip/wheels/cc/fa/8c/16c93fccce688ae1bde7d979ff102f7bee980d9cfeb8641bcf\n",
|
||
"Successfully built scikit-surprise\n",
|
||
"Installing collected packages: scikit-surprise, surprise\n",
|
||
"Successfully installed scikit-surprise-1.1.0 surprise-0.1\n"
|
||
],
|
||
"name": "stdout"
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"id": "r7NacKLTKkFD",
|
||
"colab_type": "text"
|
||
},
|
||
"source": [
|
||
"# Check metrics on toy dataset"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"metadata": {
|
||
"id": "ijxxLsVNKkFD",
|
||
"colab_type": "code",
|
||
"outputId": "0844bd14-84f0-45e1-e07d-1265a2b98b33",
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 1000
|
||
}
|
||
},
|
||
"source": [
|
||
"import evaluation_measures as ev\n",
|
||
"import imp\n",
|
||
"import helpers\n",
|
||
"imp.reload(ev)\n",
|
||
"\n",
|
||
"dir_path=\"Recommendations generated/toy-example/\"\n",
|
||
"super_reactions=[4,5]\n",
|
||
"test=pd.read_csv('./Datasets/toy-example/test.csv', sep='\\t', header=None)\n",
|
||
"\n",
|
||
"display(ev.evaluate_all(test, dir_path, super_reactions, topK=3))\n",
|
||
"#also you can just type ev.evaluate_all() - I put above values as default\n",
|
||
"\n",
|
||
"toy_train_read=pd.read_csv('./Datasets/toy-example/train.csv', sep='\\t', header=None, names=['user', 'item', 'rating', 'timestamp'])\n",
|
||
"toy_test_read=pd.read_csv('./Datasets/toy-example/test.csv', sep='\\t', header=None, names=['user', 'item', 'rating', 'timestamp'])\n",
|
||
"reco=pd.read_csv('Recommendations generated/toy-example/Self_BaselineUI_reco.csv', header=None)\n",
|
||
"estimations=pd.read_csv('Recommendations generated/toy-example/Self_BaselineUI_estimations.csv', names=['user', 'item', 'est_score'])\n",
|
||
"toy_train_ui, toy_test_ui, toy_user_code_id, toy_user_id_code, \\\n",
|
||
"toy_item_code_id, toy_item_id_code = helpers.data_to_csr(toy_train_read, toy_test_read)\n",
|
||
"\n",
|
||
"print('Training data:')\n",
|
||
"display(toy_train_ui.todense())\n",
|
||
"\n",
|
||
"print('Test data:')\n",
|
||
"display(toy_test_ui.todense())\n",
|
||
"\n",
|
||
"print('Recommendations:')\n",
|
||
"display(reco)\n",
|
||
"\n",
|
||
"print('Estimations:')\n",
|
||
"display(estimations)"
|
||
],
|
||
"execution_count": 0,
|
||
"outputs": [
|
||
{
|
||
"output_type": "stream",
|
||
"text": [
|
||
"943it [00:00, 12096.61it/s]\n"
|
||
],
|
||
"name": "stderr"
|
||
},
|
||
{
|
||
"output_type": "display_data",
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>Model</th>\n",
|
||
" <th>RMSE</th>\n",
|
||
" <th>MAE</th>\n",
|
||
" <th>precision</th>\n",
|
||
" <th>recall</th>\n",
|
||
" <th>F_1</th>\n",
|
||
" <th>F_05</th>\n",
|
||
" <th>precision_super</th>\n",
|
||
" <th>recall_super</th>\n",
|
||
" <th>NDCG</th>\n",
|
||
" <th>mAP</th>\n",
|
||
" <th>MRR</th>\n",
|
||
" <th>LAUC</th>\n",
|
||
" <th>HR</th>\n",
|
||
" <th>HR2</th>\n",
|
||
" <th>Reco in test</th>\n",
|
||
" <th>Test coverage</th>\n",
|
||
" <th>Shannon</th>\n",
|
||
" <th>Gini</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>Self_BaselineUI</td>\n",
|
||
" <td>0.967585</td>\n",
|
||
" <td>0.76274</td>\n",
|
||
" <td>0.000353</td>\n",
|
||
" <td>0.000044</td>\n",
|
||
" <td>0.000079</td>\n",
|
||
" <td>0.000147</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>0.000314</td>\n",
|
||
" <td>0.000177</td>\n",
|
||
" <td>0.00053</td>\n",
|
||
" <td>0.498923</td>\n",
|
||
" <td>0.00106</td>\n",
|
||
" <td>0.0</td>\n",
|
||
" <td>0.60053</td>\n",
|
||
" <td>0.005051</td>\n",
|
||
" <td>1.803126</td>\n",
|
||
" <td>0.99638</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" Model RMSE MAE ... Test coverage Shannon Gini\n",
|
||
"0 Self_BaselineUI 0.967585 0.76274 ... 0.005051 1.803126 0.99638\n",
|
||
"\n",
|
||
"[1 rows x 19 columns]"
|
||
]
|
||
},
|
||
"metadata": {
|
||
"tags": []
|
||
}
|
||
},
|
||
{
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Training data:\n"
|
||
],
|
||
"name": "stdout"
|
||
},
|
||
{
|
||
"output_type": "display_data",
|
||
"data": {
|
||
"text/plain": [
|
||
"matrix([[5, 3, 4, ..., 0, 0, 0],\n",
|
||
" [0, 0, 0, ..., 0, 0, 0],\n",
|
||
" [0, 0, 0, ..., 0, 0, 0],\n",
|
||
" ...,\n",
|
||
" [5, 0, 0, ..., 0, 0, 0],\n",
|
||
" [0, 0, 0, ..., 0, 0, 0],\n",
|
||
" [0, 0, 0, ..., 0, 0, 0]], dtype=int64)"
|
||
]
|
||
},
|
||
"metadata": {
|
||
"tags": []
|
||
}
|
||
},
|
||
{
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Test data:\n"
|
||
],
|
||
"name": "stdout"
|
||
},
|
||
{
|
||
"output_type": "display_data",
|
||
"data": {
|
||
"text/plain": [
|
||
"matrix([[0, 0, 0, ..., 0, 0, 0],\n",
|
||
" [4, 0, 0, ..., 0, 0, 0],\n",
|
||
" [0, 0, 0, ..., 0, 0, 0],\n",
|
||
" ...,\n",
|
||
" [0, 0, 0, ..., 0, 0, 0],\n",
|
||
" [0, 0, 0, ..., 0, 0, 0],\n",
|
||
" [0, 5, 0, ..., 0, 0, 0]], dtype=int64)"
|
||
]
|
||
},
|
||
"metadata": {
|
||
"tags": []
|
||
}
|
||
},
|
||
{
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Recommendations:\n"
|
||
],
|
||
"name": "stdout"
|
||
},
|
||
{
|
||
"output_type": "display_data",
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>0</th>\n",
|
||
" <th>1</th>\n",
|
||
" <th>2</th>\n",
|
||
" <th>3</th>\n",
|
||
" <th>4</th>\n",
|
||
" <th>5</th>\n",
|
||
" <th>6</th>\n",
|
||
" <th>7</th>\n",
|
||
" <th>8</th>\n",
|
||
" <th>9</th>\n",
|
||
" <th>10</th>\n",
|
||
" <th>11</th>\n",
|
||
" <th>12</th>\n",
|
||
" <th>13</th>\n",
|
||
" <th>14</th>\n",
|
||
" <th>15</th>\n",
|
||
" <th>16</th>\n",
|
||
" <th>17</th>\n",
|
||
" <th>18</th>\n",
|
||
" <th>19</th>\n",
|
||
" <th>20</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>1</td>\n",
|
||
" <td>814</td>\n",
|
||
" <td>5.449584</td>\n",
|
||
" <td>1491</td>\n",
|
||
" <td>5.232561</td>\n",
|
||
" <td>1536</td>\n",
|
||
" <td>5.188667</td>\n",
|
||
" <td>1306</td>\n",
|
||
" <td>5.169081</td>\n",
|
||
" <td>1599</td>\n",
|
||
" <td>4.989625</td>\n",
|
||
" <td>1467</td>\n",
|
||
" <td>4.931335</td>\n",
|
||
" <td>1080</td>\n",
|
||
" <td>4.859644</td>\n",
|
||
" <td>1642</td>\n",
|
||
" <td>4.857250</td>\n",
|
||
" <td>1189</td>\n",
|
||
" <td>4.782040</td>\n",
|
||
" <td>1500</td>\n",
|
||
" <td>4.742546</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>1</th>\n",
|
||
" <td>2</td>\n",
|
||
" <td>814</td>\n",
|
||
" <td>5.580813</td>\n",
|
||
" <td>1491</td>\n",
|
||
" <td>5.363789</td>\n",
|
||
" <td>1536</td>\n",
|
||
" <td>5.319895</td>\n",
|
||
" <td>1306</td>\n",
|
||
" <td>5.300310</td>\n",
|
||
" <td>1599</td>\n",
|
||
" <td>5.120854</td>\n",
|
||
" <td>1467</td>\n",
|
||
" <td>5.062563</td>\n",
|
||
" <td>1080</td>\n",
|
||
" <td>4.990872</td>\n",
|
||
" <td>1642</td>\n",
|
||
" <td>4.988479</td>\n",
|
||
" <td>119</td>\n",
|
||
" <td>4.921909</td>\n",
|
||
" <td>1189</td>\n",
|
||
" <td>4.913268</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>2</th>\n",
|
||
" <td>3</td>\n",
|
||
" <td>814</td>\n",
|
||
" <td>4.615469</td>\n",
|
||
" <td>1491</td>\n",
|
||
" <td>4.398445</td>\n",
|
||
" <td>1536</td>\n",
|
||
" <td>4.354551</td>\n",
|
||
" <td>1306</td>\n",
|
||
" <td>4.334965</td>\n",
|
||
" <td>1599</td>\n",
|
||
" <td>4.155510</td>\n",
|
||
" <td>1467</td>\n",
|
||
" <td>4.097219</td>\n",
|
||
" <td>1080</td>\n",
|
||
" <td>4.025528</td>\n",
|
||
" <td>1642</td>\n",
|
||
" <td>4.023134</td>\n",
|
||
" <td>119</td>\n",
|
||
" <td>3.956565</td>\n",
|
||
" <td>1189</td>\n",
|
||
" <td>3.947924</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>3</th>\n",
|
||
" <td>4</td>\n",
|
||
" <td>814</td>\n",
|
||
" <td>6.244539</td>\n",
|
||
" <td>1491</td>\n",
|
||
" <td>6.027515</td>\n",
|
||
" <td>1536</td>\n",
|
||
" <td>5.983621</td>\n",
|
||
" <td>1306</td>\n",
|
||
" <td>5.964035</td>\n",
|
||
" <td>1599</td>\n",
|
||
" <td>5.784579</td>\n",
|
||
" <td>1467</td>\n",
|
||
" <td>5.726289</td>\n",
|
||
" <td>1080</td>\n",
|
||
" <td>5.654598</td>\n",
|
||
" <td>1642</td>\n",
|
||
" <td>5.652204</td>\n",
|
||
" <td>119</td>\n",
|
||
" <td>5.585635</td>\n",
|
||
" <td>1189</td>\n",
|
||
" <td>5.576994</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>4</th>\n",
|
||
" <td>5</td>\n",
|
||
" <td>814</td>\n",
|
||
" <td>4.738510</td>\n",
|
||
" <td>1491</td>\n",
|
||
" <td>4.521486</td>\n",
|
||
" <td>1536</td>\n",
|
||
" <td>4.477592</td>\n",
|
||
" <td>1306</td>\n",
|
||
" <td>4.458007</td>\n",
|
||
" <td>1599</td>\n",
|
||
" <td>4.278551</td>\n",
|
||
" <td>1467</td>\n",
|
||
" <td>4.220260</td>\n",
|
||
" <td>1080</td>\n",
|
||
" <td>4.148569</td>\n",
|
||
" <td>1642</td>\n",
|
||
" <td>4.146176</td>\n",
|
||
" <td>119</td>\n",
|
||
" <td>4.079607</td>\n",
|
||
" <td>1189</td>\n",
|
||
" <td>4.070965</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>...</th>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>938</th>\n",
|
||
" <td>939</td>\n",
|
||
" <td>814</td>\n",
|
||
" <td>6.236644</td>\n",
|
||
" <td>1491</td>\n",
|
||
" <td>6.019620</td>\n",
|
||
" <td>1536</td>\n",
|
||
" <td>5.975726</td>\n",
|
||
" <td>1306</td>\n",
|
||
" <td>5.956140</td>\n",
|
||
" <td>1599</td>\n",
|
||
" <td>5.776685</td>\n",
|
||
" <td>1467</td>\n",
|
||
" <td>5.718394</td>\n",
|
||
" <td>1080</td>\n",
|
||
" <td>5.646703</td>\n",
|
||
" <td>1642</td>\n",
|
||
" <td>5.644310</td>\n",
|
||
" <td>119</td>\n",
|
||
" <td>5.577740</td>\n",
|
||
" <td>1189</td>\n",
|
||
" <td>5.569099</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>939</th>\n",
|
||
" <td>940</td>\n",
|
||
" <td>814</td>\n",
|
||
" <td>5.462440</td>\n",
|
||
" <td>1491</td>\n",
|
||
" <td>5.245416</td>\n",
|
||
" <td>1536</td>\n",
|
||
" <td>5.201522</td>\n",
|
||
" <td>1306</td>\n",
|
||
" <td>5.181936</td>\n",
|
||
" <td>1599</td>\n",
|
||
" <td>5.002481</td>\n",
|
||
" <td>1467</td>\n",
|
||
" <td>4.944190</td>\n",
|
||
" <td>1080</td>\n",
|
||
" <td>4.872499</td>\n",
|
||
" <td>1642</td>\n",
|
||
" <td>4.870106</td>\n",
|
||
" <td>119</td>\n",
|
||
" <td>4.803536</td>\n",
|
||
" <td>1189</td>\n",
|
||
" <td>4.794895</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>940</th>\n",
|
||
" <td>941</td>\n",
|
||
" <td>814</td>\n",
|
||
" <td>5.894539</td>\n",
|
||
" <td>1491</td>\n",
|
||
" <td>5.677515</td>\n",
|
||
" <td>1536</td>\n",
|
||
" <td>5.633621</td>\n",
|
||
" <td>1306</td>\n",
|
||
" <td>5.614035</td>\n",
|
||
" <td>1599</td>\n",
|
||
" <td>5.434579</td>\n",
|
||
" <td>1467</td>\n",
|
||
" <td>5.376289</td>\n",
|
||
" <td>1080</td>\n",
|
||
" <td>5.304598</td>\n",
|
||
" <td>1642</td>\n",
|
||
" <td>5.302204</td>\n",
|
||
" <td>119</td>\n",
|
||
" <td>5.235635</td>\n",
|
||
" <td>1189</td>\n",
|
||
" <td>5.226994</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>941</th>\n",
|
||
" <td>942</td>\n",
|
||
" <td>814</td>\n",
|
||
" <td>6.164380</td>\n",
|
||
" <td>1491</td>\n",
|
||
" <td>5.947356</td>\n",
|
||
" <td>1536</td>\n",
|
||
" <td>5.903462</td>\n",
|
||
" <td>1306</td>\n",
|
||
" <td>5.883876</td>\n",
|
||
" <td>1599</td>\n",
|
||
" <td>5.704421</td>\n",
|
||
" <td>1467</td>\n",
|
||
" <td>5.646130</td>\n",
|
||
" <td>1080</td>\n",
|
||
" <td>5.574439</td>\n",
|
||
" <td>1642</td>\n",
|
||
" <td>5.572046</td>\n",
|
||
" <td>119</td>\n",
|
||
" <td>5.505476</td>\n",
|
||
" <td>1189</td>\n",
|
||
" <td>5.496835</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>942</th>\n",
|
||
" <td>943</td>\n",
|
||
" <td>814</td>\n",
|
||
" <td>5.284783</td>\n",
|
||
" <td>1491</td>\n",
|
||
" <td>5.067759</td>\n",
|
||
" <td>1536</td>\n",
|
||
" <td>5.023865</td>\n",
|
||
" <td>1306</td>\n",
|
||
" <td>5.004279</td>\n",
|
||
" <td>1599</td>\n",
|
||
" <td>4.824823</td>\n",
|
||
" <td>1467</td>\n",
|
||
" <td>4.766533</td>\n",
|
||
" <td>1080</td>\n",
|
||
" <td>4.694842</td>\n",
|
||
" <td>1642</td>\n",
|
||
" <td>4.692448</td>\n",
|
||
" <td>119</td>\n",
|
||
" <td>4.625879</td>\n",
|
||
" <td>1189</td>\n",
|
||
" <td>4.617238</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"<p>943 rows × 21 columns</p>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" 0 1 2 3 ... 17 18 19 20\n",
|
||
"0 1 814 5.449584 1491 ... 1189 4.782040 1500 4.742546\n",
|
||
"1 2 814 5.580813 1491 ... 119 4.921909 1189 4.913268\n",
|
||
"2 3 814 4.615469 1491 ... 119 3.956565 1189 3.947924\n",
|
||
"3 4 814 6.244539 1491 ... 119 5.585635 1189 5.576994\n",
|
||
"4 5 814 4.738510 1491 ... 119 4.079607 1189 4.070965\n",
|
||
".. ... ... ... ... ... ... ... ... ...\n",
|
||
"938 939 814 6.236644 1491 ... 119 5.577740 1189 5.569099\n",
|
||
"939 940 814 5.462440 1491 ... 119 4.803536 1189 4.794895\n",
|
||
"940 941 814 5.894539 1491 ... 119 5.235635 1189 5.226994\n",
|
||
"941 942 814 6.164380 1491 ... 119 5.505476 1189 5.496835\n",
|
||
"942 943 814 5.284783 1491 ... 119 4.625879 1189 4.617238\n",
|
||
"\n",
|
||
"[943 rows x 21 columns]"
|
||
]
|
||
},
|
||
"metadata": {
|
||
"tags": []
|
||
}
|
||
},
|
||
{
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Estimations:\n"
|
||
],
|
||
"name": "stdout"
|
||
},
|
||
{
|
||
"output_type": "display_data",
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>user</th>\n",
|
||
" <th>item</th>\n",
|
||
" <th>est_score</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>1</td>\n",
|
||
" <td>5</td>\n",
|
||
" <td>3.457161</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>1</th>\n",
|
||
" <td>1</td>\n",
|
||
" <td>10</td>\n",
|
||
" <td>3.798540</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>2</th>\n",
|
||
" <td>1</td>\n",
|
||
" <td>25</td>\n",
|
||
" <td>3.435415</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>3</th>\n",
|
||
" <td>1</td>\n",
|
||
" <td>32</td>\n",
|
||
" <td>3.732018</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>4</th>\n",
|
||
" <td>1</td>\n",
|
||
" <td>33</td>\n",
|
||
" <td>3.531991</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>...</th>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>19995</th>\n",
|
||
" <td>943</td>\n",
|
||
" <td>928</td>\n",
|
||
" <td>2.907189</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>19996</th>\n",
|
||
" <td>943</td>\n",
|
||
" <td>1067</td>\n",
|
||
" <td>3.485929</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>19997</th>\n",
|
||
" <td>943</td>\n",
|
||
" <td>1074</td>\n",
|
||
" <td>2.861988</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>19998</th>\n",
|
||
" <td>943</td>\n",
|
||
" <td>1188</td>\n",
|
||
" <td>2.727428</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>19999</th>\n",
|
||
" <td>943</td>\n",
|
||
" <td>1228</td>\n",
|
||
" <td>2.568442</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"<p>20000 rows × 3 columns</p>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" user item est_score\n",
|
||
"0 1 5 3.457161\n",
|
||
"1 1 10 3.798540\n",
|
||
"2 1 25 3.435415\n",
|
||
"3 1 32 3.732018\n",
|
||
"4 1 33 3.531991\n",
|
||
"... ... ... ...\n",
|
||
"19995 943 928 2.907189\n",
|
||
"19996 943 1067 3.485929\n",
|
||
"19997 943 1074 2.861988\n",
|
||
"19998 943 1188 2.727428\n",
|
||
"19999 943 1228 2.568442\n",
|
||
"\n",
|
||
"[20000 rows x 3 columns]"
|
||
]
|
||
},
|
||
"metadata": {
|
||
"tags": []
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"id": "QSCe2WVVKkFG",
|
||
"colab_type": "text"
|
||
},
|
||
"source": [
|
||
"# Sample recommendations"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"metadata": {
|
||
"id": "uT8s_WQOKkFG",
|
||
"colab_type": "code",
|
||
"outputId": "13361cd0-f54a-4923-9303-422e501d7543",
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 890
|
||
}
|
||
},
|
||
"source": [
|
||
"train=pd.read_csv('./Datasets/ml-100k/train.csv', sep='\\t', header=None, names=['user', 'item', 'rating', 'timestamp'])\n",
|
||
"items=pd.read_csv('./Datasets/ml-100k/movies.csv')\n",
|
||
"\n",
|
||
"user=random.choice(list(set(train['user'])))\n",
|
||
"\n",
|
||
"train_content=pd.merge(train, items, left_on='item', right_on='id')\n",
|
||
"\n",
|
||
"print('Here is what user rated high:')\n",
|
||
"display(train_content[train_content['user']==user][['user', 'rating', 'title', 'genres']]\\\n",
|
||
" .sort_values(by='rating', ascending=False)[:15])\n",
|
||
"\n",
|
||
"reco = np.loadtxt('Recommendations generated/ml-100k/Self_BaselineUI_reco.csv', delimiter=',')\n",
|
||
"items=pd.read_csv('./Datasets/ml-100k/movies.csv')\n",
|
||
"\n",
|
||
"# Let's ignore scores - they are not used in evaluation: \n",
|
||
"reco_users=reco[:,:1]\n",
|
||
"reco_items=reco[:,1::2]\n",
|
||
"# Let's put them into one array\n",
|
||
"reco=np.concatenate((reco_users, reco_items), axis=1)\n",
|
||
"\n",
|
||
"# Let's rebuild it user-item dataframe\n",
|
||
"recommended=[]\n",
|
||
"for row in reco:\n",
|
||
" for rec_nb, entry in enumerate(row[1:]):\n",
|
||
" recommended.append((row[0], rec_nb+1, entry))\n",
|
||
"recommended=pd.DataFrame(recommended, columns=['user','rec_nb', 'item'])\n",
|
||
"\n",
|
||
"recommended_content=pd.merge(recommended, items, left_on='item', right_on='id')\n",
|
||
"\n",
|
||
"print('Here is what we recommend:')\n",
|
||
"recommended_content[recommended_content['user']==user][['user', 'rec_nb', 'title', 'genres']].sort_values(by='rec_nb')"
|
||
],
|
||
"execution_count": 0,
|
||
"outputs": [
|
||
{
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Here is what user rated high:\n"
|
||
],
|
||
"name": "stdout"
|
||
},
|
||
{
|
||
"output_type": "display_data",
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>user</th>\n",
|
||
" <th>rating</th>\n",
|
||
" <th>title</th>\n",
|
||
" <th>genres</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>5529</th>\n",
|
||
" <td>181</td>\n",
|
||
" <td>5</td>\n",
|
||
" <td>Jerry Maguire (1996)</td>\n",
|
||
" <td>Drama, Romance</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>38687</th>\n",
|
||
" <td>181</td>\n",
|
||
" <td>4</td>\n",
|
||
" <td>First Wives Club, The (1996)</td>\n",
|
||
" <td>Comedy</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>19390</th>\n",
|
||
" <td>181</td>\n",
|
||
" <td>4</td>\n",
|
||
" <td>Scream (1996)</td>\n",
|
||
" <td>Horror, Thriller</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>63720</th>\n",
|
||
" <td>181</td>\n",
|
||
" <td>4</td>\n",
|
||
" <td>Mirror Has Two Faces, The (1996)</td>\n",
|
||
" <td>Comedy, Romance</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>57287</th>\n",
|
||
" <td>181</td>\n",
|
||
" <td>4</td>\n",
|
||
" <td>Evita (1996)</td>\n",
|
||
" <td>Drama, Musical</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>57106</th>\n",
|
||
" <td>181</td>\n",
|
||
" <td>4</td>\n",
|
||
" <td>Up Close and Personal (1996)</td>\n",
|
||
" <td>Drama, Romance</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>24776</th>\n",
|
||
" <td>181</td>\n",
|
||
" <td>4</td>\n",
|
||
" <td>Gattaca (1997)</td>\n",
|
||
" <td>Drama, Sci-Fi, Thriller</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>11778</th>\n",
|
||
" <td>181</td>\n",
|
||
" <td>4</td>\n",
|
||
" <td>Mission: Impossible (1996)</td>\n",
|
||
" <td>Action, Adventure, Mystery</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>68469</th>\n",
|
||
" <td>181</td>\n",
|
||
" <td>4</td>\n",
|
||
" <td>Eye for an Eye (1996)</td>\n",
|
||
" <td>Drama, Thriller</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>25908</th>\n",
|
||
" <td>181</td>\n",
|
||
" <td>4</td>\n",
|
||
" <td>Dead Man Walking (1995)</td>\n",
|
||
" <td>Drama</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>9811</th>\n",
|
||
" <td>181</td>\n",
|
||
" <td>4</td>\n",
|
||
" <td>Time to Kill, A (1996)</td>\n",
|
||
" <td>Drama</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>21011</th>\n",
|
||
" <td>181</td>\n",
|
||
" <td>4</td>\n",
|
||
" <td>I Know What You Did Last Summer (1997)</td>\n",
|
||
" <td>Horror, Mystery, Thriller</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>33572</th>\n",
|
||
" <td>181</td>\n",
|
||
" <td>4</td>\n",
|
||
" <td>Independence Day (ID4) (1996)</td>\n",
|
||
" <td>Action, Sci-Fi, War</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>44631</th>\n",
|
||
" <td>181</td>\n",
|
||
" <td>4</td>\n",
|
||
" <td>Ransom (1996)</td>\n",
|
||
" <td>Drama, Thriller</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>46726</th>\n",
|
||
" <td>181</td>\n",
|
||
" <td>4</td>\n",
|
||
" <td>Hunchback of Notre Dame, The (1996)</td>\n",
|
||
" <td>Animation, Children's, Musical</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" user ... genres\n",
|
||
"5529 181 ... Drama, Romance\n",
|
||
"38687 181 ... Comedy\n",
|
||
"19390 181 ... Horror, Thriller\n",
|
||
"63720 181 ... Comedy, Romance\n",
|
||
"57287 181 ... Drama, Musical\n",
|
||
"57106 181 ... Drama, Romance\n",
|
||
"24776 181 ... Drama, Sci-Fi, Thriller\n",
|
||
"11778 181 ... Action, Adventure, Mystery\n",
|
||
"68469 181 ... Drama, Thriller\n",
|
||
"25908 181 ... Drama\n",
|
||
"9811 181 ... Drama\n",
|
||
"21011 181 ... Horror, Mystery, Thriller\n",
|
||
"33572 181 ... Action, Sci-Fi, War\n",
|
||
"44631 181 ... Drama, Thriller\n",
|
||
"46726 181 ... Animation, Children's, Musical\n",
|
||
"\n",
|
||
"[15 rows x 4 columns]"
|
||
]
|
||
},
|
||
"metadata": {
|
||
"tags": []
|
||
}
|
||
},
|
||
{
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Here is what we recommend:\n"
|
||
],
|
||
"name": "stdout"
|
||
},
|
||
{
|
||
"output_type": "execute_result",
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>user</th>\n",
|
||
" <th>rec_nb</th>\n",
|
||
" <th>title</th>\n",
|
||
" <th>genres</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>179</th>\n",
|
||
" <td>181.0</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>Great Day in Harlem, A (1994)</td>\n",
|
||
" <td>Documentary</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>1122</th>\n",
|
||
" <td>181.0</td>\n",
|
||
" <td>2</td>\n",
|
||
" <td>Tough and Deadly (1995)</td>\n",
|
||
" <td>Action, Drama, Thriller</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>2064</th>\n",
|
||
" <td>181.0</td>\n",
|
||
" <td>3</td>\n",
|
||
" <td>Aiqing wansui (1994)</td>\n",
|
||
" <td>Drama</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>3005</th>\n",
|
||
" <td>181.0</td>\n",
|
||
" <td>4</td>\n",
|
||
" <td>Delta of Venus (1994)</td>\n",
|
||
" <td>Drama</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>3948</th>\n",
|
||
" <td>181.0</td>\n",
|
||
" <td>5</td>\n",
|
||
" <td>Someone Else's America (1995)</td>\n",
|
||
" <td>Drama</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>4890</th>\n",
|
||
" <td>181.0</td>\n",
|
||
" <td>6</td>\n",
|
||
" <td>Saint of Fort Washington, The (1993)</td>\n",
|
||
" <td>Drama</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>5831</th>\n",
|
||
" <td>181.0</td>\n",
|
||
" <td>7</td>\n",
|
||
" <td>Celestial Clockwork (1994)</td>\n",
|
||
" <td>Comedy</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>6774</th>\n",
|
||
" <td>181.0</td>\n",
|
||
" <td>8</td>\n",
|
||
" <td>Some Mother's Son (1996)</td>\n",
|
||
" <td>Drama</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>8668</th>\n",
|
||
" <td>181.0</td>\n",
|
||
" <td>9</td>\n",
|
||
" <td>Maya Lin: A Strong Clear Vision (1994)</td>\n",
|
||
" <td>Documentary</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>7714</th>\n",
|
||
" <td>181.0</td>\n",
|
||
" <td>10</td>\n",
|
||
" <td>Prefontaine (1997)</td>\n",
|
||
" <td>Drama</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" user ... genres\n",
|
||
"179 181.0 ... Documentary\n",
|
||
"1122 181.0 ... Action, Drama, Thriller\n",
|
||
"2064 181.0 ... Drama\n",
|
||
"3005 181.0 ... Drama\n",
|
||
"3948 181.0 ... Drama\n",
|
||
"4890 181.0 ... Drama\n",
|
||
"5831 181.0 ... Comedy\n",
|
||
"6774 181.0 ... Drama\n",
|
||
"8668 181.0 ... Documentary\n",
|
||
"7714 181.0 ... Drama\n",
|
||
"\n",
|
||
"[10 rows x 4 columns]"
|
||
]
|
||
},
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"execution_count": 24
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"id": "gFsye0sAKkFQ",
|
||
"colab_type": "text"
|
||
},
|
||
"source": [
|
||
"# project task 3: implement some other evaluation measure"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"metadata": {
|
||
"id": "3bhpq4zOKkFR",
|
||
"colab_type": "code",
|
||
"colab": {}
|
||
},
|
||
"source": [
|
||
"# it may be your idea, modification of what we have already implemented \n",
|
||
"# (for example Hit2 rate which would count as a success users whoreceived at least 2 relevant recommendations) \n",
|
||
"# or something well-known\n",
|
||
"# expected output: modification of evaluation_measures.py such that evaluate_all will also display your measure"
|
||
],
|
||
"execution_count": 0,
|
||
"outputs": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"metadata": {
|
||
"id": "8s81gSB3KkFU",
|
||
"colab_type": "code",
|
||
"outputId": "720e0c84-ef77-4c24-9ed9-df7697e9b98a",
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 199
|
||
}
|
||
},
|
||
"source": [
|
||
"dir_path=\"Recommendations generated/ml-100k/\"\n",
|
||
"super_reactions=[4,5]\n",
|
||
"test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None)\n",
|
||
"\n",
|
||
"ev.evaluate_all(test, dir_path, super_reactions)"
|
||
],
|
||
"execution_count": 0,
|
||
"outputs": [
|
||
{
|
||
"output_type": "stream",
|
||
"text": [
|
||
"943it [00:00, 6725.92it/s]\n",
|
||
"943it [00:00, 6395.43it/s]\n"
|
||
],
|
||
"name": "stderr"
|
||
},
|
||
{
|
||
"output_type": "execute_result",
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>Model</th>\n",
|
||
" <th>RMSE</th>\n",
|
||
" <th>MAE</th>\n",
|
||
" <th>precision</th>\n",
|
||
" <th>recall</th>\n",
|
||
" <th>F_1</th>\n",
|
||
" <th>F_05</th>\n",
|
||
" <th>precision_super</th>\n",
|
||
" <th>recall_super</th>\n",
|
||
" <th>NDCG</th>\n",
|
||
" <th>mAP</th>\n",
|
||
" <th>MRR</th>\n",
|
||
" <th>LAUC</th>\n",
|
||
" <th>HR</th>\n",
|
||
" <th>HR2</th>\n",
|
||
" <th>Reco in test</th>\n",
|
||
" <th>Test coverage</th>\n",
|
||
" <th>Shannon</th>\n",
|
||
" <th>Gini</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>Ready_Baseline</td>\n",
|
||
" <td>0.949459</td>\n",
|
||
" <td>0.752487</td>\n",
|
||
" <td>0.091410</td>\n",
|
||
" <td>0.037652</td>\n",
|
||
" <td>0.046030</td>\n",
|
||
" <td>0.061286</td>\n",
|
||
" <td>0.079614</td>\n",
|
||
" <td>0.056463</td>\n",
|
||
" <td>0.095957</td>\n",
|
||
" <td>0.043178</td>\n",
|
||
" <td>0.198193</td>\n",
|
||
" <td>0.515501</td>\n",
|
||
" <td>0.437964</td>\n",
|
||
" <td>0.239661</td>\n",
|
||
" <td>1.00000</td>\n",
|
||
" <td>0.033911</td>\n",
|
||
" <td>2.836513</td>\n",
|
||
" <td>0.991139</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>Self_BaselineUI</td>\n",
|
||
" <td>0.967585</td>\n",
|
||
" <td>0.762740</td>\n",
|
||
" <td>0.000954</td>\n",
|
||
" <td>0.000170</td>\n",
|
||
" <td>0.000278</td>\n",
|
||
" <td>0.000463</td>\n",
|
||
" <td>0.000644</td>\n",
|
||
" <td>0.000189</td>\n",
|
||
" <td>0.000752</td>\n",
|
||
" <td>0.000168</td>\n",
|
||
" <td>0.001677</td>\n",
|
||
" <td>0.496424</td>\n",
|
||
" <td>0.009544</td>\n",
|
||
" <td>0.000000</td>\n",
|
||
" <td>0.60053</td>\n",
|
||
" <td>0.005051</td>\n",
|
||
" <td>1.803126</td>\n",
|
||
" <td>0.996380</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" Model RMSE MAE ... Test coverage Shannon Gini\n",
|
||
"0 Ready_Baseline 0.949459 0.752487 ... 0.033911 2.836513 0.991139\n",
|
||
"0 Self_BaselineUI 0.967585 0.762740 ... 0.005051 1.803126 0.996380\n",
|
||
"\n",
|
||
"[2 rows x 19 columns]"
|
||
]
|
||
},
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"execution_count": 25
|
||
}
|
||
]
|
||
}
|
||
]
|
||
} |