{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.9" }, "colab": { "name": "P2. Evaluation.ipynb", "provenance": [], "collapsed_sections": [], "toc_visible": true } }, "cells": [ { "cell_type": "markdown", "metadata": { "id": "XsGH6668KkEb", "colab_type": "text" }, "source": [ "### Prepare test set" ] }, { "cell_type": "code", "metadata": { "id": "6i5evQN1KkEc", "colab_type": "code", "colab": {} }, "source": [ "import pandas as pd\n", "import numpy as np\n", "import scipy.sparse as sparse\n", "from collections import defaultdict\n", "from itertools import chain\n", "import random\n", "from tqdm import tqdm\n", "\n", "# In evaluation we do not load train set - it is not needed\n", "test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None)\n", "test.columns=['user', 'item', 'rating', 'timestamp']\n", "\n", "test['user_code'] = test['user'].astype(\"category\").cat.codes\n", "test['item_code'] = test['item'].astype(\"category\").cat.codes\n", "\n", "user_code_id = dict(enumerate(test['user'].astype(\"category\").cat.categories))\n", "user_id_code = dict((v, k) for k, v in user_code_id.items())\n", "item_code_id = dict(enumerate(test['item'].astype(\"category\").cat.categories))\n", "item_id_code = dict((v, k) for k, v in item_code_id.items())\n", "\n", "test_ui = sparse.csr_matrix((test['rating'], (test['user_code'], test['item_code'])))" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "o1f-bErIK3Xx", "colab_type": "code", "colab": {} }, "source": [ "import os\n", "if not os.path.exists('./Recommendations generated/'):\n", " os.mkdir('./Recommendations generated/')\n", " os.mkdir('./Recommendations generated/ml-100k/')\n", " os.mkdir('./Recommendations generated/toy-example/')" ], "execution_count": 0, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "e-h95H_wKkEg", "colab_type": "text" }, "source": [ "### Estimations metrics" ] }, { "cell_type": "code", "metadata": { "id": "Pg63ftwyKkEg", "colab_type": "code", "colab": {} }, "source": [ "estimations_df=pd.read_csv('Recommendations generated/ml-100k/Ready_Baseline_estimations.csv', header=None)\n", "estimations_df.columns=['user', 'item' ,'score']\n", "\n", "estimations_df['user_code']=[user_id_code[user] for user in estimations_df['user']]\n", "estimations_df['item_code']=[item_id_code[item] for item in estimations_df['item']]\n", "estimations=sparse.csr_matrix((estimations_df['score'], (estimations_df['user_code'], estimations_df['item_code'])), shape=test_ui.shape)" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "g4QKDwfQKkEj", "colab_type": "code", "colab": {} }, "source": [ "def estimations_metrics(test_ui, estimations):\n", " result=[]\n", "\n", " RMSE=(np.sum((estimations.data-test_ui.data)**2)/estimations.nnz)**(1/2)\n", " result.append(['RMSE', RMSE])\n", "\n", " MAE=np.sum(abs(estimations.data-test_ui.data))/estimations.nnz\n", " result.append(['MAE', MAE])\n", " \n", " df_result=(pd.DataFrame(list(zip(*result))[1])).T\n", " df_result.columns=list(zip(*result))[0]\n", " return df_result" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "LTDMTTiiKkEl", "colab_type": "code", "outputId": "7ebd1465-3dc6-404d-9666-67897e4d2750", "colab": { "base_uri": "https://localhost:8080/", "height": 80 } }, "source": [ "# in case of error (in the laboratories) you might have to switch to the other version of pandas\n", "# try !pip3 install pandas=='1.0.3' (or pip if you use python 2) and restart the kernel\n", "\n", "estimations_metrics(test_ui, estimations)" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
RMSEMAE
00.9494590.752487
\n", "
" ], "text/plain": [ " RMSE MAE\n", "0 0.949459 0.752487" ] }, "metadata": { "tags": [] }, "execution_count": 6 } ] }, { "cell_type": "markdown", "metadata": { "id": "rSgzlQ-iKkEp", "colab_type": "text" }, "source": [ "### Ranking metrics" ] }, { "cell_type": "code", "metadata": { "id": "BX6YbfwgKkEp", "colab_type": "code", "outputId": "be746705-2a4d-4e53-f616-a3f4876ce68b", "colab": { "base_uri": "https://localhost:8080/", "height": 136 } }, "source": [ "import numpy as np\n", "reco = np.loadtxt('Recommendations generated/ml-100k/Ready_Baseline_reco.csv', delimiter=',')\n", "# Let's ignore scores - they are not used in evaluation: \n", "users=reco[:,:1]\n", "items=reco[:,1::2]\n", "# Let's use inner ids instead of real ones\n", "users=np.vectorize(lambda x: user_id_code.setdefault(x, -1))(users)\n", "items=np.vectorize(lambda x: item_id_code.setdefault(x, -1))(items) # maybe items we recommend are not in test set\n", "# Let's put them into one array\n", "reco=np.concatenate((users, items), axis=1)\n", "reco" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "array([[663, 475, 62, ..., 472, 269, 503],\n", " [ 48, 313, 475, ..., 591, 175, 466],\n", " [351, 313, 475, ..., 591, 175, 466],\n", " ...,\n", " [259, 313, 475, ..., 11, 591, 175],\n", " [ 33, 313, 475, ..., 11, 591, 175],\n", " [ 77, 313, 475, ..., 11, 591, 175]])" ] }, "metadata": { "tags": [] }, "execution_count": 7 } ] }, { "cell_type": "code", "metadata": { "id": "NjVfzif8KkEs", "colab_type": "code", "colab": {} }, "source": [ "def ranking_metrics(test_ui, reco, super_reactions=[], topK=10):\n", " \n", " nb_items=test_ui.shape[1]\n", " relevant_users, super_relevant_users, prec, rec, F_1, F_05, prec_super, rec_super, ndcg, mAP, MRR, LAUC, HR=\\\n", " 0,0,0,0,0,0,0,0,0,0,0,0,0\n", " \n", " cg = (1.0 / np.log2(np.arange(2, topK + 2)))\n", " cg_sum = np.cumsum(cg)\n", " \n", " for (nb_user, user) in tqdm(enumerate(reco[:,0])):\n", " u_rated_items=test_ui.indices[test_ui.indptr[user]:test_ui.indptr[user+1]]\n", " nb_u_rated_items=len(u_rated_items)\n", " if nb_u_rated_items>0: # skip users with no items in test set (still possible that there will be no super items)\n", " relevant_users+=1\n", " \n", " u_super_items=u_rated_items[np.vectorize(lambda x: x in super_reactions)\\\n", " (test_ui.data[test_ui.indptr[user]:test_ui.indptr[user+1]])]\n", " # more natural seems u_super_items=[item for item in u_rated_items if test_ui[user,item] in super_reactions]\n", " # but accesing test_ui[user,item] is expensive -we should avoid doing it\n", " if len(u_super_items)>0:\n", " super_relevant_users+=1\n", " \n", " user_successes=np.zeros(topK)\n", " nb_user_successes=0\n", " user_super_successes=np.zeros(topK)\n", " nb_user_super_successes=0\n", " \n", " # evaluation\n", " for (item_position,item) in enumerate(reco[nb_user,1:topK+1]):\n", " if item in u_rated_items:\n", " user_successes[item_position]=1\n", " nb_user_successes+=1\n", " if item in u_super_items:\n", " user_super_successes[item_position]=1\n", " nb_user_super_successes+=1\n", " \n", " prec_u=nb_user_successes/topK \n", " prec+=prec_u\n", " \n", " rec_u=nb_user_successes/nb_u_rated_items\n", " rec+=rec_u\n", " \n", " F_1+=2*(prec_u*rec_u)/(prec_u+rec_u) if prec_u+rec_u>0 else 0\n", " F_05+=(0.5**2+1)*(prec_u*rec_u)/(0.5**2*prec_u+rec_u) if prec_u+rec_u>0 else 0\n", " \n", " prec_super+=nb_user_super_successes/topK\n", " rec_super+=nb_user_super_successes/max(len(u_super_items),1) # to set 0 if no super items\n", " ndcg+=np.dot(user_successes,cg)/cg_sum[min(topK, nb_u_rated_items)-1]\n", " \n", " cumsum_successes=np.cumsum(user_successes)\n", " mAP+=np.dot(cumsum_successes/np.arange(1,topK+1), user_successes)/min(topK, nb_u_rated_items)\n", " MRR+=1/(user_successes.nonzero()[0][0]+1) if user_successes.nonzero()[0].size>0 else 0\n", " LAUC+=(np.dot(cumsum_successes, 1-user_successes)+\\\n", " (nb_user_successes+nb_u_rated_items)/2*((nb_items-nb_u_rated_items)-(topK-nb_user_successes)))/\\\n", " ((nb_items-nb_u_rated_items)*nb_u_rated_items)\n", " \n", " HR+=nb_user_successes>0\n", " \n", " \n", " result=[]\n", " result.append(('precision', prec/relevant_users))\n", " result.append(('recall', rec/relevant_users))\n", " result.append(('F_1', F_1/relevant_users))\n", " result.append(('F_05', F_05/relevant_users))\n", " result.append(('precision_super', prec_super/super_relevant_users))\n", " result.append(('recall_super', rec_super/super_relevant_users))\n", " result.append(('NDCG', ndcg/relevant_users))\n", " result.append(('mAP', mAP/relevant_users))\n", " result.append(('MRR', MRR/relevant_users))\n", " result.append(('LAUC', LAUC/relevant_users))\n", " result.append(('HR', HR/relevant_users))\n", "\n", " df_result=(pd.DataFrame(list(zip(*result))[1])).T\n", " df_result.columns=list(zip(*result))[0]\n", " return df_result" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "xspx_TyWKkEv", "colab_type": "code", "outputId": "41145c03-763d-4584-e7d4-f5ec16a5507f", "colab": { "base_uri": "https://localhost:8080/", "height": 97 } }, "source": [ "ranking_metrics(test_ui, reco, super_reactions=[4,5], topK=10)" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "943it [00:00, 6981.89it/s]\n" ], "name": "stderr" }, { "output_type": "execute_result", "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
precisionrecallF_1F_05precision_superrecall_superNDCGmAPMRRLAUCHR
00.091410.0376520.046030.0612860.0796140.0564630.0959570.0431780.1981930.5155010.437964
\n", "
" ], "text/plain": [ " precision recall F_1 ... MRR LAUC HR\n", "0 0.09141 0.037652 0.04603 ... 0.198193 0.515501 0.437964\n", "\n", "[1 rows x 11 columns]" ] }, "metadata": { "tags": [] }, "execution_count": 9 } ] }, { "cell_type": "markdown", "metadata": { "id": "sIV-r9MWKkEy", "colab_type": "text" }, "source": [ "### Diversity metrics" ] }, { "cell_type": "code", "metadata": { "id": "XOtNDMiRKkEy", "colab_type": "code", "colab": {} }, "source": [ "def diversity_metrics(test_ui, reco, topK=10):\n", " \n", " frequencies=defaultdict(int)\n", " \n", " # let's assign 0 to all items in test set\n", " for item in list(set(test_ui.indices)):\n", " frequencies[item]=0\n", " \n", " # counting frequencies\n", " for item in reco[:,1:].flat:\n", " frequencies[item]+=1\n", " \n", " nb_reco_outside_test=frequencies[-1]\n", " del frequencies[-1]\n", " \n", " frequencies=np.array(list(frequencies.values()))\n", " \n", " nb_rec_items=len(frequencies[frequencies>0])\n", " nb_reco_inside_test=np.sum(frequencies)\n", " \n", " frequencies=frequencies/np.sum(frequencies)\n", " frequencies=np.sort(frequencies)\n", " \n", " with np.errstate(divide='ignore'): # let's put zeros put items with 0 frequency and ignore division warning\n", " log_frequencies=np.nan_to_num(np.log(frequencies), posinf=0, neginf=0)\n", " \n", " result=[]\n", " result.append(('Reco in test', nb_reco_inside_test/(nb_reco_inside_test+nb_reco_outside_test)))\n", " result.append(('Test coverage', nb_rec_items/test_ui.shape[1]))\n", " result.append(('Shannon', -np.dot(frequencies, log_frequencies)))\n", " result.append(('Gini', np.dot(frequencies, np.arange(1-len(frequencies), len(frequencies), 2))/(len(frequencies)-1)))\n", " \n", " df_result=(pd.DataFrame(list(zip(*result))[1])).T\n", " df_result.columns=list(zip(*result))[0]\n", " return df_result" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "LmWMC0beKkE0", "colab_type": "code", "outputId": "99e50bfb-b99d-4e54-b34e-9726d421f3e0", "colab": { "base_uri": "https://localhost:8080/", "height": 80 } }, "source": [ "# in case of errors try !pip3 install numpy==1.18.4 (or pip if you use python 2) and restart the kernel\n", "\n", "import evaluation_measures as ev\n", "import imp\n", "imp.reload(ev)\n", "\n", "x=diversity_metrics(test_ui, reco, topK=10)\n", "x" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Reco in testTest coverageShannonGini
01.00.0339112.8365130.991139
\n", "
" ], "text/plain": [ " Reco in test Test coverage Shannon Gini\n", "0 1.0 0.033911 2.836513 0.991139" ] }, "metadata": { "tags": [] }, "execution_count": 13 } ] }, { "cell_type": "markdown", "metadata": { "id": "KWtRicX0KkE3", "colab_type": "text" }, "source": [ "# To be used in other notebooks" ] }, { "cell_type": "code", "metadata": { "id": "1ozOsZWMKkE3", "colab_type": "code", "outputId": "79b554c9-ec53-4565-f55d-c8343876cb67", "colab": { "base_uri": "https://localhost:8080/", "height": 114 } }, "source": [ "import evaluation_measures as ev\n", "import imp\n", "imp.reload(ev)\n", "\n", "estimations_df=pd.read_csv('Recommendations generated/ml-100k/Ready_Baseline_estimations.csv', header=None)\n", "reco=np.loadtxt('Recommendations generated/ml-100k/Ready_Baseline_reco.csv', delimiter=',')\n", "\n", "ev.evaluate(test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None),\n", " estimations_df=estimations_df, \n", " reco=reco,\n", " super_reactions=[4,5])\n", "#also you can just type ev.evaluate_all(estimations_df, reco) - I put above values as default" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "943it [00:00, 6721.23it/s]\n" ], "name": "stderr" }, { "output_type": "execute_result", "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
RMSEMAEprecisionrecallF_1F_05precision_superrecall_superNDCGmAPMRRLAUCHRHR2Reco in testTest coverageShannonGini
00.9494590.7524870.091410.0376520.046030.0612860.0796140.0564630.0959570.0431780.1981930.5155010.4379640.2396611.00.0339112.8365130.991139
\n", "
" ], "text/plain": [ " RMSE MAE precision ... Test coverage Shannon Gini\n", "0 0.949459 0.752487 0.09141 ... 0.033911 2.836513 0.991139\n", "\n", "[1 rows x 18 columns]" ] }, "metadata": { "tags": [] }, "execution_count": 14 } ] }, { "cell_type": "code", "metadata": { "id": "i1qKPl8LKkE6", "colab_type": "code", "outputId": "5f167923-0533-45a7-9536-5cc2e162f79e", "colab": { "base_uri": "https://localhost:8080/", "height": 34 } }, "source": [ "import evaluation_measures as ev\n", "import imp\n", "imp.reload(ev)\n", "\n", "dir_path=\"Recommendations generated/ml-100k/\"\n", "super_reactions=[4,5]\n", "test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None)\n", "\n", "df=ev.evaluate_all(test, dir_path, super_reactions)\n", "#also you can just type ev.evaluate_all() - I put above values as default" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "943it [00:00, 7140.87it/s]\n" ], "name": "stderr" } ] }, { "cell_type": "code", "metadata": { "id": "ADUVNxYnKkE9", "colab_type": "code", "outputId": "2d7a1008-8eed-425a-a3a0-4602fe5d5b32", "colab": { "base_uri": "https://localhost:8080/", "height": 80 } }, "source": [ "df.iloc[:,:9]" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ModelRMSEMAEprecisionrecallF_1F_05precision_superrecall_super
0Ready_Baseline0.9494590.7524870.091410.0376520.046030.0612860.0796140.056463
\n", "
" ], "text/plain": [ " Model RMSE MAE ... F_05 precision_super recall_super\n", "0 Ready_Baseline 0.949459 0.752487 ... 0.061286 0.079614 0.056463\n", "\n", "[1 rows x 9 columns]" ] }, "metadata": { "tags": [] }, "execution_count": 16 } ] }, { "cell_type": "code", "metadata": { "id": "QmTqLVNLKkFA", "colab_type": "code", "outputId": "eba7e18f-d73e-4108-a5f5-3fca175db010", "colab": { "base_uri": "https://localhost:8080/", "height": 80 } }, "source": [ "df.iloc[:,np.append(0,np.arange(9, df.shape[1]))]" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ModelNDCGmAPMRRLAUCHRHR2Reco in testTest coverageShannonGini
0Ready_Baseline0.0959570.0431780.1981930.5155010.4379640.2396611.00.0339112.8365130.991139
\n", "
" ], "text/plain": [ " Model NDCG mAP ... Test coverage Shannon Gini\n", "0 Ready_Baseline 0.095957 0.043178 ... 0.033911 2.836513 0.991139\n", "\n", "[1 rows x 11 columns]" ] }, "metadata": { "tags": [] }, "execution_count": 17 } ] }, { "cell_type": "code", "metadata": { "id": "ADNCFV1fOzER", "colab_type": "code", "outputId": "bd3b8f9b-57e6-4b67-af7c-d7abea1c02b0", "colab": { "base_uri": "https://localhost:8080/", "height": 289 } }, "source": [ "pip install surprise" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "Collecting surprise\n", " Downloading https://files.pythonhosted.org/packages/61/de/e5cba8682201fcf9c3719a6fdda95693468ed061945493dea2dd37c5618b/surprise-0.1-py2.py3-none-any.whl\n", "Collecting scikit-surprise\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/f5/da/b5700d96495fb4f092be497f02492768a3d96a3f4fa2ae7dea46d4081cfa/scikit-surprise-1.1.0.tar.gz (6.4MB)\n", "\u001b[K |████████████████████████████████| 6.5MB 2.6MB/s \n", "\u001b[?25hRequirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise->surprise) (0.15.1)\n", "Requirement already satisfied: numpy>=1.11.2 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise->surprise) (1.18.5)\n", "Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise->surprise) (1.4.1)\n", "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise->surprise) (1.12.0)\n", "Building wheels for collected packages: scikit-surprise\n", " Building wheel for scikit-surprise (setup.py) ... \u001b[?25l\u001b[?25hdone\n", " Created wheel for scikit-surprise: filename=scikit_surprise-1.1.0-cp36-cp36m-linux_x86_64.whl size=1675370 sha256=b9c28146ba19d464e7357de16052301c4261e1e492c668f6ec5ec59683796e4f\n", " Stored in directory: /root/.cache/pip/wheels/cc/fa/8c/16c93fccce688ae1bde7d979ff102f7bee980d9cfeb8641bcf\n", "Successfully built scikit-surprise\n", "Installing collected packages: scikit-surprise, surprise\n", "Successfully installed scikit-surprise-1.1.0 surprise-0.1\n" ], "name": "stdout" } ] }, { "cell_type": "markdown", "metadata": { "id": "r7NacKLTKkFD", "colab_type": "text" }, "source": [ "# Check metrics on toy dataset" ] }, { "cell_type": "code", "metadata": { "id": "ijxxLsVNKkFD", "colab_type": "code", "outputId": "0844bd14-84f0-45e1-e07d-1265a2b98b33", "colab": { "base_uri": "https://localhost:8080/", "height": 1000 } }, "source": [ "import evaluation_measures as ev\n", "import imp\n", "import helpers\n", "imp.reload(ev)\n", "\n", "dir_path=\"Recommendations generated/toy-example/\"\n", "super_reactions=[4,5]\n", "test=pd.read_csv('./Datasets/toy-example/test.csv', sep='\\t', header=None)\n", "\n", "display(ev.evaluate_all(test, dir_path, super_reactions, topK=3))\n", "#also you can just type ev.evaluate_all() - I put above values as default\n", "\n", "toy_train_read=pd.read_csv('./Datasets/toy-example/train.csv', sep='\\t', header=None, names=['user', 'item', 'rating', 'timestamp'])\n", "toy_test_read=pd.read_csv('./Datasets/toy-example/test.csv', sep='\\t', header=None, names=['user', 'item', 'rating', 'timestamp'])\n", "reco=pd.read_csv('Recommendations generated/toy-example/Self_BaselineUI_reco.csv', header=None)\n", "estimations=pd.read_csv('Recommendations generated/toy-example/Self_BaselineUI_estimations.csv', names=['user', 'item', 'est_score'])\n", "toy_train_ui, toy_test_ui, toy_user_code_id, toy_user_id_code, \\\n", "toy_item_code_id, toy_item_id_code = helpers.data_to_csr(toy_train_read, toy_test_read)\n", "\n", "print('Training data:')\n", "display(toy_train_ui.todense())\n", "\n", "print('Test data:')\n", "display(toy_test_ui.todense())\n", "\n", "print('Recommendations:')\n", "display(reco)\n", "\n", "print('Estimations:')\n", "display(estimations)" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "943it [00:00, 12096.61it/s]\n" ], "name": "stderr" }, { "output_type": "display_data", "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ModelRMSEMAEprecisionrecallF_1F_05precision_superrecall_superNDCGmAPMRRLAUCHRHR2Reco in testTest coverageShannonGini
0Self_BaselineUI0.9675850.762740.0003530.0000440.0000790.0001470.00.00.0003140.0001770.000530.4989230.001060.00.600530.0050511.8031260.99638
\n", "
" ], "text/plain": [ " Model RMSE MAE ... Test coverage Shannon Gini\n", "0 Self_BaselineUI 0.967585 0.76274 ... 0.005051 1.803126 0.99638\n", "\n", "[1 rows x 19 columns]" ] }, "metadata": { "tags": [] } }, { "output_type": "stream", "text": [ "Training data:\n" ], "name": "stdout" }, { "output_type": "display_data", "data": { "text/plain": [ "matrix([[5, 3, 4, ..., 0, 0, 0],\n", " [0, 0, 0, ..., 0, 0, 0],\n", " [0, 0, 0, ..., 0, 0, 0],\n", " ...,\n", " [5, 0, 0, ..., 0, 0, 0],\n", " [0, 0, 0, ..., 0, 0, 0],\n", " [0, 0, 0, ..., 0, 0, 0]], dtype=int64)" ] }, "metadata": { "tags": [] } }, { "output_type": "stream", "text": [ "Test data:\n" ], "name": "stdout" }, { "output_type": "display_data", "data": { "text/plain": [ "matrix([[0, 0, 0, ..., 0, 0, 0],\n", " [4, 0, 0, ..., 0, 0, 0],\n", " [0, 0, 0, ..., 0, 0, 0],\n", " ...,\n", " [0, 0, 0, ..., 0, 0, 0],\n", " [0, 0, 0, ..., 0, 0, 0],\n", " [0, 5, 0, ..., 0, 0, 0]], dtype=int64)" ] }, "metadata": { "tags": [] } }, { "output_type": "stream", "text": [ "Recommendations:\n" ], "name": "stdout" }, { "output_type": "display_data", "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
01234567891011121314151617181920
018145.44958414915.23256115365.18866713065.16908115994.98962514674.93133510804.85964416424.85725011894.78204015004.742546
128145.58081314915.36378915365.31989513065.30031015995.12085414675.06256310804.99087216424.9884791194.92190911894.913268
238144.61546914914.39844515364.35455113064.33496515994.15551014674.09721910804.02552816424.0231341193.95656511893.947924
348146.24453914916.02751515365.98362113065.96403515995.78457914675.72628910805.65459816425.6522041195.58563511895.576994
458144.73851014914.52148615364.47759213064.45800715994.27855114674.22026010804.14856916424.1461761194.07960711894.070965
..................................................................
9389398146.23664414916.01962015365.97572613065.95614015995.77668514675.71839410805.64670316425.6443101195.57774011895.569099
9399408145.46244014915.24541615365.20152213065.18193615995.00248114674.94419010804.87249916424.8701061194.80353611894.794895
9409418145.89453914915.67751515365.63362113065.61403515995.43457914675.37628910805.30459816425.3022041195.23563511895.226994
9419428146.16438014915.94735615365.90346213065.88387615995.70442114675.64613010805.57443916425.5720461195.50547611895.496835
9429438145.28478314915.06775915365.02386513065.00427915994.82482314674.76653310804.69484216424.6924481194.62587911894.617238
\n", "

943 rows × 21 columns

\n", "
" ], "text/plain": [ " 0 1 2 3 ... 17 18 19 20\n", "0 1 814 5.449584 1491 ... 1189 4.782040 1500 4.742546\n", "1 2 814 5.580813 1491 ... 119 4.921909 1189 4.913268\n", "2 3 814 4.615469 1491 ... 119 3.956565 1189 3.947924\n", "3 4 814 6.244539 1491 ... 119 5.585635 1189 5.576994\n", "4 5 814 4.738510 1491 ... 119 4.079607 1189 4.070965\n", ".. ... ... ... ... ... ... ... ... ...\n", "938 939 814 6.236644 1491 ... 119 5.577740 1189 5.569099\n", "939 940 814 5.462440 1491 ... 119 4.803536 1189 4.794895\n", "940 941 814 5.894539 1491 ... 119 5.235635 1189 5.226994\n", "941 942 814 6.164380 1491 ... 119 5.505476 1189 5.496835\n", "942 943 814 5.284783 1491 ... 119 4.625879 1189 4.617238\n", "\n", "[943 rows x 21 columns]" ] }, "metadata": { "tags": [] } }, { "output_type": "stream", "text": [ "Estimations:\n" ], "name": "stdout" }, { "output_type": "display_data", "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
useritemest_score
0153.457161
11103.798540
21253.435415
31323.732018
41333.531991
............
199959439282.907189
1999694310673.485929
1999794310742.861988
1999894311882.727428
1999994312282.568442
\n", "

20000 rows × 3 columns

\n", "
" ], "text/plain": [ " user item est_score\n", "0 1 5 3.457161\n", "1 1 10 3.798540\n", "2 1 25 3.435415\n", "3 1 32 3.732018\n", "4 1 33 3.531991\n", "... ... ... ...\n", "19995 943 928 2.907189\n", "19996 943 1067 3.485929\n", "19997 943 1074 2.861988\n", "19998 943 1188 2.727428\n", "19999 943 1228 2.568442\n", "\n", "[20000 rows x 3 columns]" ] }, "metadata": { "tags": [] } } ] }, { "cell_type": "markdown", "metadata": { "id": "QSCe2WVVKkFG", "colab_type": "text" }, "source": [ "# Sample recommendations" ] }, { "cell_type": "code", "metadata": { "id": "uT8s_WQOKkFG", "colab_type": "code", "outputId": "13361cd0-f54a-4923-9303-422e501d7543", "colab": { "base_uri": "https://localhost:8080/", "height": 890 } }, "source": [ "train=pd.read_csv('./Datasets/ml-100k/train.csv', sep='\\t', header=None, names=['user', 'item', 'rating', 'timestamp'])\n", "items=pd.read_csv('./Datasets/ml-100k/movies.csv')\n", "\n", "user=random.choice(list(set(train['user'])))\n", "\n", "train_content=pd.merge(train, items, left_on='item', right_on='id')\n", "\n", "print('Here is what user rated high:')\n", "display(train_content[train_content['user']==user][['user', 'rating', 'title', 'genres']]\\\n", " .sort_values(by='rating', ascending=False)[:15])\n", "\n", "reco = np.loadtxt('Recommendations generated/ml-100k/Self_BaselineUI_reco.csv', delimiter=',')\n", "items=pd.read_csv('./Datasets/ml-100k/movies.csv')\n", "\n", "# Let's ignore scores - they are not used in evaluation: \n", "reco_users=reco[:,:1]\n", "reco_items=reco[:,1::2]\n", "# Let's put them into one array\n", "reco=np.concatenate((reco_users, reco_items), axis=1)\n", "\n", "# Let's rebuild it user-item dataframe\n", "recommended=[]\n", "for row in reco:\n", " for rec_nb, entry in enumerate(row[1:]):\n", " recommended.append((row[0], rec_nb+1, entry))\n", "recommended=pd.DataFrame(recommended, columns=['user','rec_nb', 'item'])\n", "\n", "recommended_content=pd.merge(recommended, items, left_on='item', right_on='id')\n", "\n", "print('Here is what we recommend:')\n", "recommended_content[recommended_content['user']==user][['user', 'rec_nb', 'title', 'genres']].sort_values(by='rec_nb')" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "Here is what user rated high:\n" ], "name": "stdout" }, { "output_type": "display_data", "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
userratingtitlegenres
55291815Jerry Maguire (1996)Drama, Romance
386871814First Wives Club, The (1996)Comedy
193901814Scream (1996)Horror, Thriller
637201814Mirror Has Two Faces, The (1996)Comedy, Romance
572871814Evita (1996)Drama, Musical
571061814Up Close and Personal (1996)Drama, Romance
247761814Gattaca (1997)Drama, Sci-Fi, Thriller
117781814Mission: Impossible (1996)Action, Adventure, Mystery
684691814Eye for an Eye (1996)Drama, Thriller
259081814Dead Man Walking (1995)Drama
98111814Time to Kill, A (1996)Drama
210111814I Know What You Did Last Summer (1997)Horror, Mystery, Thriller
335721814Independence Day (ID4) (1996)Action, Sci-Fi, War
446311814Ransom (1996)Drama, Thriller
467261814Hunchback of Notre Dame, The (1996)Animation, Children's, Musical
\n", "
" ], "text/plain": [ " user ... genres\n", "5529 181 ... Drama, Romance\n", "38687 181 ... Comedy\n", "19390 181 ... Horror, Thriller\n", "63720 181 ... Comedy, Romance\n", "57287 181 ... Drama, Musical\n", "57106 181 ... Drama, Romance\n", "24776 181 ... Drama, Sci-Fi, Thriller\n", "11778 181 ... Action, Adventure, Mystery\n", "68469 181 ... Drama, Thriller\n", "25908 181 ... Drama\n", "9811 181 ... Drama\n", "21011 181 ... Horror, Mystery, Thriller\n", "33572 181 ... Action, Sci-Fi, War\n", "44631 181 ... Drama, Thriller\n", "46726 181 ... Animation, Children's, Musical\n", "\n", "[15 rows x 4 columns]" ] }, "metadata": { "tags": [] } }, { "output_type": "stream", "text": [ "Here is what we recommend:\n" ], "name": "stdout" }, { "output_type": "execute_result", "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
userrec_nbtitlegenres
179181.01Great Day in Harlem, A (1994)Documentary
1122181.02Tough and Deadly (1995)Action, Drama, Thriller
2064181.03Aiqing wansui (1994)Drama
3005181.04Delta of Venus (1994)Drama
3948181.05Someone Else's America (1995)Drama
4890181.06Saint of Fort Washington, The (1993)Drama
5831181.07Celestial Clockwork (1994)Comedy
6774181.08Some Mother's Son (1996)Drama
8668181.09Maya Lin: A Strong Clear Vision (1994)Documentary
7714181.010Prefontaine (1997)Drama
\n", "
" ], "text/plain": [ " user ... genres\n", "179 181.0 ... Documentary\n", "1122 181.0 ... Action, Drama, Thriller\n", "2064 181.0 ... Drama\n", "3005 181.0 ... Drama\n", "3948 181.0 ... Drama\n", "4890 181.0 ... Drama\n", "5831 181.0 ... Comedy\n", "6774 181.0 ... Drama\n", "8668 181.0 ... Documentary\n", "7714 181.0 ... Drama\n", "\n", "[10 rows x 4 columns]" ] }, "metadata": { "tags": [] }, "execution_count": 24 } ] }, { "cell_type": "markdown", "metadata": { "id": "gFsye0sAKkFQ", "colab_type": "text" }, "source": [ "# project task 3: implement some other evaluation measure" ] }, { "cell_type": "code", "metadata": { "id": "3bhpq4zOKkFR", "colab_type": "code", "colab": {} }, "source": [ "# it may be your idea, modification of what we have already implemented \n", "# (for example Hit2 rate which would count as a success users whoreceived at least 2 relevant recommendations) \n", "# or something well-known\n", "# expected output: modification of evaluation_measures.py such that evaluate_all will also display your measure" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "8s81gSB3KkFU", "colab_type": "code", "outputId": "720e0c84-ef77-4c24-9ed9-df7697e9b98a", "colab": { "base_uri": "https://localhost:8080/", "height": 199 } }, "source": [ "dir_path=\"Recommendations generated/ml-100k/\"\n", "super_reactions=[4,5]\n", "test=pd.read_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None)\n", "\n", "ev.evaluate_all(test, dir_path, super_reactions)" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "943it [00:00, 6725.92it/s]\n", "943it [00:00, 6395.43it/s]\n" ], "name": "stderr" }, { "output_type": "execute_result", "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ModelRMSEMAEprecisionrecallF_1F_05precision_superrecall_superNDCGmAPMRRLAUCHRHR2Reco in testTest coverageShannonGini
0Ready_Baseline0.9494590.7524870.0914100.0376520.0460300.0612860.0796140.0564630.0959570.0431780.1981930.5155010.4379640.2396611.000000.0339112.8365130.991139
0Self_BaselineUI0.9675850.7627400.0009540.0001700.0002780.0004630.0006440.0001890.0007520.0001680.0016770.4964240.0095440.0000000.600530.0050511.8031260.996380
\n", "
" ], "text/plain": [ " Model RMSE MAE ... Test coverage Shannon Gini\n", "0 Ready_Baseline 0.949459 0.752487 ... 0.033911 2.836513 0.991139\n", "0 Self_BaselineUI 0.967585 0.762740 ... 0.005051 1.803126 0.996380\n", "\n", "[2 rows x 19 columns]" ] }, "metadata": { "tags": [] }, "execution_count": 25 } ] } ] }