s452101/workshops_recommender_systems
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

project task 3: implement some other evaluation measure

Browse Source
This commit is contained in:
s452101 2020-09-30 01:04:05 +02:00
parent e2434d4046
commit 8dc3f4ceb0
1 changed files with 321 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

530
P2. Evaluation.ipynb
View File

@ -9,7 +9,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 1, "execution_count": 20,
"metadata": { "metadata": {
"slideshow": { "slideshow": {
"slide_type": "-" "slide_type": "-"
@ -49,7 +49,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 2, "execution_count": 21,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -63,7 +63,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 3, "execution_count": 22,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -83,7 +83,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 4, "execution_count": 23,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -126,7 +126,7 @@
"0 0.949459 0.752487" "0 0.949459 0.752487"
] ]
}, },
"execution_count": 4, "execution_count": 23,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -147,7 +147,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 5, "execution_count": 40,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -162,7 +162,7 @@
" [ 77, 313, 475, ..., 11, 591, 175]])" " [ 77, 313, 475, ..., 11, 591, 175]])"
] ]
}, },
"execution_count": 5, "execution_count": 40,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -183,15 +183,15 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 6, "execution_count": 25,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"def ranking_metrics(test_ui, reco, super_reactions=[], topK=10):\n", "def ranking_metrics(test_ui, reco, super_reactions=[], topK=10):\n",
" \n", " \n",
" nb_items=test_ui.shape[1]\n", " nb_items=test_ui.shape[1]\n",
" relevant_users, super_relevant_users, prec, rec, F_1, F_05, prec_super, rec_super, ndcg, mAP, MRR, LAUC, HR=\\\n", " relevant_users, super_relevant_users, prec, rec, F_1, F_05, prec_super, rec_super, ndcg, mAP, MRR, LAUC, HR, HR2=\\\n",
" 0,0,0,0,0,0,0,0,0,0,0,0,0\n", " 0,0,0,0,0,0,0,0,0,0,0,0,0,0\n",
" \n", " \n",
" cg = (1.0 / np.log2(np.arange(2, topK + 2)))\n", " cg = (1.0 / np.log2(np.arange(2, topK + 2)))\n",
" cg_sum = np.cumsum(cg)\n", " cg_sum = np.cumsum(cg)\n",
@ -244,6 +244,7 @@
" ((nb_items-nb_u_rated_items)*nb_u_rated_items)\n", " ((nb_items-nb_u_rated_items)*nb_u_rated_items)\n",
" \n", " \n",
" HR+=nb_user_successes>0\n", " HR+=nb_user_successes>0\n",
" HR2+=nb_user_successes>1\n",
" \n", " \n",
" \n", " \n",
" result=[]\n", " result=[]\n",
@ -258,6 +259,7 @@
" result.append(('MRR', MRR/relevant_users))\n", " result.append(('MRR', MRR/relevant_users))\n",
" result.append(('LAUC', LAUC/relevant_users))\n", " result.append(('LAUC', LAUC/relevant_users))\n",
" result.append(('HR', HR/relevant_users))\n", " result.append(('HR', HR/relevant_users))\n",
" result.append(('HR2', HR2/relevant_users))\n",
"\n", "\n",
" df_result=(pd.DataFrame(list(zip(*result))[1])).T\n", " df_result=(pd.DataFrame(list(zip(*result))[1])).T\n",
" df_result.columns=list(zip(*result))[0]\n", " df_result.columns=list(zip(*result))[0]\n",
@ -266,14 +268,14 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 7, "execution_count": 41,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stderr", "name": "stderr",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"943it [00:00, 11673.30it/s]\n" "943it [00:00, 9975.21it/s]\n"
] ]
}, },
{ {
@ -308,6 +310,7 @@
" <th>MRR</th>\n", " <th>MRR</th>\n",
" <th>LAUC</th>\n", " <th>LAUC</th>\n",
" <th>HR</th>\n", " <th>HR</th>\n",
" <th>HR2</th>\n",
" </tr>\n", " </tr>\n",
" </thead>\n", " </thead>\n",
" <tbody>\n", " <tbody>\n",
@ -324,6 +327,7 @@
" <td>0.198193</td>\n", " <td>0.198193</td>\n",
" <td>0.515501</td>\n", " <td>0.515501</td>\n",
" <td>0.437964</td>\n", " <td>0.437964</td>\n",
" <td>0.239661</td>\n",
" </tr>\n", " </tr>\n",
" </tbody>\n", " </tbody>\n",
"</table>\n", "</table>\n",
@ -333,11 +337,11 @@
" precision recall F_1 F_05 precision_super recall_super \\\n", " precision recall F_1 F_05 precision_super recall_super \\\n",
"0 0.09141 0.037652 0.04603 0.061286 0.079614 0.056463 \n", "0 0.09141 0.037652 0.04603 0.061286 0.079614 0.056463 \n",
"\n", "\n",
" NDCG mAP MRR LAUC HR \n", " NDCG mAP MRR LAUC HR HR2 \n",
"0 0.095957 0.043178 0.198193 0.515501 0.437964 " "0 0.095957 0.043178 0.198193 0.515501 0.437964 0.239661 "
] ]
}, },
"execution_count": 7, "execution_count": 41,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -355,7 +359,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 8, "execution_count": 27,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -398,7 +402,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 9, "execution_count": 28,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -445,7 +449,7 @@
"0 1.0 0.033911 2.836513 0.991139" "0 1.0 0.033911 2.836513 0.991139"
] ]
}, },
"execution_count": 9, "execution_count": 28,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -470,14 +474,14 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 10, "execution_count": 30,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stderr", "name": "stderr",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"943it [00:00, 10629.42it/s]\n" "943it [00:00, 11673.16it/s]\n"
] ]
}, },
{ {
@ -556,7 +560,7 @@
"0 0.437964 1.0 0.033911 2.836513 0.991139 " "0 0.437964 1.0 0.033911 2.836513 0.991139 "
] ]
}, },
"execution_count": 10, "execution_count": 30,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -578,20 +582,20 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 11, "execution_count": 31,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stderr", "name": "stderr",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"943it [00:00, 11391.79it/s]\n", "943it [00:00, 12771.00it/s]\n",
"943it [00:00, 10740.34it/s]\n", "943it [00:00, 10989.86it/s]\n",
"943it [00:00, 11967.55it/s]\n", "943it [00:00, 11962.99it/s]\n",
"943it [00:00, 12152.51it/s]\n", "943it [00:00, 12435.05it/s]\n",
"943it [00:00, 12581.48it/s]\n", "943it [00:00, 12446.08it/s]\n",
"943it [00:00, 12607.27it/s]\n", "943it [00:00, 11974.18it/s]\n",
"943it [00:00, 12116.29it/s]\n" "943it [00:00, 11536.02it/s]\n"
] ]
} }
], ],
@ -610,7 +614,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 12, "execution_count": 32,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -754,7 +758,7 @@
"0 0.000463 0.000644 0.000189 " "0 0.000463 0.000644 0.000189 "
] ]
}, },
"execution_count": 12, "execution_count": 32,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -765,7 +769,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 13, "execution_count": 33,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -917,7 +921,7 @@
"0 0.600530 0.005051 1.803126 0.996380 " "0 0.600530 0.005051 1.803126 0.996380 "
] ]
}, },
"execution_count": 13, "execution_count": 33,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -935,14 +939,14 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 14, "execution_count": 34,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stderr", "name": "stderr",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"3it [00:00, 3005.95it/s]\n" "3it [00:00, 3111.50it/s]\n"
] ]
}, },
{ {
@ -1267,7 +1271,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 15, "execution_count": 35,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -1306,148 +1310,148 @@
" </thead>\n", " </thead>\n",
" <tbody>\n", " <tbody>\n",
" <tr>\n", " <tr>\n",
" <th>24578</th>\n", " <th>38020</th>\n",
" <td>163</td>\n", " <td>751</td>\n",
" <td>4</td>\n", " <td>5</td>\n",
" <td>One Flew Over the Cuckoo's Nest (1975)</td>\n", " <td>Beauty and the Beast (1991)</td>\n",
" <td>Drama</td>\n", " <td>Animation, Children's, Musical</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>12760</th>\n", " <th>55943</th>\n",
" <td>163</td>\n", " <td>751</td>\n",
" <td>4</td>\n", " <td>5</td>\n",
" <td>Dances with Wolves (1990)</td>\n", " <td>Room with a View, A (1986)</td>\n",
" <td>Adventure, Drama, Western</td>\n", " <td>Drama, Romance</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>43866</th>\n", " <th>28786</th>\n",
" <td>163</td>\n", " <td>751</td>\n",
" <td>4</td>\n", " <td>5</td>\n",
" <td>Pulp Fiction (1994)</td>\n", " <td>Cinderella (1950)</td>\n",
" <td>Crime, Drama</td>\n", " <td>Animation, Children's, Musical</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>37331</th>\n", " <th>24330</th>\n",
" <td>163</td>\n", " <td>751</td>\n",
" <td>4</td>\n", " <td>5</td>\n",
" <td>Contact (1997)</td>\n", " <td>Muppet Treasure Island (1996)</td>\n",
" <td>Drama, Sci-Fi</td>\n", " <td>Action, Adventure, Comedy, Musical, Thriller</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>36057</th>\n", " <th>23632</th>\n",
" <td>163</td>\n", " <td>751</td>\n",
" <td>4</td>\n", " <td>5</td>\n",
" <td>Silence of the Lambs, The (1991)</td>\n", " <td>Casablanca (1942)</td>\n",
" <td>Drama, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
" <th>31838</th>\n",
" <td>163</td>\n",
" <td>4</td>\n",
" <td>Shawshank Redemption, The (1994)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
" <th>67376</th>\n",
" <td>163</td>\n",
" <td>4</td>\n",
" <td>Good Will Hunting (1997)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
" <th>21714</th>\n",
" <td>163</td>\n",
" <td>3</td>\n",
" <td>In &amp; Out (1997)</td>\n",
" <td>Comedy</td>\n",
" </tr>\n",
" <tr>\n",
" <th>22153</th>\n",
" <td>163</td>\n",
" <td>3</td>\n",
" <td>Groundhog Day (1993)</td>\n",
" <td>Comedy, Romance</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19155</th>\n",
" <td>163</td>\n",
" <td>3</td>\n",
" <td>Scream (1996)</td>\n",
" <td>Horror, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
" <th>26457</th>\n",
" <td>163</td>\n",
" <td>3</td>\n",
" <td>Air Force One (1997)</td>\n",
" <td>Action, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17924</th>\n",
" <td>163</td>\n",
" <td>3</td>\n",
" <td>English Patient, The (1996)</td>\n",
" <td>Drama, Romance, War</td>\n", " <td>Drama, Romance, War</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>16410</th>\n", " <th>39028</th>\n",
" <td>163</td>\n", " <td>751</td>\n",
" <td>3</td>\n", " <td>5</td>\n",
" <td>Apollo 13 (1995)</td>\n",
" <td>Action, Drama, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
" <th>38849</th>\n",
" <td>163</td>\n",
" <td>3</td>\n",
" <td>Full Monty, The (1997)</td>\n", " <td>Full Monty, The (1997)</td>\n",
" <td>Comedy</td>\n", " <td>Comedy</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>3992</th>\n", " <th>21591</th>\n",
" <td>163</td>\n", " <td>751</td>\n",
" <td>3</td>\n", " <td>5</td>\n",
" <td>G.I. Jane (1997)</td>\n", " <td>In &amp; Out (1997)</td>\n",
" <td>Action, Drama, War</td>\n", " <td>Comedy</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20177</th>\n",
" <td>751</td>\n",
" <td>5</td>\n",
" <td>Return of the Jedi (1983)</td>\n",
" <td>Action, Adventure, Romance, Sci-Fi, War</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19460</th>\n",
" <td>751</td>\n",
" <td>5</td>\n",
" <td>Searching for Bobby Fischer (1993)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16345</th>\n",
" <td>751</td>\n",
" <td>5</td>\n",
" <td>Apollo 13 (1995)</td>\n",
" <td>Action, Drama, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15044</th>\n",
" <td>751</td>\n",
" <td>5</td>\n",
" <td>Indiana Jones and the Last Crusade (1989)</td>\n",
" <td>Action, Adventure</td>\n",
" </tr>\n",
" <tr>\n",
" <th>28596</th>\n",
" <td>751</td>\n",
" <td>5</td>\n",
" <td>Right Stuff, The (1983)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
" <th>54274</th>\n",
" <td>751</td>\n",
" <td>5</td>\n",
" <td>12 Angry Men (1957)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10317</th>\n",
" <td>751</td>\n",
" <td>5</td>\n",
" <td>Breakfast at Tiffany's (1961)</td>\n",
" <td>Drama, Romance</td>\n",
" </tr>\n",
" <tr>\n",
" <th>27371</th>\n",
" <td>751</td>\n",
" <td>5</td>\n",
" <td>Empire Strikes Back, The (1980)</td>\n",
" <td>Action, Adventure, Drama, Romance, Sci-Fi, War</td>\n",
" </tr>\n", " </tr>\n",
" </tbody>\n", " </tbody>\n",
"</table>\n", "</table>\n",
"</div>" "</div>"
], ],
"text/plain": [ "text/plain": [
" user rating title \\\n", " user rating title \\\n",
"24578 163 4 One Flew Over the Cuckoo's Nest (1975) \n", "38020 751 5 Beauty and the Beast (1991) \n",
"12760 163 4 Dances with Wolves (1990) \n", "55943 751 5 Room with a View, A (1986) \n",
"43866 163 4 Pulp Fiction (1994) \n", "28786 751 5 Cinderella (1950) \n",
"37331 163 4 Contact (1997) \n", "24330 751 5 Muppet Treasure Island (1996) \n",
"36057 163 4 Silence of the Lambs, The (1991) \n", "23632 751 5 Casablanca (1942) \n",
"31838 163 4 Shawshank Redemption, The (1994) \n", "39028 751 5 Full Monty, The (1997) \n",
"67376 163 4 Good Will Hunting (1997) \n", "21591 751 5 In & Out (1997) \n",
"21714 163 3 In & Out (1997) \n", "20177 751 5 Return of the Jedi (1983) \n",
"22153 163 3 Groundhog Day (1993) \n", "19460 751 5 Searching for Bobby Fischer (1993) \n",
"19155 163 3 Scream (1996) \n", "16345 751 5 Apollo 13 (1995) \n",
"26457 163 3 Air Force One (1997) \n", "15044 751 5 Indiana Jones and the Last Crusade (1989) \n",
"17924 163 3 English Patient, The (1996) \n", "28596 751 5 Right Stuff, The (1983) \n",
"16410 163 3 Apollo 13 (1995) \n", "54274 751 5 12 Angry Men (1957) \n",
"38849 163 3 Full Monty, The (1997) \n", "10317 751 5 Breakfast at Tiffany's (1961) \n",
"3992 163 3 G.I. Jane (1997) \n", "27371 751 5 Empire Strikes Back, The (1980) \n",
"\n", "\n",
" genres \n", " genres \n",
"24578 Drama \n", "38020 Animation, Children's, Musical \n",
"12760 Adventure, Drama, Western \n", "55943 Drama, Romance \n",
"43866 Crime, Drama \n", "28786 Animation, Children's, Musical \n",
"37331 Drama, Sci-Fi \n", "24330 Action, Adventure, Comedy, Musical, Thriller \n",
"36057 Drama, Thriller \n", "23632 Drama, Romance, War \n",
"31838 Drama \n", "39028 Comedy \n",
"67376 Drama \n", "21591 Comedy \n",
"21714 Comedy \n", "20177 Action, Adventure, Romance, Sci-Fi, War \n",
"22153 Comedy, Romance \n", "19460 Drama \n",
"19155 Horror, Thriller \n", "16345 Action, Drama, Thriller \n",
"26457 Action, Thriller \n", "15044 Action, Adventure \n",
"17924 Drama, Romance, War \n", "28596 Drama \n",
"16410 Action, Drama, Thriller \n", "54274 Drama \n",
"38849 Comedy \n", "10317 Drama, Romance \n",
"3992 Action, Drama, War " "27371 Action, Adventure, Drama, Romance, Sci-Fi, War "
] ]
}, },
"metadata": {}, "metadata": {},
@ -1489,71 +1493,71 @@
" </thead>\n", " </thead>\n",
" <tbody>\n", " <tbody>\n",
" <tr>\n", " <tr>\n",
" <th>161</th>\n", " <th>749</th>\n",
" <td>163.0</td>\n", " <td>751.0</td>\n",
" <td>1</td>\n", " <td>1</td>\n",
" <td>Great Day in Harlem, A (1994)</td>\n", " <td>Great Day in Harlem, A (1994)</td>\n",
" <td>Documentary</td>\n", " <td>Documentary</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>1104</th>\n", " <th>1691</th>\n",
" <td>163.0</td>\n", " <td>751.0</td>\n",
" <td>2</td>\n", " <td>2</td>\n",
" <td>Tough and Deadly (1995)</td>\n", " <td>Tough and Deadly (1995)</td>\n",
" <td>Action, Drama, Thriller</td>\n", " <td>Action, Drama, Thriller</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>2046</th>\n", " <th>2633</th>\n",
" <td>163.0</td>\n", " <td>751.0</td>\n",
" <td>3</td>\n", " <td>3</td>\n",
" <td>Aiqing wansui (1994)</td>\n", " <td>Aiqing wansui (1994)</td>\n",
" <td>Drama</td>\n", " <td>Drama</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>2988</th>\n", " <th>3575</th>\n",
" <td>163.0</td>\n", " <td>751.0</td>\n",
" <td>4</td>\n", " <td>4</td>\n",
" <td>Delta of Venus (1994)</td>\n", " <td>Delta of Venus (1994)</td>\n",
" <td>Drama</td>\n", " <td>Drama</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>3930</th>\n", " <th>4517</th>\n",
" <td>163.0</td>\n", " <td>751.0</td>\n",
" <td>5</td>\n", " <td>5</td>\n",
" <td>Someone Else's America (1995)</td>\n", " <td>Someone Else's America (1995)</td>\n",
" <td>Drama</td>\n", " <td>Drama</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>4872</th>\n", " <th>5459</th>\n",
" <td>163.0</td>\n", " <td>751.0</td>\n",
" <td>6</td>\n", " <td>6</td>\n",
" <td>Saint of Fort Washington, The (1993)</td>\n", " <td>Saint of Fort Washington, The (1993)</td>\n",
" <td>Drama</td>\n", " <td>Drama</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>5813</th>\n", " <th>6401</th>\n",
" <td>163.0</td>\n", " <td>751.0</td>\n",
" <td>7</td>\n", " <td>7</td>\n",
" <td>Celestial Clockwork (1994)</td>\n", " <td>Celestial Clockwork (1994)</td>\n",
" <td>Comedy</td>\n", " <td>Comedy</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>6756</th>\n", " <th>7342</th>\n",
" <td>163.0</td>\n", " <td>751.0</td>\n",
" <td>8</td>\n", " <td>8</td>\n",
" <td>Some Mother's Son (1996)</td>\n", " <td>Some Mother's Son (1996)</td>\n",
" <td>Drama</td>\n", " <td>Drama</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>8650</th>\n", " <th>9238</th>\n",
" <td>163.0</td>\n", " <td>751.0</td>\n",
" <td>9</td>\n", " <td>9</td>\n",
" <td>Maya Lin: A Strong Clear Vision (1994)</td>\n", " <td>Maya Lin: A Strong Clear Vision (1994)</td>\n",
" <td>Documentary</td>\n", " <td>Documentary</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>7696</th>\n", " <th>8284</th>\n",
" <td>163.0</td>\n", " <td>751.0</td>\n",
" <td>10</td>\n", " <td>10</td>\n",
" <td>Prefontaine (1997)</td>\n", " <td>Prefontaine (1997)</td>\n",
" <td>Drama</td>\n", " <td>Drama</td>\n",
@ -1564,31 +1568,31 @@
], ],
"text/plain": [ "text/plain": [
" user rec_nb title \\\n", " user rec_nb title \\\n",
"161 163.0 1 Great Day in Harlem, A (1994) \n", "749 751.0 1 Great Day in Harlem, A (1994) \n",
"1104 163.0 2 Tough and Deadly (1995) \n", "1691 751.0 2 Tough and Deadly (1995) \n",
"2046 163.0 3 Aiqing wansui (1994) \n", "2633 751.0 3 Aiqing wansui (1994) \n",
"2988 163.0 4 Delta of Venus (1994) \n", "3575 751.0 4 Delta of Venus (1994) \n",
"3930 163.0 5 Someone Else's America (1995) \n", "4517 751.0 5 Someone Else's America (1995) \n",
"4872 163.0 6 Saint of Fort Washington, The (1993) \n", "5459 751.0 6 Saint of Fort Washington, The (1993) \n",
"5813 163.0 7 Celestial Clockwork (1994) \n", "6401 751.0 7 Celestial Clockwork (1994) \n",
"6756 163.0 8 Some Mother's Son (1996) \n", "7342 751.0 8 Some Mother's Son (1996) \n",
"8650 163.0 9 Maya Lin: A Strong Clear Vision (1994) \n", "9238 751.0 9 Maya Lin: A Strong Clear Vision (1994) \n",
"7696 163.0 10 Prefontaine (1997) \n", "8284 751.0 10 Prefontaine (1997) \n",
"\n", "\n",
" genres \n", " genres \n",
"161 Documentary \n", "749 Documentary \n",
"1104 Action, Drama, Thriller \n", "1691 Action, Drama, Thriller \n",
"2046 Drama \n", "2633 Drama \n",
"2988 Drama \n", "3575 Drama \n",
"3930 Drama \n", "4517 Drama \n",
"4872 Drama \n", "5459 Drama \n",
"5813 Comedy \n", "6401 Comedy \n",
"6756 Drama \n", "7342 Drama \n",
"8650 Documentary \n", "9238 Documentary \n",
"7696 Drama " "8284 Drama "
] ]
}, },
"execution_count": 15, "execution_count": 35,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -1636,32 +1640,37 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 16, "execution_count": 36,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"# it may be your idea, modification of what we have already implemented \n", "# it may be your idea, modification of what we have already implemented \n",
"# (for example Hit2 rate which would count as a success users whoreceived at least 2 relevant recommendations) \n", "# (for example Hit2 rate which would count as a success users whoreceived at least 2 relevant recommendations) \n",
"# or something well-known\n", "# or something well-known\n",
"# expected output: modification of evaluation_measures.py such that evaluate_all will also display your measure" "# expected output: modification of evaluation_measures.py such that evaluate_all will also display your measure\n",
"\n",
"#ROZWIĄZANIE:\n",
"# Implementacja Hit2 rate w definicji funkcji ranking_metrics"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 17, "execution_count": 44,
"metadata": {}, "metadata": {
"scrolled": true
},
"outputs": [ "outputs": [
{ {
"name": "stderr", "name": "stderr",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"943it [00:00, 11530.77it/s]\n", "943it [00:00, 7386.42it/s]\n",
"943it [00:00, 11386.61it/s]\n", "943it [00:00, 6178.65it/s]\n",
"943it [00:00, 12285.65it/s]\n", "943it [00:00, 6755.10it/s]\n",
"943it [00:00, 11963.28it/s]\n", "943it [00:00, 6705.79it/s]\n",
"943it [00:00, 12440.68it/s]\n", "943it [00:00, 6952.37it/s]\n",
"943it [00:00, 11968.68it/s]\n", "943it [00:00, 6261.78it/s]\n",
"943it [00:00, 11530.37it/s]\n" "943it [00:00, 6612.16it/s]\n"
] ]
}, },
{ {
@ -1886,7 +1895,7 @@
"0 0.496424 0.009544 0.600530 0.005051 1.803126 0.996380 " "0 0.496424 0.009544 0.600530 0.005051 1.803126 0.996380 "
] ]
}, },
"execution_count": 17, "execution_count": 44,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -1898,6 +1907,109 @@
"\n", "\n",
"ev.evaluate_all(test, dir_path, super_reactions)" "ev.evaluate_all(test, dir_path, super_reactions)"
] ]
},
{
"cell_type": "code",
"execution_count": 43,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"943it [00:00, 11668.03it/s]\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" <th>HR2</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.09141</td>\n",
" <td>0.037652</td>\n",
" <td>0.04603</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
" <td>0.239661</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" precision recall F_1 F_05 precision_super recall_super \\\n",
"0 0.09141 0.037652 0.04603 0.061286 0.079614 0.056463 \n",
"\n",
" NDCG mAP MRR LAUC HR HR2 \n",
"0 0.095957 0.043178 0.198193 0.515501 0.437964 0.239661 "
]
},
"execution_count": 43,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"reco = np.loadtxt('Recommendations generated/ml-100k/Ready_Baseline_reco.csv', delimiter=',')\n",
"# Let's ignore scores - they are not used in evaluation: \n",
"users=reco[:,:1]\n",
"items=reco[:,1::2]\n",
"# Let's use inner ids instead of real ones\n",
"users=np.vectorize(lambda x: user_id_code.setdefault(x, -1))(users)\n",
"items=np.vectorize(lambda x: item_id_code.setdefault(x, -1))(items) # maybe items we recommend are not in test set\n",
"# Let's put them into one array\n",
"reco=np.concatenate((users, items), axis=1)\n",
"\n",
"ranking_metrics(test_ui, reco, super_reactions=[4,5], topK=10)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
} }
], ],
"metadata": { "metadata": {