From 587f3d36619e9e374798ca9e2d0f92d5a536f842 Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 12 Jun 2021 10:56:22 +0200 Subject: [PATCH] Zadanie 3 --- P2. Evaluation.ipynb | 502 +++++++++++++++++++++++++------------------ 1 file changed, 292 insertions(+), 210 deletions(-) diff --git a/P2. Evaluation.ipynb b/P2. Evaluation.ipynb index e89d78d..116377e 100644 --- a/P2. Evaluation.ipynb +++ b/P2. Evaluation.ipynb @@ -397,7 +397,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -453,7 +453,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -500,7 +500,7 @@ "0 1.0 0.033911 2.836513 0.991139" ] }, - "execution_count": 9, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -521,14 +521,14 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "943it [00:00, 11012.47it/s]\n" + "943it [00:00, 12952.59it/s]\n" ] }, { @@ -565,6 +565,8 @@ " MRR\n", " LAUC\n", " HR\n", + " HitRate2\n", + " HitRate3\n", " Reco in test\n", " Test coverage\n", " Shannon\n", @@ -587,6 +589,8 @@ " 0.198193\n", " 0.515501\n", " 0.437964\n", + " 0.239661\n", + " 0.126193\n", " 1.0\n", " 0.033911\n", " 2.836513\n", @@ -603,11 +607,14 @@ " precision_super recall_super NDCG mAP MRR LAUC \\\n", "0 0.079614 0.056463 0.095957 0.043178 0.198193 0.515501 \n", "\n", - " HR Reco in test Test coverage Shannon Gini \n", - "0 0.437964 1.0 0.033911 2.836513 0.991139 " + " HR HitRate2 HitRate3 Reco in test Test coverage Shannon \\\n", + "0 0.437964 0.239661 0.126193 1.0 0.033911 2.836513 \n", + "\n", + " Gini \n", + "0 0.991139 " ] }, - "execution_count": 10, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -633,18 +640,19 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "943it [00:00, 10346.82it/s]\n", - "943it [00:00, 11772.32it/s]\n", - "943it [00:00, 10636.62it/s]\n", - "943it [00:00, 10767.92it/s]\n", - "943it [00:00, 12019.93it/s]\n" + "943it [00:00, 13130.52it/s]\n", + "943it [00:00, 12777.31it/s]\n", + "943it [00:00, 13513.65it/s]\n", + "943it [00:00, 13323.06it/s]\n", + "943it [00:00, 13507.69it/s]\n", + "943it [00:00, 13697.48it/s]\n" ] } ], @@ -659,7 +667,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -722,14 +730,14 @@ " \n", " 0\n", " Ready_Random\n", - " 1.521845\n", - " 1.225949\n", - " 0.047190\n", - " 0.020753\n", - " 0.024810\n", - " 0.032269\n", - " 0.029506\n", - " 0.023707\n", + " 1.516512\n", + " 1.217214\n", + " 0.045599\n", + " 0.021001\n", + " 0.024136\n", + " 0.031226\n", + " 0.028541\n", + " 0.022057\n", " \n", " \n", " 0\n", @@ -755,6 +763,18 @@ " 0.000644\n", " 0.000189\n", " \n", + " \n", + " 0\n", + " Self_IKNN\n", + " 1.018363\n", + " 0.808793\n", + " 0.000318\n", + " 0.000108\n", + " 0.000140\n", + " 0.000189\n", + " 0.000000\n", + " 0.000000\n", + " \n", " \n", "\n", "" @@ -763,19 +783,21 @@ " Model RMSE MAE precision recall F_1 \\\n", "0 Self_TopPop 2.508258 2.217909 0.188865 0.116919 0.118732 \n", "0 Ready_Baseline 0.949459 0.752487 0.091410 0.037652 0.046030 \n", - "0 Ready_Random 1.521845 1.225949 0.047190 0.020753 0.024810 \n", + "0 Ready_Random 1.516512 1.217214 0.045599 0.021001 0.024136 \n", "0 Self_TopRated 1.030712 0.820904 0.000954 0.000188 0.000298 \n", "0 Self_BaselineUI 0.967585 0.762740 0.000954 0.000170 0.000278 \n", + "0 Self_IKNN 1.018363 0.808793 0.000318 0.000108 0.000140 \n", "\n", " F_05 precision_super recall_super \n", "0 0.141584 0.130472 0.137473 \n", "0 0.061286 0.079614 0.056463 \n", - "0 0.032269 0.029506 0.023707 \n", + "0 0.031226 0.028541 0.022057 \n", "0 0.000481 0.000644 0.000223 \n", - "0 0.000463 0.000644 0.000189 " + "0 0.000463 0.000644 0.000189 \n", + "0 0.000189 0.000000 0.000000 " ] }, - "execution_count": 12, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -786,7 +808,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -816,6 +838,8 @@ " MRR\n", " LAUC\n", " HR\n", + " HitRate2\n", + " HitRate3\n", " Reco in test\n", " Test coverage\n", " Shannon\n", @@ -831,6 +855,8 @@ " 0.400939\n", " 0.555546\n", " 0.765642\n", + " 0.492047\n", + " 0.290562\n", " 1.000000\n", " 0.038961\n", " 3.159079\n", @@ -844,6 +870,8 @@ " 0.198193\n", " 0.515501\n", " 0.437964\n", + " 0.239661\n", + " 0.126193\n", " 1.000000\n", " 0.033911\n", " 2.836513\n", @@ -852,15 +880,17 @@ " \n", " 0\n", " Ready_Random\n", - " 0.050075\n", - " 0.018728\n", - " 0.121957\n", - " 0.506893\n", - " 0.329799\n", - " 0.986532\n", - " 0.184704\n", - " 5.099706\n", - " 0.907217\n", + " 0.050154\n", + " 0.019000\n", + " 0.125089\n", + " 0.507013\n", + " 0.327678\n", + " 0.093319\n", + " 0.026511\n", + " 0.988017\n", + " 0.192641\n", + " 5.141246\n", + " 0.903763\n", " \n", " \n", " 0\n", @@ -870,6 +900,8 @@ " 0.003348\n", " 0.496433\n", " 0.009544\n", + " 0.000000\n", + " 0.000000\n", " 0.699046\n", " 0.005051\n", " 1.945910\n", @@ -883,11 +915,28 @@ " 0.001677\n", " 0.496424\n", " 0.009544\n", + " 0.000000\n", + " 0.000000\n", " 0.600530\n", " 0.005051\n", " 1.803126\n", " 0.996380\n", " \n", + " \n", + " 0\n", + " Self_IKNN\n", + " 0.000214\n", + " 0.000037\n", + " 0.000368\n", + " 0.496391\n", + " 0.003181\n", + " 0.000000\n", + " 0.000000\n", + " 0.392153\n", + " 0.115440\n", + " 4.174741\n", + " 0.965327\n", + " \n", " \n", "\n", "" @@ -896,19 +945,21 @@ " Model NDCG mAP MRR LAUC HR \\\n", "0 Self_TopPop 0.214651 0.111707 0.400939 0.555546 0.765642 \n", "0 Ready_Baseline 0.095957 0.043178 0.198193 0.515501 0.437964 \n", - "0 Ready_Random 0.050075 0.018728 0.121957 0.506893 0.329799 \n", + "0 Ready_Random 0.050154 0.019000 0.125089 0.507013 0.327678 \n", "0 Self_TopRated 0.001043 0.000335 0.003348 0.496433 0.009544 \n", "0 Self_BaselineUI 0.000752 0.000168 0.001677 0.496424 0.009544 \n", + "0 Self_IKNN 0.000214 0.000037 0.000368 0.496391 0.003181 \n", "\n", - " Reco in test Test coverage Shannon Gini \n", - "0 1.000000 0.038961 3.159079 0.987317 \n", - "0 1.000000 0.033911 2.836513 0.991139 \n", - "0 0.986532 0.184704 5.099706 0.907217 \n", - "0 0.699046 0.005051 1.945910 0.995669 \n", - "0 0.600530 0.005051 1.803126 0.996380 " + " HitRate2 HitRate3 Reco in test Test coverage Shannon Gini \n", + "0 0.492047 0.290562 1.000000 0.038961 3.159079 0.987317 \n", + "0 0.239661 0.126193 1.000000 0.033911 2.836513 0.991139 \n", + "0 0.093319 0.026511 0.988017 0.192641 5.141246 0.903763 \n", + "0 0.000000 0.000000 0.699046 0.005051 1.945910 0.995669 \n", + "0 0.000000 0.000000 0.600530 0.005051 1.803126 0.996380 \n", + "0 0.000000 0.000000 0.392153 0.115440 4.174741 0.965327 " ] }, - "execution_count": 13, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -926,14 +977,15 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "3it [00:00, 5771.98it/s]\n" + "3it [00:00, ?it/s]\n", + "3it [00:00, ?it/s]\n" ] }, { @@ -971,6 +1023,8 @@ " MRR\n", " LAUC\n", " HR\n", + " HitRate2\n", + " HitRate3\n", " Reco in test\n", " Test coverage\n", " Shannon\n", @@ -982,7 +1036,7 @@ " 0\n", " Self_BaselineUI\n", " 1.612452\n", - " 1.4\n", + " 1.400\n", " 0.444444\n", " 0.888889\n", " 0.555556\n", @@ -994,24 +1048,52 @@ " 0.611111\n", " 0.638889\n", " 1.0\n", + " 0.333333\n", + " 0.0\n", " 0.888889\n", " 0.8\n", " 1.386294\n", - " 0.25\n", + " 0.250000\n", + " \n", + " \n", + " 0\n", + " Self_BaselineIU\n", + " 1.648337\n", + " 1.575\n", + " 0.444444\n", + " 0.888889\n", + " 0.555556\n", + " 0.478632\n", + " 0.333333\n", + " 0.75\n", + " 0.720550\n", + " 0.629630\n", + " 0.666667\n", + " 0.722222\n", + " 1.0\n", + " 0.333333\n", + " 0.0\n", + " 0.777778\n", + " 0.8\n", + " 1.351784\n", + " 0.357143\n", " \n", " \n", "\n", "" ], "text/plain": [ - " Model RMSE MAE precision recall F_1 F_05 \\\n", - "0 Self_BaselineUI 1.612452 1.4 0.444444 0.888889 0.555556 0.478632 \n", + " Model RMSE MAE precision recall F_1 F_05 \\\n", + "0 Self_BaselineUI 1.612452 1.400 0.444444 0.888889 0.555556 0.478632 \n", + "0 Self_BaselineIU 1.648337 1.575 0.444444 0.888889 0.555556 0.478632 \n", "\n", " precision_super recall_super NDCG mAP MRR LAUC HR \\\n", "0 0.333333 0.75 0.676907 0.574074 0.611111 0.638889 1.0 \n", + "0 0.333333 0.75 0.720550 0.629630 0.666667 0.722222 1.0 \n", "\n", - " Reco in test Test coverage Shannon Gini \n", - "0 0.888889 0.8 1.386294 0.25 " + " HitRate2 HitRate3 Reco in test Test coverage Shannon Gini \n", + "0 0.333333 0.0 0.888889 0.8 1.386294 0.250000 \n", + "0 0.333333 0.0 0.777778 0.8 1.351784 0.357143 " ] }, "metadata": {}, @@ -1029,7 +1111,7 @@ "text/plain": [ "matrix([[3, 4, 0, 0, 5, 0, 0, 4],\n", " [0, 1, 2, 3, 0, 0, 0, 0],\n", - " [0, 0, 0, 5, 0, 3, 4, 0]])" + " [0, 0, 0, 5, 0, 3, 4, 0]], dtype=int64)" ] }, "metadata": {}, @@ -1047,7 +1129,7 @@ "text/plain": [ "matrix([[0, 0, 0, 0, 0, 0, 3, 0],\n", " [0, 0, 0, 0, 5, 0, 0, 0],\n", - " [5, 0, 4, 0, 0, 0, 0, 2]])" + " [5, 0, 4, 0, 0, 0, 0, 2]], dtype=int64)" ] }, "metadata": {}, @@ -1276,7 +1358,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -1315,148 +1397,148 @@ " \n", " \n", " \n", - " 57482\n", - " 2\n", + " 36652\n", + " 735\n", " 5\n", - " Emma (1996)\n", - " Drama, Romance\n", + " Lone Star (1996)\n", + " Drama, Mystery\n", " \n", " \n", - " 54506\n", - " 2\n", - " 5\n", - " Sense and Sensibility (1995)\n", - " Drama, Romance\n", - " \n", - " \n", - " 40581\n", - " 2\n", - " 5\n", - " Titanic (1997)\n", - " Action, Drama, Romance\n", - " \n", - " \n", - " 2949\n", - " 2\n", + " 3143\n", + " 735\n", " 5\n", " Star Wars (1977)\n", " Action, Adventure, Romance, Sci-Fi, War\n", " \n", " \n", - " 69653\n", - " 2\n", - " 5\n", - " Wings of the Dove, The (1997)\n", - " Drama, Romance, Thriller\n", - " \n", - " \n", - " 7906\n", - " 2\n", - " 5\n", - " As Good As It Gets (1997)\n", - " Comedy, Drama\n", - " \n", - " \n", - " 69400\n", - " 2\n", - " 5\n", - " Shall We Dance? (1996)\n", - " Comedy\n", - " \n", - " \n", - " 14469\n", - " 2\n", - " 5\n", - " Fargo (1996)\n", - " Crime, Drama, Thriller\n", - " \n", - " \n", - " 46151\n", - " 2\n", - " 5\n", - " L.A. Confidential (1997)\n", - " Crime, Film-Noir, Mystery, Thriller\n", - " \n", - " \n", - " 67293\n", - " 2\n", - " 5\n", - " Good Will Hunting (1997)\n", - " Drama\n", - " \n", - " \n", - " 20923\n", - " 2\n", - " 5\n", - " Secrets & Lies (1996)\n", - " Drama\n", - " \n", - " \n", - " 52921\n", - " 2\n", + " 52919\n", + " 735\n", " 5\n", " Kolya (1996)\n", " Comedy\n", " \n", " \n", - " 50103\n", - " 2\n", + " 275\n", + " 735\n", " 4\n", - " Mrs. Brown (Her Majesty, Mrs. Brown) (1997)\n", - " Drama, Romance\n", + " Toy Story (1995)\n", + " Animation, Children's, Comedy\n", " \n", " \n", - " 51972\n", - " 2\n", + " 41134\n", + " 735\n", + " 4\n", + " Trainspotting (1996)\n", + " Drama\n", + " \n", + " \n", + " 28094\n", + " 735\n", + " 4\n", + " Face/Off (1997)\n", + " Action, Sci-Fi, Thriller\n", + " \n", + " \n", + " 26548\n", + " 735\n", + " 4\n", + " Everyone Says I Love You (1996)\n", + " Comedy, Musical, Romance\n", + " \n", + " \n", + " 26186\n", + " 735\n", + " 4\n", + " Air Force One (1997)\n", + " Action, Thriller\n", + " \n", + " \n", + " 25791\n", + " 735\n", + " 4\n", + " Dead Man Walking (1995)\n", + " Drama\n", + " \n", + " \n", + " 51948\n", + " 735\n", " 4\n", " Mighty Aphrodite (1995)\n", " Comedy\n", " \n", " \n", - " 515\n", - " 2\n", + " 52778\n", + " 735\n", " 4\n", - " Heat (1995)\n", - " Action, Crime, Thriller\n", + " Fly Away Home (1996)\n", + " Adventure, Children's\n", + " \n", + " \n", + " 20966\n", + " 735\n", + " 4\n", + " Secrets & Lies (1996)\n", + " Drama\n", + " \n", + " \n", + " 19301\n", + " 735\n", + " 4\n", + " Scream (1996)\n", + " Horror, Thriller\n", + " \n", + " \n", + " 54450\n", + " 735\n", + " 4\n", + " Sense and Sensibility (1995)\n", + " Drama, Romance\n", + " \n", + " \n", + " 17177\n", + " 735\n", + " 4\n", + " Leaving Las Vegas (1995)\n", + " Drama, Romance\n", " \n", " \n", "\n", "" ], "text/plain": [ - " user rating title \\\n", - "57482 2 5 Emma (1996) \n", - "54506 2 5 Sense and Sensibility (1995) \n", - "40581 2 5 Titanic (1997) \n", - "2949 2 5 Star Wars (1977) \n", - "69653 2 5 Wings of the Dove, The (1997) \n", - "7906 2 5 As Good As It Gets (1997) \n", - "69400 2 5 Shall We Dance? (1996) \n", - "14469 2 5 Fargo (1996) \n", - "46151 2 5 L.A. Confidential (1997) \n", - "67293 2 5 Good Will Hunting (1997) \n", - "20923 2 5 Secrets & Lies (1996) \n", - "52921 2 5 Kolya (1996) \n", - "50103 2 4 Mrs. Brown (Her Majesty, Mrs. Brown) (1997) \n", - "51972 2 4 Mighty Aphrodite (1995) \n", - "515 2 4 Heat (1995) \n", + " user rating title \\\n", + "36652 735 5 Lone Star (1996) \n", + "3143 735 5 Star Wars (1977) \n", + "52919 735 5 Kolya (1996) \n", + "275 735 4 Toy Story (1995) \n", + "41134 735 4 Trainspotting (1996) \n", + "28094 735 4 Face/Off (1997) \n", + "26548 735 4 Everyone Says I Love You (1996) \n", + "26186 735 4 Air Force One (1997) \n", + "25791 735 4 Dead Man Walking (1995) \n", + "51948 735 4 Mighty Aphrodite (1995) \n", + "52778 735 4 Fly Away Home (1996) \n", + "20966 735 4 Secrets & Lies (1996) \n", + "19301 735 4 Scream (1996) \n", + "54450 735 4 Sense and Sensibility (1995) \n", + "17177 735 4 Leaving Las Vegas (1995) \n", "\n", " genres \n", - "57482 Drama, Romance \n", - "54506 Drama, Romance \n", - "40581 Action, Drama, Romance \n", - "2949 Action, Adventure, Romance, Sci-Fi, War \n", - "69653 Drama, Romance, Thriller \n", - "7906 Comedy, Drama \n", - "69400 Comedy \n", - "14469 Crime, Drama, Thriller \n", - "46151 Crime, Film-Noir, Mystery, Thriller \n", - "67293 Drama \n", - "20923 Drama \n", - "52921 Comedy \n", - "50103 Drama, Romance \n", - "51972 Comedy \n", - "515 Action, Crime, Thriller " + "36652 Drama, Mystery \n", + "3143 Action, Adventure, Romance, Sci-Fi, War \n", + "52919 Comedy \n", + "275 Animation, Children's, Comedy \n", + "41134 Drama \n", + "28094 Action, Sci-Fi, Thriller \n", + "26548 Comedy, Musical, Romance \n", + "26186 Action, Thriller \n", + "25791 Drama \n", + "51948 Comedy \n", + "52778 Adventure, Children's \n", + "20966 Drama \n", + "19301 Horror, Thriller \n", + "54450 Drama, Romance \n", + "17177 Drama, Romance " ] }, "metadata": {}, @@ -1498,71 +1580,71 @@ " \n", " \n", " \n", - " 1\n", - " 2.0\n", + " 733\n", + " 735.0\n", " 1\n", " Great Day in Harlem, A (1994)\n", " Documentary\n", " \n", " \n", - " 943\n", - " 2.0\n", + " 1675\n", + " 735.0\n", " 2\n", " Tough and Deadly (1995)\n", " Action, Drama, Thriller\n", " \n", " \n", - " 1885\n", - " 2.0\n", + " 2617\n", + " 735.0\n", " 3\n", " Aiqing wansui (1994)\n", " Drama\n", " \n", " \n", - " 2827\n", - " 2.0\n", + " 3559\n", + " 735.0\n", " 4\n", " Delta of Venus (1994)\n", " Drama\n", " \n", " \n", - " 3769\n", - " 2.0\n", + " 4501\n", + " 735.0\n", " 5\n", " Someone Else's America (1995)\n", " Drama\n", " \n", " \n", - " 4711\n", - " 2.0\n", + " 5443\n", + " 735.0\n", " 6\n", " Saint of Fort Washington, The (1993)\n", " Drama\n", " \n", " \n", - " 5653\n", - " 2.0\n", + " 6385\n", + " 735.0\n", " 7\n", " Celestial Clockwork (1994)\n", " Comedy\n", " \n", " \n", - " 6595\n", - " 2.0\n", + " 7326\n", + " 735.0\n", " 8\n", " Some Mother's Son (1996)\n", " Drama\n", " \n", " \n", - " 8489\n", - " 2.0\n", + " 9222\n", + " 735.0\n", " 9\n", " Maya Lin: A Strong Clear Vision (1994)\n", " Documentary\n", " \n", " \n", - " 7536\n", - " 2.0\n", + " 8268\n", + " 735.0\n", " 10\n", " Prefontaine (1997)\n", " Drama\n", @@ -1572,32 +1654,32 @@ "" ], "text/plain": [ - " user rec_nb title \\\n", - "1 2.0 1 Great Day in Harlem, A (1994) \n", - "943 2.0 2 Tough and Deadly (1995) \n", - "1885 2.0 3 Aiqing wansui (1994) \n", - "2827 2.0 4 Delta of Venus (1994) \n", - "3769 2.0 5 Someone Else's America (1995) \n", - "4711 2.0 6 Saint of Fort Washington, The (1993) \n", - "5653 2.0 7 Celestial Clockwork (1994) \n", - "6595 2.0 8 Some Mother's Son (1996) \n", - "8489 2.0 9 Maya Lin: A Strong Clear Vision (1994) \n", - "7536 2.0 10 Prefontaine (1997) \n", + " user rec_nb title \\\n", + "733 735.0 1 Great Day in Harlem, A (1994) \n", + "1675 735.0 2 Tough and Deadly (1995) \n", + "2617 735.0 3 Aiqing wansui (1994) \n", + "3559 735.0 4 Delta of Venus (1994) \n", + "4501 735.0 5 Someone Else's America (1995) \n", + "5443 735.0 6 Saint of Fort Washington, The (1993) \n", + "6385 735.0 7 Celestial Clockwork (1994) \n", + "7326 735.0 8 Some Mother's Son (1996) \n", + "9222 735.0 9 Maya Lin: A Strong Clear Vision (1994) \n", + "8268 735.0 10 Prefontaine (1997) \n", "\n", " genres \n", - "1 Documentary \n", - "943 Action, Drama, Thriller \n", - "1885 Drama \n", - "2827 Drama \n", - "3769 Drama \n", - "4711 Drama \n", - "5653 Comedy \n", - "6595 Drama \n", - "8489 Documentary \n", - "7536 Drama " + "733 Documentary \n", + "1675 Action, Drama, Thriller \n", + "2617 Drama \n", + "3559 Drama \n", + "4501 Drama \n", + "5443 Drama \n", + "6385 Comedy \n", + "7326 Drama \n", + "9222 Documentary \n", + "8268 Drama " ] }, - "execution_count": 15, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" }