robkw/introduction_to_recommender_systems
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity
bdf06c9c28
introduction_to_recommender.../P2. Evaluation.ipynb

1775 lines
56 KiB
Plaintext
Raw Normal View History

first 2 lessons
2021-03-20 20:01:22 +01:00
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Prepare test set"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"slideshow": {
"slide_type": "-"
}
},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import scipy.sparse as sparse\n",
"from collections import defaultdict\n",
"from itertools import chain\n",
"import random\n",
"from tqdm import tqdm\n",
"\n",
"# In evaluation we do not load train set - it is not needed\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"test = pd.read_csv(\"./Datasets/ml-100k/test.csv\", sep=\"\\t\", header=None)\n",
"test.columns = [\"user\", \"item\", \"rating\", \"timestamp\"]\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"test[\"user_code\"] = test[\"user\"].astype(\"category\").cat.codes\n",
"test[\"item_code\"] = test[\"item\"].astype(\"category\").cat.codes\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"user_code_id = dict(enumerate(test[\"user\"].astype(\"category\").cat.categories))\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"user_id_code = dict((v, k) for k, v in user_code_id.items())\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"item_code_id = dict(enumerate(test[\"item\"].astype(\"category\").cat.categories))\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"item_id_code = dict((v, k) for k, v in item_code_id.items())\n",
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"test_ui = sparse.csr_matrix((test[\"rating\"], (test[\"user_code\"], test[\"item_code\"])))"
first 2 lessons
2021-03-20 20:01:22 +01:00
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Estimations metrics"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
2nd meeting
2021-04-16 22:41:06 +02:00
"estimations_df = pd.read_csv(\n",
" \"Recommendations generated/ml-100k/Ready_Baseline_estimations.csv\", header=None\n",
")\n",
"estimations_df.columns = [\"user\", \"item\", \"score\"]\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"estimations_df[\"user_code\"] = [user_id_code[user] for user in estimations_df[\"user\"]]\n",
"estimations_df[\"item_code\"] = [item_id_code[item] for item in estimations_df[\"item\"]]\n",
"estimations = sparse.csr_matrix(\n",
" (\n",
" estimations_df[\"score\"],\n",
" (estimations_df[\"user_code\"], estimations_df[\"item_code\"]),\n",
" ),\n",
" shape=test_ui.shape,\n",
")"
first 2 lessons
2021-03-20 20:01:22 +01:00
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"def estimations_metrics(test_ui, estimations):\n",
2nd meeting
2021-04-16 22:41:06 +02:00
" result = []\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
" RMSE = (np.sum((estimations.data - test_ui.data) ** 2) / estimations.nnz) ** (1 / 2)\n",
" result.append([\"RMSE\", RMSE])\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
" MAE = np.sum(abs(estimations.data - test_ui.data)) / estimations.nnz\n",
" result.append([\"MAE\", MAE])\n",
"\n",
" df_result = (pd.DataFrame(list(zip(*result))[1])).T\n",
" df_result.columns = list(zip(*result))[0]\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" return df_result"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" RMSE MAE\n",
"0 0.949459 0.752487"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# in case of error (in the laboratories) you might have to switch to the other version of pandas\n",
"# try !pip3 install pandas=='1.0.3' (or pip if you use python 2) and restart the kernel\n",
"\n",
"estimations_metrics(test_ui, estimations)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Ranking metrics"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[663, 475, 62, ..., 472, 269, 503],\n",
" [ 48, 313, 475, ..., 591, 175, 466],\n",
" [351, 313, 475, ..., 591, 175, 466],\n",
" ...,\n",
" [259, 313, 475, ..., 11, 591, 175],\n",
" [ 33, 313, 475, ..., 11, 591, 175],\n",
" [ 77, 313, 475, ..., 11, 591, 175]])"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import numpy as np\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"\n",
"reco = np.loadtxt(\n",
" \"Recommendations generated/ml-100k/Ready_Baseline_reco.csv\", delimiter=\",\"\n",
")\n",
"# Let's ignore scores - they are not used in evaluation:\n",
"users = reco[:, :1]\n",
"items = reco[:, 1::2]\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"# Let's use inner ids instead of real ones\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"users = np.vectorize(lambda x: user_id_code.setdefault(x, -1))(users)\n",
"items = np.vectorize(lambda x: item_id_code.setdefault(x, -1))(items)\n",
"reco = np.concatenate((users, items), axis=1)\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"reco"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"def ranking_metrics(test_ui, reco, super_reactions=[], topK=10):\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"\n",
" nb_items = test_ui.shape[1]\n",
" (\n",
" relevant_users,\n",
" super_relevant_users,\n",
" prec,\n",
" rec,\n",
" F_1,\n",
" F_05,\n",
" prec_super,\n",
" rec_super,\n",
" ndcg,\n",
" mAP,\n",
" MRR,\n",
" LAUC,\n",
" HR,\n",
" ) = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)\n",
"\n",
" cg = 1.0 / np.log2(np.arange(2, topK + 2))\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" cg_sum = np.cumsum(cg)\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"\n",
" for (nb_user, user) in tqdm(enumerate(reco[:, 0])):\n",
" u_rated_items = test_ui.indices[test_ui.indptr[user] : test_ui.indptr[user + 1]]\n",
" nb_u_rated_items = len(u_rated_items)\n",
" if (\n",
" nb_u_rated_items > 0\n",
" ): # skip users with no items in test set (still possible that there will be no super items)\n",
" relevant_users += 1\n",
"\n",
" u_super_items = u_rated_items[\n",
" np.vectorize(lambda x: x in super_reactions)(\n",
" test_ui.data[test_ui.indptr[user] : test_ui.indptr[user + 1]]\n",
" )\n",
" ]\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" # more natural seems u_super_items=[item for item in u_rated_items if test_ui[user,item] in super_reactions]\n",
" # but accesing test_ui[user,item] is expensive -we should avoid doing it\n",
2nd meeting
2021-04-16 22:41:06 +02:00
" if len(u_super_items) > 0:\n",
" super_relevant_users += 1\n",
"\n",
" user_successes = np.zeros(topK)\n",
" nb_user_successes = 0\n",
" user_super_successes = np.zeros(topK)\n",
" nb_user_super_successes = 0\n",
"\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" # evaluation\n",
2nd meeting
2021-04-16 22:41:06 +02:00
" for (item_position, item) in enumerate(reco[nb_user, 1 : topK + 1]):\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" if item in u_rated_items:\n",
2nd meeting
2021-04-16 22:41:06 +02:00
" user_successes[item_position] = 1\n",
" nb_user_successes += 1\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" if item in u_super_items:\n",
2nd meeting
2021-04-16 22:41:06 +02:00
" user_super_successes[item_position] = 1\n",
" nb_user_super_successes += 1\n",
"\n",
" prec_u = nb_user_successes / topK\n",
" prec += prec_u\n",
"\n",
" rec_u = nb_user_successes / nb_u_rated_items\n",
" rec += rec_u\n",
"\n",
" F_1 += 2 * (prec_u * rec_u) / (prec_u + rec_u) if prec_u + rec_u > 0 else 0\n",
" F_05 += (\n",
" (0.5 ** 2 + 1) * (prec_u * rec_u) / (0.5 ** 2 * prec_u + rec_u)\n",
" if prec_u + rec_u > 0\n",
" else 0\n",
" )\n",
"\n",
" prec_super += nb_user_super_successes / topK\n",
" rec_super += nb_user_super_successes / max(\n",
" len(u_super_items), 1\n",
" ) # to set 0 if no super items\n",
" ndcg += np.dot(user_successes, cg) / cg_sum[min(topK, nb_u_rated_items) - 1]\n",
"\n",
" cumsum_successes = np.cumsum(user_successes)\n",
" mAP += np.dot(\n",
" cumsum_successes / np.arange(1, topK + 1), user_successes\n",
" ) / min(topK, nb_u_rated_items)\n",
" MRR += (\n",
" 1 / (user_successes.nonzero()[0][0] + 1)\n",
" if user_successes.nonzero()[0].size > 0\n",
" else 0\n",
" )\n",
" LAUC += (\n",
" np.dot(cumsum_successes, 1 - user_successes)\n",
" + (nb_user_successes + nb_u_rated_items)\n",
" / 2\n",
" * ((nb_items - nb_u_rated_items) - (topK - nb_user_successes))\n",
" ) / ((nb_items - nb_u_rated_items) * nb_u_rated_items)\n",
"\n",
" HR += nb_user_successes > 0\n",
"\n",
" result = []\n",
" result.append((\"precision\", prec / relevant_users))\n",
" result.append((\"recall\", rec / relevant_users))\n",
" result.append((\"F_1\", F_1 / relevant_users))\n",
" result.append((\"F_05\", F_05 / relevant_users))\n",
" result.append((\"precision_super\", prec_super / super_relevant_users))\n",
" result.append((\"recall_super\", rec_super / super_relevant_users))\n",
" result.append((\"NDCG\", ndcg / relevant_users))\n",
" result.append((\"mAP\", mAP / relevant_users))\n",
" result.append((\"MRR\", MRR / relevant_users))\n",
" result.append((\"LAUC\", LAUC / relevant_users))\n",
" result.append((\"HR\", HR / relevant_users))\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
" df_result = (pd.DataFrame(list(zip(*result))[1])).T\n",
" df_result.columns = list(zip(*result))[0]\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" return df_result"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
2nd meeting
2021-04-16 22:41:06 +02:00
"943it [00:00, 9434.06it/s]\n"
first 2 lessons
2021-03-20 20:01:22 +01:00
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.09141</td>\n",
" <td>0.037652</td>\n",
" <td>0.04603</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" precision recall F_1 F_05 precision_super recall_super \\\n",
"0 0.09141 0.037652 0.04603 0.061286 0.079614 0.056463 \n",
"\n",
" NDCG mAP MRR LAUC HR \n",
"0 0.095957 0.043178 0.198193 0.515501 0.437964 "
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
2nd meeting
2021-04-16 22:41:06 +02:00
"ranking_metrics(test_ui, reco, super_reactions=[4, 5], topK=10)"
first 2 lessons
2021-03-20 20:01:22 +01:00
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Diversity metrics"
]
},
{
"cell_type": "code",
Zadanie 3
2021-06-12 10:56:22 +02:00
"execution_count": 7,
first 2 lessons
2021-03-20 20:01:22 +01:00
"metadata": {},
"outputs": [],
"source": [
"def diversity_metrics(test_ui, reco, topK=10):\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"\n",
" frequencies = defaultdict(int)\n",
"\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" # let's assign 0 to all items in test set\n",
" for item in list(set(test_ui.indices)):\n",
2nd meeting
2021-04-16 22:41:06 +02:00
" frequencies[item] = 0\n",
"\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" # counting frequencies\n",
2nd meeting
2021-04-16 22:41:06 +02:00
" for item in reco[:, 1:].flat:\n",
" frequencies[item] += 1\n",
"\n",
" nb_reco_outside_test = frequencies[-1]\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" del frequencies[-1]\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"\n",
" frequencies = np.array(list(frequencies.values()))\n",
"\n",
" nb_rec_items = len(frequencies[frequencies > 0])\n",
" nb_reco_inside_test = np.sum(frequencies)\n",
"\n",
" frequencies = frequencies / np.sum(frequencies)\n",
" frequencies = np.sort(frequencies)\n",
"\n",
" with np.errstate(\n",
" divide=\"ignore\"\n",
" ): # let's put zeros put items with 0 frequency and ignore division warning\n",
" log_frequencies = np.nan_to_num(np.log(frequencies), posinf=0, neginf=0)\n",
"\n",
" result = []\n",
" result.append(\n",
" (\n",
" \"Reco in test\",\n",
" nb_reco_inside_test / (nb_reco_inside_test + nb_reco_outside_test),\n",
" )\n",
" )\n",
" result.append((\"Test coverage\", nb_rec_items / test_ui.shape[1]))\n",
" result.append((\"Shannon\", -np.dot(frequencies, log_frequencies)))\n",
" result.append(\n",
" (\n",
" \"Gini\",\n",
" np.dot(frequencies, np.arange(1 - len(frequencies), len(frequencies), 2))\n",
" / (len(frequencies) - 1),\n",
" )\n",
" )\n",
"\n",
" df_result = (pd.DataFrame(list(zip(*result))[1])).T\n",
" df_result.columns = list(zip(*result))[0]\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" return df_result"
]
},
{
"cell_type": "code",
Zadanie 3
2021-06-12 10:56:22 +02:00
"execution_count": 8,
first 2 lessons
2021-03-20 20:01:22 +01:00
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1.0</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Reco in test Test coverage Shannon Gini\n",
"0 1.0 0.033911 2.836513 0.991139"
]
},
Zadanie 3
2021-06-12 10:56:22 +02:00
"execution_count": 8,
first 2 lessons
2021-03-20 20:01:22 +01:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# in case of errors try !pip3 install numpy==1.18.4 (or pip if you use python 2) and restart the kernel\n",
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"x = diversity_metrics(test_ui, reco, topK=10)\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"x"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# To be used in other notebooks"
]
},
{
"cell_type": "code",
Zadanie 3
2021-06-12 10:56:22 +02:00
"execution_count": 9,
first 2 lessons
2021-03-20 20:01:22 +01:00
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
Zadanie 3
2021-06-12 10:56:22 +02:00
"943it [00:00, 12952.59it/s]\n"
first 2 lessons
2021-03-20 20:01:22 +01:00
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>HitRate2</th>\n",
" <th>HitRate3</th>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" <td>0.09141</td>\n",
" <td>0.037652</td>\n",
" <td>0.04603</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <td>0.239661</td>\n",
" <td>0.126193</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <td>1.0</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" RMSE MAE precision recall F_1 F_05 \\\n",
"0 0.949459 0.752487 0.09141 0.037652 0.04603 0.061286 \n",
"\n",
" precision_super recall_super NDCG mAP MRR LAUC \\\n",
"0 0.079614 0.056463 0.095957 0.043178 0.198193 0.515501 \n",
"\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" HR HitRate2 HitRate3 Reco in test Test coverage Shannon \\\n",
"0 0.437964 0.239661 0.126193 1.0 0.033911 2.836513 \n",
"\n",
" Gini \n",
"0 0.991139 "
first 2 lessons
2021-03-20 20:01:22 +01:00
]
},
Zadanie 3
2021-06-12 10:56:22 +02:00
"execution_count": 9,
first 2 lessons
2021-03-20 20:01:22 +01:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import evaluation_measures as ev\n",
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"estimations_df = pd.read_csv(\n",
" \"Recommendations generated/ml-100k/Ready_Baseline_estimations.csv\", header=None\n",
")\n",
"reco = np.loadtxt(\n",
" \"Recommendations generated/ml-100k/Ready_Baseline_reco.csv\", delimiter=\",\"\n",
")\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"ev.evaluate(\n",
" test=pd.read_csv(\"./Datasets/ml-100k/test.csv\", sep=\"\\t\", header=None),\n",
" estimations_df=estimations_df,\n",
" reco=reco,\n",
" super_reactions=[4, 5],\n",
")\n",
"# also you can just type ev.evaluate_all(estimations_df, reco) - I put above values as default"
first 2 lessons
2021-03-20 20:01:22 +01:00
]
},
{
"cell_type": "code",
Zadanie 3
2021-06-12 10:56:22 +02:00
"execution_count": 10,
first 2 lessons
2021-03-20 20:01:22 +01:00
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
Zadanie 3
2021-06-12 10:56:22 +02:00
"943it [00:00, 13130.52it/s]\n",
"943it [00:00, 12777.31it/s]\n",
"943it [00:00, 13513.65it/s]\n",
"943it [00:00, 13323.06it/s]\n",
"943it [00:00, 13507.69it/s]\n",
"943it [00:00, 13697.48it/s]\n"
first 2 lessons
2021-03-20 20:01:22 +01:00
]
}
],
"source": [
2nd meeting
2021-04-16 22:41:06 +02:00
"dir_path = \"Recommendations generated/ml-100k/\"\n",
"super_reactions = [4, 5]\n",
"test = pd.read_csv(\"./Datasets/ml-100k/test.csv\", sep=\"\\t\", header=None)\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"df = ev.evaluate_all(test, dir_path, super_reactions)\n",
"# also you can just type ev.evaluate_all() - I put above values as default"
first 2 lessons
2021-03-20 20:01:22 +01:00
]
},
{
"cell_type": "code",
Zadanie 3
2021-06-12 10:56:22 +02:00
"execution_count": 11,
first 2 lessons
2021-03-20 20:01:22 +01:00
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_TopPop</td>\n",
" <td>2.508258</td>\n",
" <td>2.217909</td>\n",
" <td>0.188865</td>\n",
" <td>0.116919</td>\n",
" <td>0.118732</td>\n",
" <td>0.141584</td>\n",
" <td>0.130472</td>\n",
" <td>0.137473</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Baseline</td>\n",
" <td>0.949459</td>\n",
" <td>0.752487</td>\n",
" <td>0.091410</td>\n",
" <td>0.037652</td>\n",
" <td>0.046030</td>\n",
" <td>0.061286</td>\n",
" <td>0.079614</td>\n",
" <td>0.056463</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
finished first 2 lectures
2021-03-23 21:52:46 +01:00
" <td>Ready_Random</td>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <td>1.516512</td>\n",
" <td>1.217214</td>\n",
" <td>0.045599</td>\n",
" <td>0.021001</td>\n",
" <td>0.024136</td>\n",
" <td>0.031226</td>\n",
" <td>0.028541</td>\n",
" <td>0.022057</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
finished first 2 lectures
2021-03-23 21:52:46 +01:00
" <td>Self_TopRated</td>\n",
" <td>1.030712</td>\n",
" <td>0.820904</td>\n",
" <td>0.000954</td>\n",
" <td>0.000188</td>\n",
" <td>0.000298</td>\n",
" <td>0.000481</td>\n",
" <td>0.000644</td>\n",
" <td>0.000223</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineUI</td>\n",
" <td>0.967585</td>\n",
" <td>0.762740</td>\n",
" <td>0.000954</td>\n",
" <td>0.000170</td>\n",
" <td>0.000278</td>\n",
" <td>0.000463</td>\n",
" <td>0.000644</td>\n",
" <td>0.000189</td>\n",
" </tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <tr>\n",
" <th>0</th>\n",
" <td>Self_IKNN</td>\n",
" <td>1.018363</td>\n",
" <td>0.808793</td>\n",
" <td>0.000318</td>\n",
" <td>0.000108</td>\n",
" <td>0.000140</td>\n",
" <td>0.000189</td>\n",
" <td>0.000000</td>\n",
" <td>0.000000</td>\n",
" </tr>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
slides updated
2021-03-20 20:13:28 +01:00
" Model RMSE MAE precision recall F_1 \\\n",
"0 Self_TopPop 2.508258 2.217909 0.188865 0.116919 0.118732 \n",
"0 Ready_Baseline 0.949459 0.752487 0.091410 0.037652 0.046030 \n",
Zadanie 3
2021-06-12 10:56:22 +02:00
"0 Ready_Random 1.516512 1.217214 0.045599 0.021001 0.024136 \n",
finished first 2 lectures
2021-03-23 21:52:46 +01:00
"0 Self_TopRated 1.030712 0.820904 0.000954 0.000188 0.000298 \n",
slides updated
2021-03-20 20:13:28 +01:00
"0 Self_BaselineUI 0.967585 0.762740 0.000954 0.000170 0.000278 \n",
Zadanie 3
2021-06-12 10:56:22 +02:00
"0 Self_IKNN 1.018363 0.808793 0.000318 0.000108 0.000140 \n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
slides updated
2021-03-20 20:13:28 +01:00
" F_05 precision_super recall_super \n",
"0 0.141584 0.130472 0.137473 \n",
"0 0.061286 0.079614 0.056463 \n",
Zadanie 3
2021-06-12 10:56:22 +02:00
"0 0.031226 0.028541 0.022057 \n",
finished first 2 lectures
2021-03-23 21:52:46 +01:00
"0 0.000481 0.000644 0.000223 \n",
Zadanie 3
2021-06-12 10:56:22 +02:00
"0 0.000463 0.000644 0.000189 \n",
"0 0.000189 0.000000 0.000000 "
first 2 lessons
2021-03-20 20:01:22 +01:00
]
},
Zadanie 3
2021-06-12 10:56:22 +02:00
"execution_count": 11,
first 2 lessons
2021-03-20 20:01:22 +01:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
2nd meeting
2021-04-16 22:41:06 +02:00
"df.iloc[:, :9]"
first 2 lessons
2021-03-20 20:01:22 +01:00
]
},
{
"cell_type": "code",
Zadanie 3
2021-06-12 10:56:22 +02:00
"execution_count": 12,
first 2 lessons
2021-03-20 20:01:22 +01:00
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>HitRate2</th>\n",
" <th>HitRate3</th>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_TopPop</td>\n",
" <td>0.214651</td>\n",
" <td>0.111707</td>\n",
" <td>0.400939</td>\n",
" <td>0.555546</td>\n",
" <td>0.765642</td>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <td>0.492047</td>\n",
" <td>0.290562</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <td>1.000000</td>\n",
" <td>0.038961</td>\n",
" <td>3.159079</td>\n",
" <td>0.987317</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Ready_Baseline</td>\n",
" <td>0.095957</td>\n",
" <td>0.043178</td>\n",
" <td>0.198193</td>\n",
" <td>0.515501</td>\n",
" <td>0.437964</td>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <td>0.239661</td>\n",
" <td>0.126193</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <td>1.000000</td>\n",
" <td>0.033911</td>\n",
" <td>2.836513</td>\n",
" <td>0.991139</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
finished first 2 lectures
2021-03-23 21:52:46 +01:00
" <td>Ready_Random</td>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <td>0.050154</td>\n",
" <td>0.019000</td>\n",
" <td>0.125089</td>\n",
" <td>0.507013</td>\n",
" <td>0.327678</td>\n",
" <td>0.093319</td>\n",
" <td>0.026511</td>\n",
" <td>0.988017</td>\n",
" <td>0.192641</td>\n",
" <td>5.141246</td>\n",
" <td>0.903763</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
finished first 2 lectures
2021-03-23 21:52:46 +01:00
" <td>Self_TopRated</td>\n",
" <td>0.001043</td>\n",
" <td>0.000335</td>\n",
" <td>0.003348</td>\n",
" <td>0.496433</td>\n",
" <td>0.009544</td>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <td>0.000000</td>\n",
" <td>0.000000</td>\n",
finished first 2 lectures
2021-03-23 21:52:46 +01:00
" <td>0.699046</td>\n",
" <td>0.005051</td>\n",
" <td>1.945910</td>\n",
" <td>0.995669</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineUI</td>\n",
" <td>0.000752</td>\n",
" <td>0.000168</td>\n",
" <td>0.001677</td>\n",
" <td>0.496424</td>\n",
" <td>0.009544</td>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <td>0.000000</td>\n",
" <td>0.000000</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <td>0.600530</td>\n",
" <td>0.005051</td>\n",
" <td>1.803126</td>\n",
" <td>0.996380</td>\n",
" </tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <tr>\n",
" <th>0</th>\n",
" <td>Self_IKNN</td>\n",
" <td>0.000214</td>\n",
" <td>0.000037</td>\n",
" <td>0.000368</td>\n",
" <td>0.496391</td>\n",
" <td>0.003181</td>\n",
" <td>0.000000</td>\n",
" <td>0.000000</td>\n",
" <td>0.392153</td>\n",
" <td>0.115440</td>\n",
" <td>4.174741</td>\n",
" <td>0.965327</td>\n",
" </tr>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
slides updated
2021-03-20 20:13:28 +01:00
" Model NDCG mAP MRR LAUC HR \\\n",
"0 Self_TopPop 0.214651 0.111707 0.400939 0.555546 0.765642 \n",
"0 Ready_Baseline 0.095957 0.043178 0.198193 0.515501 0.437964 \n",
Zadanie 3
2021-06-12 10:56:22 +02:00
"0 Ready_Random 0.050154 0.019000 0.125089 0.507013 0.327678 \n",
finished first 2 lectures
2021-03-23 21:52:46 +01:00
"0 Self_TopRated 0.001043 0.000335 0.003348 0.496433 0.009544 \n",
slides updated
2021-03-20 20:13:28 +01:00
"0 Self_BaselineUI 0.000752 0.000168 0.001677 0.496424 0.009544 \n",
Zadanie 3
2021-06-12 10:56:22 +02:00
"0 Self_IKNN 0.000214 0.000037 0.000368 0.496391 0.003181 \n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" HitRate2 HitRate3 Reco in test Test coverage Shannon Gini \n",
"0 0.492047 0.290562 1.000000 0.038961 3.159079 0.987317 \n",
"0 0.239661 0.126193 1.000000 0.033911 2.836513 0.991139 \n",
"0 0.093319 0.026511 0.988017 0.192641 5.141246 0.903763 \n",
"0 0.000000 0.000000 0.699046 0.005051 1.945910 0.995669 \n",
"0 0.000000 0.000000 0.600530 0.005051 1.803126 0.996380 \n",
"0 0.000000 0.000000 0.392153 0.115440 4.174741 0.965327 "
first 2 lessons
2021-03-20 20:01:22 +01:00
]
},
Zadanie 3
2021-06-12 10:56:22 +02:00
"execution_count": 12,
first 2 lessons
2021-03-20 20:01:22 +01:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
2nd meeting
2021-04-16 22:41:06 +02:00
"df.iloc[:, np.append(0, np.arange(9, df.shape[1]))]"
first 2 lessons
2021-03-20 20:01:22 +01:00
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Check metrics on toy dataset"
]
},
{
"cell_type": "code",
Zadanie 3
2021-06-12 10:56:22 +02:00
"execution_count": 13,
first 2 lessons
2021-03-20 20:01:22 +01:00
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
Zadanie 3
2021-06-12 10:56:22 +02:00
"3it [00:00, ?it/s]\n",
"3it [00:00, ?it/s]\n"
first 2 lessons
2021-03-20 20:01:22 +01:00
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Model</th>\n",
" <th>RMSE</th>\n",
" <th>MAE</th>\n",
" <th>precision</th>\n",
" <th>recall</th>\n",
" <th>F_1</th>\n",
" <th>F_05</th>\n",
" <th>precision_super</th>\n",
" <th>recall_super</th>\n",
" <th>NDCG</th>\n",
" <th>mAP</th>\n",
" <th>MRR</th>\n",
" <th>LAUC</th>\n",
" <th>HR</th>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>HitRate2</th>\n",
" <th>HitRate3</th>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <th>Reco in test</th>\n",
" <th>Test coverage</th>\n",
" <th>Shannon</th>\n",
" <th>Gini</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineUI</td>\n",
" <td>1.612452</td>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <td>1.400</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <td>0.444444</td>\n",
" <td>0.888889</td>\n",
" <td>0.555556</td>\n",
" <td>0.478632</td>\n",
" <td>0.333333</td>\n",
" <td>0.75</td>\n",
" <td>0.676907</td>\n",
" <td>0.574074</td>\n",
" <td>0.611111</td>\n",
" <td>0.638889</td>\n",
" <td>1.0</td>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <td>0.333333</td>\n",
" <td>0.0</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <td>0.888889</td>\n",
" <td>0.8</td>\n",
" <td>1.386294</td>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <td>0.250000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Self_BaselineIU</td>\n",
" <td>1.648337</td>\n",
" <td>1.575</td>\n",
" <td>0.444444</td>\n",
" <td>0.888889</td>\n",
" <td>0.555556</td>\n",
" <td>0.478632</td>\n",
" <td>0.333333</td>\n",
" <td>0.75</td>\n",
" <td>0.720550</td>\n",
" <td>0.629630</td>\n",
" <td>0.666667</td>\n",
" <td>0.722222</td>\n",
" <td>1.0</td>\n",
" <td>0.333333</td>\n",
" <td>0.0</td>\n",
" <td>0.777778</td>\n",
" <td>0.8</td>\n",
" <td>1.351784</td>\n",
" <td>0.357143</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
Zadanie 3
2021-06-12 10:56:22 +02:00
" Model RMSE MAE precision recall F_1 F_05 \\\n",
"0 Self_BaselineUI 1.612452 1.400 0.444444 0.888889 0.555556 0.478632 \n",
"0 Self_BaselineIU 1.648337 1.575 0.444444 0.888889 0.555556 0.478632 \n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
" precision_super recall_super NDCG mAP MRR LAUC HR \\\n",
"0 0.333333 0.75 0.676907 0.574074 0.611111 0.638889 1.0 \n",
Zadanie 3
2021-06-12 10:56:22 +02:00
"0 0.333333 0.75 0.720550 0.629630 0.666667 0.722222 1.0 \n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" HitRate2 HitRate3 Reco in test Test coverage Shannon Gini \n",
"0 0.333333 0.0 0.888889 0.8 1.386294 0.250000 \n",
"0 0.333333 0.0 0.777778 0.8 1.351784 0.357143 "
first 2 lessons
2021-03-20 20:01:22 +01:00
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Training data:\n"
]
},
{
"data": {
"text/plain": [
"matrix([[3, 4, 0, 0, 5, 0, 0, 4],\n",
" [0, 1, 2, 3, 0, 0, 0, 0],\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" [0, 0, 0, 5, 0, 3, 4, 0]], dtype=int64)"
first 2 lessons
2021-03-20 20:01:22 +01:00
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Test data:\n"
]
},
{
"data": {
"text/plain": [
"matrix([[0, 0, 0, 0, 0, 0, 3, 0],\n",
" [0, 0, 0, 0, 5, 0, 0, 0],\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" [5, 0, 4, 0, 0, 0, 0, 2]], dtype=int64)"
first 2 lessons
2021-03-20 20:01:22 +01:00
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Recommendations:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>0</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" <th>3</th>\n",
" <th>4</th>\n",
" <th>5</th>\n",
" <th>6</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0</td>\n",
" <td>30</td>\n",
" <td>5.0</td>\n",
" <td>20</td>\n",
" <td>4.0</td>\n",
" <td>60</td>\n",
" <td>4.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>10</td>\n",
" <td>40</td>\n",
" <td>3.0</td>\n",
" <td>60</td>\n",
" <td>2.0</td>\n",
" <td>70</td>\n",
" <td>2.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>20</td>\n",
" <td>40</td>\n",
" <td>5.0</td>\n",
" <td>20</td>\n",
" <td>4.0</td>\n",
" <td>70</td>\n",
" <td>4.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" 0 1 2 3 4 5 6\n",
"0 0 30 5.0 20 4.0 60 4.0\n",
"1 10 40 3.0 60 2.0 70 2.0\n",
"2 20 40 5.0 20 4.0 70 4.0"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Estimations:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>user</th>\n",
" <th>item</th>\n",
" <th>est_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0</td>\n",
" <td>60</td>\n",
" <td>4.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>10</td>\n",
" <td>40</td>\n",
" <td>3.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>20</td>\n",
" <td>0</td>\n",
" <td>3.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>20</td>\n",
" <td>20</td>\n",
" <td>4.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>20</td>\n",
" <td>70</td>\n",
" <td>4.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" user item est_score\n",
"0 0 60 4.0\n",
"1 10 40 3.0\n",
"2 20 0 3.0\n",
"3 20 20 4.0\n",
"4 20 70 4.0"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import helpers\n",
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"dir_path = \"Recommendations generated/toy-example/\"\n",
"super_reactions = [4, 5]\n",
"test = pd.read_csv(\"./Datasets/toy-example/test.csv\", sep=\"\\t\", header=None)\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
"display(ev.evaluate_all(test, dir_path, super_reactions, topK=3))\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"# also you can just type ev.evaluate_all() - I put above values as default\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"toy_train_read = pd.read_csv(\n",
" \"./Datasets/toy-example/train.csv\",\n",
" sep=\"\\t\",\n",
" header=None,\n",
" names=[\"user\", \"item\", \"rating\", \"timestamp\"],\n",
")\n",
"toy_test_read = pd.read_csv(\n",
" \"./Datasets/toy-example/test.csv\",\n",
" sep=\"\\t\",\n",
" header=None,\n",
" names=[\"user\", \"item\", \"rating\", \"timestamp\"],\n",
")\n",
"reco = pd.read_csv(\n",
" \"Recommendations generated/toy-example/Self_BaselineUI_reco.csv\", header=None\n",
")\n",
"estimations = pd.read_csv(\n",
" \"Recommendations generated/toy-example/Self_BaselineUI_estimations.csv\",\n",
" names=[\"user\", \"item\", \"est_score\"],\n",
")\n",
"(\n",
" toy_train_ui,\n",
" toy_test_ui,\n",
" toy_user_code_id,\n",
" toy_user_id_code,\n",
" toy_item_code_id,\n",
" toy_item_id_code,\n",
") = helpers.data_to_csr(toy_train_read, toy_test_read)\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"print(\"Training data:\")\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"display(toy_train_ui.todense())\n",
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"print(\"Test data:\")\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"display(toy_test_ui.todense())\n",
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"print(\"Recommendations:\")\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"display(reco)\n",
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"print(\"Estimations:\")\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"display(estimations)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Sample recommendations"
]
},
{
"cell_type": "code",
Zadanie 3
2021-06-12 10:56:22 +02:00
"execution_count": 14,
first 2 lessons
2021-03-20 20:01:22 +01:00
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Here is what user rated high:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>user</th>\n",
" <th>rating</th>\n",
" <th>title</th>\n",
" <th>genres</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>36652</th>\n",
" <td>735</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <td>5</td>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <td>Lone Star (1996)</td>\n",
" <td>Drama, Mystery</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" </tr>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>3143</th>\n",
" <td>735</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <td>5</td>\n",
2nd meeting
2021-04-16 22:41:06 +02:00
" <td>Star Wars (1977)</td>\n",
" <td>Action, Adventure, Romance, Sci-Fi, War</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" </tr>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>52919</th>\n",
" <td>735</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <td>5</td>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <td>Kolya (1996)</td>\n",
" <td>Comedy</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" </tr>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>275</th>\n",
" <td>735</td>\n",
" <td>4</td>\n",
" <td>Toy Story (1995)</td>\n",
" <td>Animation, Children's, Comedy</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" </tr>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>41134</th>\n",
" <td>735</td>\n",
" <td>4</td>\n",
" <td>Trainspotting (1996)</td>\n",
" <td>Drama</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" </tr>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>28094</th>\n",
" <td>735</td>\n",
" <td>4</td>\n",
" <td>Face/Off (1997)</td>\n",
" <td>Action, Sci-Fi, Thriller</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" </tr>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>26548</th>\n",
" <td>735</td>\n",
" <td>4</td>\n",
" <td>Everyone Says I Love You (1996)</td>\n",
" <td>Comedy, Musical, Romance</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" </tr>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>26186</th>\n",
" <td>735</td>\n",
" <td>4</td>\n",
" <td>Air Force One (1997)</td>\n",
" <td>Action, Thriller</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" </tr>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>25791</th>\n",
" <td>735</td>\n",
" <td>4</td>\n",
" <td>Dead Man Walking (1995)</td>\n",
2nd meeting
2021-04-16 22:41:06 +02:00
" <td>Drama</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" </tr>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>51948</th>\n",
" <td>735</td>\n",
" <td>4</td>\n",
" <td>Mighty Aphrodite (1995)</td>\n",
2nd meeting
2021-04-16 22:41:06 +02:00
" <td>Comedy</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" </tr>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>52778</th>\n",
" <td>735</td>\n",
last changes
2021-03-26 21:00:52 +01:00
" <td>4</td>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <td>Fly Away Home (1996)</td>\n",
" <td>Adventure, Children's</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" </tr>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>20966</th>\n",
" <td>735</td>\n",
last changes
2021-03-26 21:00:52 +01:00
" <td>4</td>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <td>Secrets &amp; Lies (1996)</td>\n",
" <td>Drama</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" </tr>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>19301</th>\n",
" <td>735</td>\n",
last changes
2021-03-26 21:00:52 +01:00
" <td>4</td>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <td>Scream (1996)</td>\n",
" <td>Horror, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
" <th>54450</th>\n",
" <td>735</td>\n",
" <td>4</td>\n",
" <td>Sense and Sensibility (1995)</td>\n",
" <td>Drama, Romance</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17177</th>\n",
" <td>735</td>\n",
" <td>4</td>\n",
" <td>Leaving Las Vegas (1995)</td>\n",
" <td>Drama, Romance</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
Zadanie 3
2021-06-12 10:56:22 +02:00
" user rating title \\\n",
"36652 735 5 Lone Star (1996) \n",
"3143 735 5 Star Wars (1977) \n",
"52919 735 5 Kolya (1996) \n",
"275 735 4 Toy Story (1995) \n",
"41134 735 4 Trainspotting (1996) \n",
"28094 735 4 Face/Off (1997) \n",
"26548 735 4 Everyone Says I Love You (1996) \n",
"26186 735 4 Air Force One (1997) \n",
"25791 735 4 Dead Man Walking (1995) \n",
"51948 735 4 Mighty Aphrodite (1995) \n",
"52778 735 4 Fly Away Home (1996) \n",
"20966 735 4 Secrets & Lies (1996) \n",
"19301 735 4 Scream (1996) \n",
"54450 735 4 Sense and Sensibility (1995) \n",
"17177 735 4 Leaving Las Vegas (1995) \n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
" genres \n",
Zadanie 3
2021-06-12 10:56:22 +02:00
"36652 Drama, Mystery \n",
"3143 Action, Adventure, Romance, Sci-Fi, War \n",
"52919 Comedy \n",
"275 Animation, Children's, Comedy \n",
"41134 Drama \n",
"28094 Action, Sci-Fi, Thriller \n",
"26548 Comedy, Musical, Romance \n",
"26186 Action, Thriller \n",
"25791 Drama \n",
"51948 Comedy \n",
"52778 Adventure, Children's \n",
"20966 Drama \n",
"19301 Horror, Thriller \n",
"54450 Drama, Romance \n",
"17177 Drama, Romance "
first 2 lessons
2021-03-20 20:01:22 +01:00
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Here is what we recommend:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>user</th>\n",
" <th>rec_nb</th>\n",
" <th>title</th>\n",
" <th>genres</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>733</th>\n",
" <td>735.0</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <td>1</td>\n",
" <td>Great Day in Harlem, A (1994)</td>\n",
" <td>Documentary</td>\n",
" </tr>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>1675</th>\n",
" <td>735.0</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <td>2</td>\n",
" <td>Tough and Deadly (1995)</td>\n",
" <td>Action, Drama, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>2617</th>\n",
" <td>735.0</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <td>3</td>\n",
" <td>Aiqing wansui (1994)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>3559</th>\n",
" <td>735.0</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <td>4</td>\n",
" <td>Delta of Venus (1994)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>4501</th>\n",
" <td>735.0</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <td>5</td>\n",
slides updated
2021-03-20 20:13:28 +01:00
" <td>Someone Else's America (1995)</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>5443</th>\n",
" <td>735.0</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <td>6</td>\n",
slides updated
2021-03-20 20:13:28 +01:00
" <td>Saint of Fort Washington, The (1993)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>6385</th>\n",
" <td>735.0</td>\n",
slides updated
2021-03-20 20:13:28 +01:00
" <td>7</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <td>Celestial Clockwork (1994)</td>\n",
" <td>Comedy</td>\n",
" </tr>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>7326</th>\n",
" <td>735.0</td>\n",
slides updated
2021-03-20 20:13:28 +01:00
" <td>8</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <td>Some Mother's Son (1996)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>9222</th>\n",
" <td>735.0</td>\n",
slides updated
2021-03-20 20:13:28 +01:00
" <td>9</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <td>Maya Lin: A Strong Clear Vision (1994)</td>\n",
" <td>Documentary</td>\n",
" </tr>\n",
" <tr>\n",
Zadanie 3
2021-06-12 10:56:22 +02:00
" <th>8268</th>\n",
" <td>735.0</td>\n",
slides updated
2021-03-20 20:13:28 +01:00
" <td>10</td>\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
" <td>Prefontaine (1997)</td>\n",
" <td>Drama</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
Zadanie 3
2021-06-12 10:56:22 +02:00
" user rec_nb title \\\n",
"733 735.0 1 Great Day in Harlem, A (1994) \n",
"1675 735.0 2 Tough and Deadly (1995) \n",
"2617 735.0 3 Aiqing wansui (1994) \n",
"3559 735.0 4 Delta of Venus (1994) \n",
"4501 735.0 5 Someone Else's America (1995) \n",
"5443 735.0 6 Saint of Fort Washington, The (1993) \n",
"6385 735.0 7 Celestial Clockwork (1994) \n",
"7326 735.0 8 Some Mother's Son (1996) \n",
"9222 735.0 9 Maya Lin: A Strong Clear Vision (1994) \n",
"8268 735.0 10 Prefontaine (1997) \n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
" genres \n",
Zadanie 3
2021-06-12 10:56:22 +02:00
"733 Documentary \n",
"1675 Action, Drama, Thriller \n",
"2617 Drama \n",
"3559 Drama \n",
"4501 Drama \n",
"5443 Drama \n",
"6385 Comedy \n",
"7326 Drama \n",
"9222 Documentary \n",
"8268 Drama "
first 2 lessons
2021-03-20 20:01:22 +01:00
]
},
Zadanie 3
2021-06-12 10:56:22 +02:00
"execution_count": 14,
first 2 lessons
2021-03-20 20:01:22 +01:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
2nd meeting
2021-04-16 22:41:06 +02:00
"train = pd.read_csv(\n",
" \"./Datasets/ml-100k/train.csv\",\n",
" sep=\"\\t\",\n",
" header=None,\n",
" names=[\"user\", \"item\", \"rating\", \"timestamp\"],\n",
")\n",
"items = pd.read_csv(\"./Datasets/ml-100k/movies.csv\")\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"user = random.choice(list(set(train[\"user\"])))\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"train_content = pd.merge(train, items, left_on=\"item\", right_on=\"id\")\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"print(\"Here is what user rated high:\")\n",
"display(\n",
" train_content[train_content[\"user\"] == user][\n",
" [\"user\", \"rating\", \"title\", \"genres\"]\n",
" ].sort_values(by=\"rating\", ascending=False)[:15]\n",
")\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"reco = np.loadtxt(\n",
" \"Recommendations generated/ml-100k/Self_BaselineUI_reco.csv\", delimiter=\",\"\n",
")\n",
"items = pd.read_csv(\"./Datasets/ml-100k/movies.csv\")\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"# Let's ignore scores - they are not used in evaluation:\n",
"reco_users = reco[:, :1]\n",
"reco_items = reco[:, 1::2]\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"# Let's put them into one array\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"reco = np.concatenate((reco_users, reco_items), axis=1)\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
"# Let's rebuild it user-item dataframe\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"recommended = []\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"for row in reco:\n",
" for rec_nb, entry in enumerate(row[1:]):\n",
2nd meeting
2021-04-16 22:41:06 +02:00
" recommended.append((row[0], rec_nb + 1, entry))\n",
"recommended = pd.DataFrame(recommended, columns=[\"user\", \"rec_nb\", \"item\"])\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"recommended_content = pd.merge(recommended, items, left_on=\"item\", right_on=\"id\")\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"\n",
2nd meeting
2021-04-16 22:41:06 +02:00
"print(\"Here is what we recommend:\")\n",
"recommended_content[recommended_content[\"user\"] == user][\n",
" [\"user\", \"rec_nb\", \"title\", \"genres\"]\n",
"].sort_values(by=\"rec_nb\")"
first 2 lessons
2021-03-20 20:01:22 +01:00
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
last changes
2021-03-26 21:00:52 +01:00
"# project task 2: implement some other evaluation measure"
first 2 lessons
2021-03-20 20:01:22 +01:00
]
},
{
"cell_type": "code",
slides updated
2021-03-20 20:13:28 +01:00
"execution_count": 16,
first 2 lessons
2021-03-20 20:01:22 +01:00
"metadata": {},
"outputs": [],
"source": [
2nd meeting
2021-04-16 22:41:06 +02:00
"# it may be your idea, modification of what we have already implemented\n",
"# (for example Hit2 rate which would count as a success users whoreceived at least 2 relevant recommendations)\n",
first 2 lessons
2021-03-20 20:01:22 +01:00
"# or something well-known\n",
"# expected output: modification of evaluation_measures.py such that evaluate_all will also display your measure"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
Add another files Project 2
2021-06-09 22:16:22 +02:00
"version": "3.8.8"
first 2 lessons
2021-03-20 20:01:22 +01:00
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Reference in New Issue Copy Permalink