{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "verified-accommodation", "metadata": {}, "outputs": [], "source": [ "%matplotlib inline\n", "%load_ext autoreload\n", "%autoreload 2\n", "\n", "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "from IPython.display import Markdown, display, HTML\n", "from collections import defaultdict, deque\n", "\n", "import torch\n", "import torch.nn as nn\n", "import torch.optim as optim\n", "\n", "# Fix the dying kernel problem (only a problem in some installations - you can remove it, if it works without it)\n", "import os\n", "os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'\n", "os.environ['CUDA_LAUNCH_BLOCKING'] = '1'" ] }, { "cell_type": "markdown", "id": "educated-tourist", "metadata": {}, "source": [ "# Load data" ] }, { "cell_type": "code", "execution_count": 2, "id": "prepared-fraction", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
item_idtitlegenres
118145Bad Boys (1995)Action|Comedy|Crime|Drama|Thriller
143171Jeffrey (1995)Comedy|Drama
194228Destiny Turns on the Radio (1995)Comedy
199233Exotica (1994)Drama
230267Major Payne (1995)Comedy
313355Flintstones, The (1994)Children|Comedy|Fantasy
379435Coneheads (1993)Comedy|Sci-Fi
419481Kalifornia (1993)Drama|Thriller
615780Independence Day (a.k.a. ID4) (1996)Action|Adventure|Sci-Fi|Thriller
737959Of Human Bondage (1934)Drama
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Number of interactions left: 1170\n" ] } ], "source": [ "ml_ratings_df = pd.read_csv(os.path.join(\"data\", \"movielens_small\", \"ratings.csv\")).rename(columns={'userId': 'user_id', 'movieId': 'item_id'})\n", "ml_movies_df = pd.read_csv(os.path.join(\"data\", \"movielens_small\", \"movies.csv\")).rename(columns={'movieId': 'item_id'})\n", "ml_df = pd.merge(ml_ratings_df, ml_movies_df, on='item_id')\n", "\n", "# Filter the data to reduce the number of movies\n", "seed = 6789\n", "rng = np.random.RandomState(seed=seed)\n", "left_ids = rng.choice(ml_movies_df['item_id'], size=100, replace=False)\n", "\n", "ml_ratings_df = ml_ratings_df.loc[ml_ratings_df['item_id'].isin(left_ids)]\n", "ml_movies_df = ml_movies_df.loc[ml_movies_df['item_id'].isin(left_ids)]\n", "ml_df = ml_df.loc[ml_df['item_id'].isin(left_ids)]\n", "\n", "display(HTML(ml_movies_df.head(10).to_html()))\n", "\n", "print(\"Number of interactions left: {}\".format(len(ml_ratings_df)))" ] }, { "cell_type": "markdown", "id": "opponent-prediction", "metadata": {}, "source": [ "# Generalized Matrix Factorization (GMF)" ] }, { "cell_type": "code", "execution_count": 32, "id": "fancy-return", "metadata": {}, "outputs": [], "source": [ "from livelossplot import PlotLosses\n", "\n", "from recommenders.recommender import Recommender\n", "\n", "\n", "class GMFModel(nn.Module):\n", " def __init__(self, n_items, n_users, embedding_dim, seed):\n", " super().__init__()\n", "\n", " self.seed = torch.manual_seed(seed)\n", " self.item_embedding = nn.Embedding(n_items, embedding_dim)\n", " self.user_embedding = nn.Embedding(n_users, embedding_dim)\n", " self.fc = nn.Linear(embedding_dim, 1, bias=False)\n", "\n", " def forward(self, x):\n", " user_ids = x[:, 0]\n", " item_ids = x[:, 1]\n", " user_embedding = self.user_embedding(user_ids)\n", " item_embedding = self.item_embedding(item_ids)\n", " x = self.fc(user_embedding * item_embedding)\n", " x = torch.sigmoid(x)\n", "\n", " return x\n", "\n", "\n", "class GMFRecommender(Recommender):\n", " \"\"\"\n", " General Matrix Factorization recommender as described in:\n", " - He X., Liao L., Zhang H., Nie L., Hu X., Chua T., Neural Collaborative Filtering, WWW Conference, 2017\n", " \"\"\"\n", "\n", " def __init__(self, seed=6789, n_neg_per_pos=5, print_type=None, **params):\n", " super().__init__()\n", " self.recommender_df = pd.DataFrame(columns=['user_id', 'item_id', 'score'])\n", " self.interactions_df = None\n", " self.item_id_mapping = None\n", " self.user_id_mapping = None\n", " self.item_id_reverse_mapping = None\n", " self.user_id_reverse_mapping = None\n", " self.r = None\n", " self.most_popular_items = None\n", " \n", " self.nn_model = None\n", " self.optimizer = None\n", " \n", " self.n_neg_per_pos = n_neg_per_pos\n", " if 'n_epochs' in params: # number of epochs (each epoch goes through the entire training set)\n", " self.n_epochs = params['n_epochs']\n", " else:\n", " self.n_epochs = 10\n", " if 'lr' in params: # learning rate\n", " self.lr = params['lr']\n", " else:\n", " self.lr = 0.01\n", " if 'weight_decay' in params: # weight decay (L2 regularization)\n", " self.weight_decay = params['weight_decay']\n", " else:\n", " self.weight_decay = 0.001\n", " if 'embedding_dim' in params:\n", " self.embedding_dim = params['embedding_dim']\n", " else:\n", " self.embedding_dim = 4\n", " if 'batch_size' in params:\n", " self.batch_size = params['batch_size']\n", " else:\n", " self.batch_size = 64\n", " if 'device' in params:\n", " self.device = params['device']\n", " else:\n", " self.device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n", " \n", " if 'should_recommend_already_bought' in params:\n", " self.should_recommend_already_bought = params['should_recommend_already_bought']\n", " else:\n", " self.should_recommend_already_bought = False\n", " \n", " if 'train' in params:\n", " self.train = params['train']\n", " else:\n", " self.train = False\n", " self.validation_set_size = 0.2\n", " \n", " self.seed = seed\n", " self.rng = np.random.RandomState(seed=seed)\n", " torch.manual_seed(seed)\n", " \n", " if 'should_save_model' in params:\n", " self.should_save_model = params['should_save_model']\n", " self.print_type = print_type\n", "\n", " def fit(self, interactions_df, users_df, items_df):\n", " \"\"\"\n", " Training of the recommender.\n", "\n", " :param pd.DataFrame interactions_df: DataFrame with recorded interactions between users and items\n", " defined by user_id, item_id and features of the interaction.\n", " :param pd.DataFrame users_df: DataFrame with users and their features defined by\n", " user_id and the user feature columns.\n", " :param pd.DataFrame items_df: DataFrame with items and their features defined\n", " by item_id and the item feature columns.\n", " \"\"\"\n", "\n", " del users_df, items_df\n", "\n", " # Shift item ids and user ids so that they are consecutive\n", "\n", " unique_item_ids = interactions_df['item_id'].unique()\n", " self.item_id_mapping = dict(zip(unique_item_ids, list(range(len(unique_item_ids)))))\n", " self.item_id_reverse_mapping = dict(zip(list(range(len(unique_item_ids))), unique_item_ids))\n", " unique_user_ids = interactions_df['user_id'].unique()\n", " self.user_id_mapping = dict(zip(unique_user_ids, list(range(len(unique_user_ids)))))\n", " self.user_id_reverse_mapping = dict(zip(list(range(len(unique_user_ids))), unique_user_ids))\n", "\n", " interactions_df = interactions_df.copy()\n", " interactions_df.replace({'item_id': self.item_id_mapping, 'user_id': self.user_id_mapping}, inplace=True)\n", "\n", " # Get the number of items and users\n", "\n", " self.interactions_df = interactions_df.copy()\n", " n_users = np.max(interactions_df['user_id']) + 1\n", " n_items = np.max(interactions_df['item_id']) + 1\n", "\n", " # Get the user-item interaction matrix (mapping to int is necessary because of how iterrows works)\n", " r = np.zeros(shape=(n_users, n_items))\n", " for idx, interaction in interactions_df.iterrows():\n", " r[int(interaction['user_id'])][int(interaction['item_id'])] = 1\n", "\n", " self.r = r\n", " \n", " # Indicate positive interactions\n", " \n", " interactions_df.loc[:, 'interacted'] = 1\n", "\n", " # Generate negative interactions\n", " negative_interactions = []\n", "\n", " i = 0\n", " while i < self.n_neg_per_pos * len(interactions_df):\n", " sample_size = 1000\n", " user_ids = self.rng.choice(np.arange(n_users), size=sample_size)\n", " item_ids = self.rng.choice(np.arange(n_items), size=sample_size)\n", "\n", " j = 0\n", " while j < sample_size and i < self.n_neg_per_pos * len(interactions_df):\n", " if r[user_ids[j]][item_ids[j]] == 0:\n", " negative_interactions.append([user_ids[j], item_ids[j], 0])\n", " i += 1\n", " j += 1\n", " \n", " interactions_df = pd.concat(\n", " [interactions_df, pd.DataFrame(negative_interactions, columns=['user_id', 'item_id', 'interacted'])])\n", " interactions_df = interactions_df.reset_index(drop=True)\n", " \n", " # Initialize losses and loss visualization\n", " \n", " if self.print_type is not None and self.print_type == 'live':\n", " liveloss = PlotLosses()\n", "\n", " training_losses = deque(maxlen=50)\n", " training_avg_losses = []\n", " training_epoch_losses = []\n", " validation_losses = deque(maxlen=50)\n", " validation_avg_losses = []\n", " validation_epoch_losses = []\n", " last_training_total_loss = 0.0\n", " last_validation_total_loss = 0.0\n", " \n", " # Initialize the network\n", " \n", " self.nn_model = GMFModel(n_items, n_users, self.embedding_dim, self.seed)\n", " self.nn_model.train()\n", " self.nn_model.to(self.device)\n", " self.optimizer = optim.Adam(self.nn_model.parameters(), lr=self.lr, weight_decay=self.weight_decay)\n", " \n", " # Split the data\n", " \n", " if self.train:\n", " interaction_ids = self.rng.permutation(len(interactions_df))\n", " train_validation_slice_idx = int(len(interactions_df) * (1 - self.validation_set_size))\n", " training_ids = interaction_ids[:train_validation_slice_idx]\n", " validation_ids = interaction_ids[train_validation_slice_idx:]\n", " else:\n", " interaction_ids = self.rng.permutation(len(interactions_df))\n", " training_ids = interaction_ids\n", " validation_ids = []\n", " \n", " # Train the model\n", " \n", " for epoch in range(self.n_epochs):\n", " if self.print_type is not None and self.print_type == 'live':\n", " logs = {}\n", " \n", " # Train\n", " \n", " training_losses.clear()\n", " training_total_loss = 0.0\n", " \n", " self.rng.shuffle(training_ids)\n", " \n", " batch_idx = 0\n", " n_batches = int(np.ceil(len(training_ids) / self.batch_size))\n", " \n", " for batch_idx in range(n_batches):\n", " \n", " batch_ids = training_ids[(batch_idx * self.batch_size):((batch_idx + 1) * self.batch_size)]\n", " \n", " batch = interactions_df.loc[batch_ids]\n", " batch_input = torch.from_numpy(batch.loc[:, ['user_id', 'item_id']].values).long().to(self.device)\n", " y_target = torch.from_numpy(batch.loc[:, ['interacted']].values).float().to(self.device)\n", " \n", " # Create responses\n", "\n", " y = self.nn_model(batch_input).clip(0.000001, 0.999999)\n", "\n", " # Define loss and backpropagate\n", "\n", " self.optimizer.zero_grad()\n", " loss = -(y_target * y.log() + (1 - y_target) * (1 - y).log()).sum()\n", " \n", " loss.backward()\n", " self.optimizer.step()\n", " \n", " training_total_loss += loss.item()\n", " \n", " if self.print_type is not None and self.print_type == 'text':\n", " print(\"\\rEpoch: {}\\tBatch: {}\\tLast epoch - avg training loss: {:.2f} avg validation loss: {:.2f} loss: {}\".format(\n", " epoch, batch_idx, last_training_total_loss, last_validation_total_loss, loss), end=\"\")\n", " \n", " training_losses.append(loss.item())\n", " training_avg_losses.append(np.mean(training_losses))\n", " \n", " # Validate\n", "\n", " validation_total_loss = 0.0\n", " \n", " batch = interactions_df.loc[validation_ids]\n", " batch_input = torch.from_numpy(batch.loc[:, ['user_id', 'item_id']].values).long().to(self.device)\n", " y_target = torch.from_numpy(batch.loc[:, ['interacted']].values).float().to(self.device)\n", " \n", " # Create responses\n", "\n", " y = self.nn_model(batch_input).clip(0.000001, 0.999999)\n", "\n", " # Calculate validation loss\n", "\n", " loss = -(y_target * y.log() + (1 - y_target) * (1 - y).log()).sum()\n", " validation_total_loss += loss.item()\n", " \n", " # Save and print epoch losses\n", " \n", " training_last_avg_loss = training_total_loss / len(training_ids)\n", " validation_last_avg_loss = validation_total_loss / len(validation_ids)\n", "\n", " if self.print_type is not None and self.print_type == 'live' and epoch >= 0:\n", " # A bound on epoch prevents showing extremely high losses in the first epochs\n", " logs['loss'] = training_last_avg_loss\n", " logs['val_loss'] = validation_last_avg_loss\n", " liveloss.update(logs)\n", " liveloss.send()\n", "\n", " # Find the most popular items for the cold start problem\n", "\n", " offers_count = interactions_df.loc[:, ['item_id', 'user_id']].groupby(by='item_id').count()\n", " offers_count = offers_count.sort_values('user_id', ascending=False)\n", " self.most_popular_items = offers_count.index\n", "\n", " def recommend(self, users_df, items_df, n_recommendations=1):\n", " \"\"\"\n", " Serving of recommendations. Scores items in items_df for each user in users_df and returns\n", " top n_recommendations for each user.\n", "\n", " :param pd.DataFrame users_df: DataFrame with users and their features for which\n", " recommendations should be generated.\n", " :param pd.DataFrame items_df: DataFrame with items and their features which should be scored.\n", " :param int n_recommendations: Number of recommendations to be returned for each user.\n", " :return: DataFrame with user_id, item_id and score as columns returning n_recommendations top recommendations\n", " for each user.\n", " :rtype: pd.DataFrame\n", " \"\"\"\n", "\n", " # Clean previous recommendations (iloc could be used alternatively)\n", " self.recommender_df = self.recommender_df[:0]\n", "\n", " # Handle users not in the training data\n", "\n", " # Map item ids\n", "\n", " items_df = items_df.copy()\n", " items_df = items_df.loc[items_df['item_id'].isin(self.item_id_mapping)]\n", " items_df.replace({'item_id': self.item_id_mapping}, inplace=True)\n", "\n", " # Generate recommendations\n", "\n", " for idx, user in users_df.iterrows():\n", " recommendations = []\n", "\n", " user_id = user['user_id']\n", "\n", " if user_id in self.user_id_mapping:\n", " \n", " mapped_user_id = self.user_id_mapping[user_id]\n", " \n", " ids_list = items_df['item_id'].tolist()\n", " id_to_pos = np.array([0]*len(ids_list))\n", " for k in range(len(ids_list)):\n", " id_to_pos[ids_list[k]] = k\n", " \n", " net_input = torch.tensor(list(zip([mapped_user_id]*len(ids_list), ids_list))).to(self.device)\n", " \n", " scores = self.nn_model(net_input).flatten().detach().cpu().numpy()\n", " \n", " # Choose n recommendations based on highest scores\n", " if not self.should_recommend_already_bought:\n", " x_list = self.interactions_df.loc[\n", " self.interactions_df['user_id'] == mapped_user_id]['item_id'].tolist()\n", " scores[id_to_pos[x_list]] = -np.inf\n", "\n", " chosen_pos = np.argsort(-scores)[:n_recommendations]\n", "\n", " for item_pos in chosen_pos:\n", " recommendations.append(\n", " {\n", " 'user_id': self.user_id_reverse_mapping[mapped_user_id],\n", " 'item_id': self.item_id_reverse_mapping[ids_list[item_pos]],\n", " 'score': scores[item_pos]\n", " }\n", " )\n", " else: # For new users recommend most popular items\n", " for i in range(n_recommendations):\n", " recommendations.append(\n", " {\n", " 'user_id': user['user_id'],\n", " 'item_id': self.item_id_reverse_mapping[self.most_popular_items[i]],\n", " 'score': 1.0\n", " }\n", " )\n", "\n", " user_recommendations = pd.DataFrame(recommendations)\n", "\n", " self.recommender_df = pd.concat([self.recommender_df, user_recommendations])\n", "\n", " return self.recommender_df\n", " \n", " def get_user_repr(self, user_id):\n", " mapped_user_id = self.user_id_mapping[user_id]\n", " return self.nn_model.user_embedding(torch.tensor(mapped_user_id).to(self.device)).detach().cpu().numpy()\n", " \n", " def get_item_repr(self, item_id):\n", " mapped_item_id = self.item_id_mapping[item_id]\n", " return self.nn_model.item_embedding(torch.tensor(mapped_item_id).to(self.device)).detach().cpu().numpy()\n", "\n", " \n", "class MLPModel(nn.Module):\n", " def __init__(self, n_items, n_users, embedding_dim, seed):\n", " super().__init__()\n", "\n", " self.seed = torch.manual_seed(seed)\n", " self.item_embedding = nn.Embedding(n_items, embedding_dim)\n", " self.user_embedding = nn.Embedding(n_users, embedding_dim)\n", " self.fc1 = nn.Linear(2 * embedding_dim, 32, bias=False)\n", " self.fc2 = nn.Linear(32, 16, bias=False)\n", " self.fc3 = nn.Linear(16, 1, bias=False)\n", "\n", " def forward(self, x):\n", " user = x[:, 0]\n", " item = x[:, 1]\n", " user_embedding = self.user_embedding(user)\n", " item_embedding = self.item_embedding(item)\n", " x = torch.cat([user_embedding, item_embedding], dim=1)\n", " x = torch.relu(self.fc1(x))\n", " x = torch.relu(self.fc2(x))\n", " x = torch.sigmoid(self.fc3(x))\n", "\n", " return x\n", "\n", " \n", "class NeuMFModel(nn.Module):\n", " def __init__(self, n_items, n_users, gmf_embedding_dim, mlp_embedding_dim, seed):\n", " super().__init__()\n", "\n", " self.seed = torch.manual_seed(seed)\n", "\n", " # GMF\n", "\n", " self.gmf_user_embedding = nn.Embedding(n_users, gmf_embedding_dim)\n", " self.gmf_item_embedding = nn.Embedding(n_items, gmf_embedding_dim)\n", "\n", " # MLP\n", "\n", " self.mlp_user_embedding = nn.Embedding(n_users, mlp_embedding_dim)\n", " self.mlp_item_embedding = nn.Embedding(n_items, mlp_embedding_dim)\n", " self.mlp_fc1 = nn.Linear(2 * mlp_embedding_dim, 32, bias=False)\n", " self.mlp_fc2 = nn.Linear(32, 16, bias=False)\n", "\n", " # Merge\n", "\n", " self.fc = nn.Linear(32, 1, bias=False)\n", "\n", " def forward(self, x):\n", " user = x[:, 0]\n", " item = x[:, 1]\n", "\n", " # GMF\n", "\n", " gmf_user_embedding = self.gmf_user_embedding(user)\n", " gmf_item_embedding = self.gmf_item_embedding(item)\n", " gmf_x = gmf_user_embedding * gmf_item_embedding\n", "\n", " # MLP\n", "\n", " mlp_user_embedding = self.mlp_user_embedding(user)\n", " mlp_item_embedding = self.mlp_item_embedding(item)\n", " mlp_x = torch.cat([mlp_user_embedding, mlp_item_embedding], dim=1)\n", " mlp_x = torch.relu(self.mlp_fc1(mlp_x))\n", " mlp_x = torch.relu(self.mlp_fc2(mlp_x))\n", "\n", " # Final score\n", "\n", " x = torch.cat([gmf_x, mlp_x], dim=1)\n", " x = torch.sigmoid(self.fc(x))\n", "\n", " return x" ] }, { "cell_type": "markdown", "id": "expensive-offering", "metadata": {}, "source": [ "## Quick test of the recommender (training)" ] }, { "cell_type": "code", "execution_count": 42, "id": "nonprofit-roads", "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbkAAAI4CAYAAAD3UJfIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABN6klEQVR4nO3deXzU1b3/8deZmez7BglZCCCEJewJBhHRqhXc96Vqq63aWq31drXtvfXW297bvdb+tNZa7eZStO7iUnetC4Qt7IusCQHCkkDInjm/P2YICSQQyCTfWd7PxyOPyZz5zsxnSODN+Z7zPcdYaxEREQlHLqcLEBER6S8KORERCVsKORERCVsKORERCVsKORERCVsKORERCVsKORERCVsKOZEBZIzZZIw5y+k6RCKFQk5ERMKWQk7EYcaYGGPMvcaYbf6ve40xMf7HMo0xLxljao0xe4wx7xtjXP7HvmuMqTLG7DfGrDHGnOnsJxEJPh6nCxARfgCUAZMACzwP/CfwX8A3gUogy39sGWCNMUXA7UCptXabMaYQcA9s2SLBTz05EeddC9xjrd1pra0BfgRc73+sFcgBhlprW62171vfgrPtQAww1hgTZa3dZK391JHqRYKYQk7EeUOAzZ3ub/a3AfwCWA+8bozZYIy5C8Baux64E/hvYKcx5kljzBBEpAuFnIjztgFDO90v8Ldhrd1vrf2mtXY4cCHwjYNjb9bax621p/qfa4GfDWzZIsFPIScy8KKMMbEHv4AngP80xmQZYzKBHwJ/BzDGnG+MOckYY4A6fKcpvcaYImPMZ/wTVJqARsDrzMcRCV4KOZGBNw9fKB38igXKgQpgGbAI+LH/2JHAG0A98BHwgLX2bXzjcT8FdgHbgUHA9wbuI4iEBqNNU0VEJFypJyciImFLISciImFLISciImFLISciImHLsWW9MjMzbWFhoVNvLyIiYWLhwoW7rLVZ3T3mWMgVFhZSXl7u1NuLiEiYMMZs7ukxna4UEZGwpZATEZGwpZATEZGwpZATEZGwpZATEZGwpZATEZGwpZATEZGwpZATEZGwpZATEZGwpZATEZGwpZATEZGwpZATEZGwpZATEZGwpZATEZGwpZATEZGwpZATEZGwpZATEZGwpZATEZGwpZATEZGwpZATEZGwFdIh19Lm5cWl22j3WqdLERGRIBTSITdvWTVfe2Ixn/3Nu7y4dBtehZ2IiHQS0iF34cQh3P+5KbiM4WtPLGbOb9/n1eXVWKuwExGRXoacMWa2MWaNMWa9Meaubh7/jTFmif9rrTGmNuCVdsPlMpw3IYdX7zyN3149idZ2L1/5+yLO/90HvLFyh8JORCTCmWMFgTHGDawFzgYqgQXANdbalT0c/zVgsrX2i0d73ZKSElteXn5CRfekrd3Lc0u2cd+b69iyp4GJeSn8x9mjmDUqC2NMQN9LRESCgzFmobW2pLvHetOTmwast9ZusNa2AE8CFx3l+GuAJ46/zL7zuF1cPjWPN785i59eOp5d9S3c8OgCLn/wI/69fpd6diIiEaY3IZcLbO10v9LfdgRjzFBgGPBW30s7cVFuF1dPK+Dtb53O/1xcTNXeRq59+BOufuhj5m/c42RpIiIygAI98eRq4GlrbXt3DxpjbjHGlBtjymtqagL81keK9ri4vmwo73z7dO6+YCwbdh3gyj98xHUPf8LCzXv7/f1FRMRZvQm5KiC/0/08f1t3ruYopyqttQ9Za0ustSVZWVm9r7KPYqPc3DhjGO99+wx+cO4YVlXv47Lff8gNj86norJ2wOoQEZGB1ZuJJx58E0/OxBduC4DPWWtXHHbcaOBVYJjtxeBXQCaebFsC7/wUErMgcTAkDDry+5hkOGzSyYHmNv7y0SYeem8DtQ2tnD12MP9x1ijGDknuWz0iIjLgjjbxxHOsJ1tr24wxtwOvAW7gEWvtCmPMPUC5tfYF/6FXA0/2JuACpqUe6iph2yI4UAPWe+QxntjDwi+LhMRBfDVxMDdemM7LG9p5tGI5V963kdOKh3F60WCS4zwkxUaRHBvV8X1SrIcod0hfVigiEnGO2ZPrLwG/hMDbDg174MBOqN8B9TXdfO//atjVbSA22SgaicGLwYsLL4Z2XL7vrQHjwho3xuUC/61xHbz14Ha5MG4Pbrcbt9uNy+3BuA7eenC5D35F4fJ4cLk84HKDy+P/ch+6NYe1u6MgNgXi0iAu3X+bBvHpEBUXuD9HEZEQ06eeXMhwuf29tSwYPO7ox3rboWG3P/R2+HqB9Tvx7NtOVGMDLW1teFvbaG1ro7Wtnfa2Vlrb22lva6OtvZ329jba29t9X23t2PY22r1eXLYdQxtuWnAZi5t23P649ODtuO/Gi4d23MbX7jHtuLG+Nv/xvmPbcXHs/4RYdyzEpWHiDwZgatcQPPh953BMygGXeqYiEt7CJ+SOh8sNiYN8XxR3NHuAxBN8SWstzW1e9jW2sq+plX1NbRxobqOlzUtzm5fmtnaaW3v4vs3rv99OS7u3y2MtrW20trbR1tqMbdyLu7mOVFNPKvWHbtvqSW2uJ33fAbLce0g1W0mhniS7jyjb2m293rThuMq+ApM+BzFJJ/ipRUSCW/icrowQbe1eahtb2Xughb0Nrew50EJtg+/7vQ0t/nb//fpmGhvqoWlvRyimcIAsU8slnn8zxayjxZNI28TriD/1q5A21OmPJyJy3I52ulIhFwHavZZ9ja3safAFYs3+Fj7ZuJuq5e9zXsPznOv6BJexbMw8g+hTbyN/whm+cUcRkRCgkJNuWWtZs2M/Hy6qIHnZnzmrYR6p5gCrXSNYPfQ6sqdfQ8mIwXg0q1REgphCTnpl+649bH77EfLX/JkhbVvZaVOZa2azfeQ1TB9fxKyiLBJjInMYV0SCl0JOjo/XS9OaN9j/zn1k7XifZqJ4tm0Gf7fnkj5iMmePHcxZYwaRk6JLF0TEeQo5OXE1a/B+/Hvskidwtzex0DWBB5rO5i3vZMblpnL2mGwumJjD8KwTnZcqItI3Cjnpu4Y9sOgv2Pl/xOyrojYun6fd53Hv7lLaPAk8f9upFGXrUgQRGXh93U9OxHdR+an/gfn6Urj8EVIzsrmp/kEqku7ka1EvcOtjCznQ3OZ0lSIiXSjk5Pi4o6D4MrjpDfjSG7gKyrjN+zjJu5fy/WeXaWNaEQkqCjk5cfmlcMWjEJfGvdmv8fySbTw+f4vTVYmIdFDISd/EJMEpX6Nwz7+5YehufvTCSpZX1TldlYgIoJCTQJh2C8Sl8f2EF0hPiOarjy1iX1P3a2aKiAwkhZz0XUwSTL+d6A3/4tHPuthW28h3nqrQ+JyIOE4hJ4Ex7RaITWXM2gf57uzRvLpiO4/+e5PTVYlIhFPISWDEJsP022Htq9w0oo6zxw7mf+etYtGWvU5XJiIRTCEngXOyrzdn3v0Zv7x8Ijmpsdz+2CL2HmhxujIRiVAKOQmc2BSYfhusfYWU2hXc/7kp7Kpv4Rtzl+D1anxORAaeQk4C6+Qv+8Lu3Z8xIS+V/zp/DG+vqeHB9z51ujIRiUAKOQms2BQouw3WzIPqpVxXNpTzJ+Twy9fW8PGG3U5XJyIRRiEngXewN/fOzzDG8NPLJlCYkcAdTyymZn+z09WJSARRyEngxaVC2VdhzctQXUFijIf7r51CXWMrd/5jMe0anxORAaKQk/5x8lcgxjc2BzAmJ5n/uaiYf6/fzX1vrnO4OBGJFAo56R9xqVB2K6x+CbYvA+CKkjwum5LHfW+t4/11Nc7WJyIRQSEn/afsKxCT3NGbM8bwPxePY+SgRO58cgnb65ocLlBEwp1CTvpPXJqvN7fqRdi+HID4aA8PXDuFxtZ2vvbEItravQ4XKSLhTCEn/avs1i69OYCTBiXxf5eOZ8Gmvfzy9bUOFici4U4hJ/0rLs03CWXVC7BjRUfzRZNy+dzJBTz47qe8uWqHgwWKSDhTyEn/K7sVopO69OYAfnj+WMYNSeYbc5eydU+DQ8WJSDhTyEn/i0/3XSC+8nnYsbKjOTbKzQPXTsHrtdz++CJa2jQ+JyKBpZCTgTH9tm57c0MzEvjFFRNYWlnH/85b5VBxIhKuFHIyMOLTfVvxHNabA5hdnMMXZwzjzx9uYt6yaocKFJFwpJCTgTP9dohOgPd+fsRDd80ZzaT8VL7zdAVVtY0OFCci4UghJwMnPh2m3QIrnoOdq7s8FO1x8ZurJlHf3Ma8CvXmRCQwFHIysI7SmxuWmcCowYm8u1ZLfolIYCjkZGAlZMC0m2H5M0f05gBmjcpi/sY9NLS0OVCciIQbhZwMvOlfg6h4eO8XRzw0a9QgWtq92mBVRAJCIScDr6M390+oWdPloZLCNOKi3Ly7RqcsRaTvFHLijFO+BlFxR/TmYqPcTB+RoXE5EQkIhZw4IyHT15tb9jTUdF2kedaoLDbtbmDTrgMOFSci4UIhJ8455Y5ue3OzRmUB8J42VhWRPlLIiXMSMqH0Jlj+NOxa19FcmJnA0Ix4jcuJSJ8p5MRZp9wBnthue3Mffrqb5rZ2hwoTkXCgkBNnJWZByRdh2VOwa31H86xRWTS2tlO+aa+DxYlIqFPIifNmfB3cMV16c2XDM4h2uzTLUkT6RCEnzkscBKVfgmVzYfenACTEeCgdlqZxORHpE4WcBIdT7vD35n7Z0TRrVBZrduynuk67EojIiVHISXBIGuwbm6v4B+zZCPiW+AJ4T6csReQEKeQkeJx8C9h2WPcvAEYNTiQ7OVbjciJywhRyEjzSCiE5F7Z8BIAxhlmjsnh/3S7a2r3O1iYiIUkhJ8GloMwXctYCMKsoi/1NbSzZWutsXSISkhRyElwKpsP+aqjdAsCMkzJxu4xOWYrICVHISXApKPPdbv0EgJS4KCbnpyrkROSEKOQkuAwaCzHJHeNy4LuUoKKyjl31zQ4WJiKhSCEnwcXlhrxS2PJxR9OsIt+uBB+s2+VUVSISohRyEnwKpsPOldDoW7eyeEgK6QnROmUpIsdNISfBp2NcbgEALpfhtJGZvLe2Bq/XOliYiIQahZwEn9yp4PJ0HZcrymL3gRZWbNvnYGEiEmoUchJ8ouMhZ2KXcbmZI33jcu+u3elUVSISghRyEpwKpkPVQmjzzajMTIxhfG6KxuVE5Lgo5CQ4FZRBezNUL+1omjUqi0VbaqlrbHWwMBEJJQo5CU75/sknh43LtXstH67XpQQi0jsKOQlOiVmQPqLLuNzk/FSSYj06ZSkivaaQk+BVMN0Xcl7fDgQet4tTT8rk3bU1WKtLCUTk2BRyErwKyqBxD+xe19E0a1QW1XVNrNtZ72BhIhIqFHISvAqm+247nbI8bZT/UoI1OmUpIsemkJPglTEC4jO7hNyQ1DhGDU7UuJyI9IpCToKXMYc2Ue1k1qgs5m/cQ0NLm0OFiUioUMhJcCsog70bYf+OjqZZowbR0u7l4w27HSxMREKBQk6C28Fxua2HTlmWFKYRF+XWuJyIHFOvQs4YM9sYs8YYs94Yc1cPx1xpjFlpjFlhjHk8sGVKxMqeAJ7YLuNysVFupo/I0LiciBzTMUPOGOMG7gfmAGOBa4wxYw87ZiTwPWCGtXYccGfgS5WI5ImG3JJux+U27W5g064DDhUmIqGgNz25acB6a+0Ga20L8CRw0WHH3Azcb63dC2Ct1VLxEjgFZVBdAc2Hro2b5b+U4L116s2JSM96E3K5wNZO9yv9bZ2NAkYZY/5tjPnYGDM7UAWKUDAdbLtvVwK/wswEhmbEa1xORI4qUBNPPMBI4HTgGuCPxpjUww8yxtxijCk3xpTX1OgfJ+ml/FLAdBmXA19v7sNPd9Pc1u5MXSIS9HoTclVAfqf7ef62ziqBF6y1rdbajcBafKHXhbX2IWttibW2JCsr60RrlkgTmwKDx3U7LtfY2k75pr0OFSYiwa43IbcAGGmMGWaMiQauBl447Jjn8PXiMMZk4jt9uSFwZUrEKyiDygXQfugC8LLhGUS7XZplKSI9OmbIWWvbgNuB14BVwFxr7QpjzD3GmAv9h70G7DbGrATeBr5trdWVuhI4BdOhpR52LO9oSojxUDosTeNyItIjT28OstbOA+Yd1vbDTt9b4Bv+L5HAyz/Zd7v1ExgyqaN51qgs/nfeaqrrGslJiXOmNhEJWlrxREJDaj4k53UzLjcIgPd0ylJEuqGQk9BRUOabYdlpw9RRgxPJTo7VuJyIdEshJ6GjoAz2V0Pt5o4mYwyzRmXx/rpdtLV7HSxORIKRQk5CR0GZ73bLJ12aZxVlsb+pjSVbawe+JhEJago5CR2DxkJM8hHjcjNOysTtMjplKSJHUMhJ6HC5IX/aESufpMRFMTk/VSEnIkdQyEloKSiDmlXQsKdL86xRWVRU1rGrvtmhwkQkGCnkJLQc3ES1ckGX5llFvmXiPli3a6ArEpEgppCT0DJkCrg8R4zLFQ9JIT0hWqcsRaQLhZyEluh4yJl0xLicy2U4bWQm762tweu13T9XRCKOQk5CT0EZVC2Ctq7jb7OKsth9oIUV2/Y5VJiIBBuFnISegunQ3gzblnRpnjnSNy737lptTC8iPgo5CT0HF2s+bFwuMzGG8bkpGpcTkQ4KOQk9iVmQcdIR43Lgu5Rg0ZZa6hpbHShMRIKNQk5CU0EZbP0YvF3Xq5xVlEW71/Lhel1KICIKOQlVBdOhcS/sWtuleXJ+KkmxHp2yFBFAISehKt+/WPPWrqcsPW4Xp56Uybtra7BWlxKIRDqFnISmjBEQn9njuFx1XRPrdtY7UJiIBBOFnIQmY/ybqH50xEOnjfJfSrBGpyxFIp1CTkJXwXTYuwn2b+/SPCQ1jlGDEzUuJyIKOQlhHZuodn/Kcv7GPTS0tA1wUSISTBRyErqyJ4AnroeQG0RLu5ePN+x2oDARCRYKOQldnmjIK+l2XK50WBpxUW6Ny4lEOIWchLaCMti+DJq7zqSM8biZmJ/Ckq21ztQlIkFBISehLb8MbDtUlR/x0IS8VFZV76elzdvNE0UkEijkJLTllwKm23G58bkptLR7Wbtj/8DXJSJBQSEnoS02BQYXdzsuNz43BYBlVXUDXZWIBAmFnIS+gjLYugDau14uMDQjnqRYj0JOJIIp5CT0FZRB6wHYsbxLszGG8bkpLKtUyIlEKoWchL6jXBQ+Pi+F1dv30dzWPsBFiUgwUMhJ6EvJg5T8bsflJuSm0tpuWbtdizWLRCKFnISHgjJfT+6w7XUOTj6pqKp1oCgRcZpCTsJDQRnUb/ct2NxJfnocKXFRLNfkE5GIpJCT8NCxieonXZoPTj6p0OQTkYikkJPwMGgMxKR0f71cXgprd+ynqVWTT0QijUJOwoPLDfnTelz5pLXdsma7Vj4RiTQKOQkfBWVQsxoa9nRpPjT5RKcsRSKNQk7Cx8Hr5bbO79KclxZHWnwUyzUuJxJxFHISPoZMAVfUEeNyxhiKc1PUkxOJQAo5CR/R8TBkUrfjchPyUlinySciEUchJ+GloAy2LYLWpi7N43NTaPNaVlXvc6gwEXGCQk7CS34ZtLdA9ZIuzePzUgF0UbhIhFHISXjpWKy567jckJRYMhKidVG4SIRRyEl4SciEjJFHjMsdnHyiveVEIotCTsLPwcWavd4uzRPyUli3s57GFk0+EYkUCjkJPwVl0FQLu9Z2aS7OTaHda1mpySciEUMhJ+GnYLrv9rBxuQl5vpVPNPlEJHIo5CT8pA+HhKwjxuWyk2PJTNTkE5FIopCT8GOMf1zuyJVPxuemqCcnEkEUchKe8sugdjPsq+7SPD4vlXU799PQ0uZQYSIykBRyEp4Ojstt7XrKcnxuCl4LK7dp8olIJFDISXjKmQCeONjc/eQTXS8nEhkUchKe3FGQOxUqF3RpHpwcS1ZSDMs0+UQkIijkJHzlTYXty45YrHmCVj4RiRgKOQlfeaXgbYXtFV2ax+elsL6mngPNmnwiEu4UchK+ckt8t5XlXZrH56ZgLVr5RCQCKOQkfCXnQHLeEeNy43N9k090UbhI+FPISXjLKzmiJzcoOZbByTEsq6x1piYRGTAKOQlveSVQtwX27+jSPD43VZNPRCKAQk7CW16p77bqyHG5DbsOUK/JJyJhTSEn4S1nIrg8R5yynJDnm3yyQr05kbCmkJPwFhUHg4uPmHxSnKuVT0QigUJOwl9eKWxbDN5DO4JnJcWQkxKrGZYiYU4hJ+EvrwRa6qFmdZdmbbsjEv4UchL+Dk4+6eZ6uQ27DrCvqdWBokRkICjkJPylD4e4tCNXPvHvSLCiSiufiIQrhZyEP2N8S3x1s7wXwLKqWgeKEpGBoJCTyJBX6huTazrUa8tIjCE3NY5l6smJhC2FnESGvKmAhW2LujSPz03R8l4iYUwhJ5Ehd6rv9vDJJ3kpbNrdQF2jJp+IhCOFnESGuDTIGAmVC7s0HxyX08onIuGpVyFnjJltjFljjFlvjLmrm8dvMMbUGGOW+L9uCnypIn2UV+rryVnb0dSx7Y5CTiQsHTPkjDFu4H5gDjAWuMYYM7abQ/9hrZ3k/3o4wHWK9F1eCTTsgtrNHU1pCdHkpcVpeS+RMNWbntw0YL21doO1tgV4Eriof8sS6Qd53e8UPiEvhWVa3kskLPUm5HKBrZ3uV/rbDneZMabCGPO0MSY/INWJBNKgceCJ63ax5i17Gqhr0OQTkXATqIknLwKF1toJwL+Av3R3kDHmFmNMuTGmvKamJkBvLdJLbg8MmXxkTy43FdCOBCLhqDchVwV07pnl+ds6WGt3W2ub/XcfBqZ290LW2oestSXW2pKsrKwTqVekb/JKYHsFtDV3NBXnJgNQoZVPRMJOb0JuATDSGDPMGBMNXA280PkAY0xOp7sXAqsCV6JIAOWVQnsLbF/W0ZQaH01Berx2JBAJQ8cMOWttG3A78Bq+8JprrV1hjLnHGHOh/7A7jDErjDFLgTuAG/qrYJE+6Zh8cuRF4dpbTiT8eHpzkLV2HjDvsLYfdvr+e8D3AluaSD9IHgLJuf6Qu7WjeXxuCi9XVLP3QAtpCdHO1SciAaUVTyTy5E7tZvLJwR0J1JsTCScKOYk8eaW+C8LrD83wHaeQEwlLCjmJPAd3Cq861JtLiYuiMCNeF4WLhBmFnESenIlg3N1eFK6enEh4UchJ5ImOh+ziI0JuQl4KVbWN7K5v7uGJIhJqFHISmXJLoGoxeNs7msZr5RORsKOQk8iUVwot+6FmTUfTOP/KJ7ooXCR8KOQkMnUz+SQ5NorhmQm6KFwkjCjkJDJljIDYVE0+EQlzCjmJTMb4lviqXNileUJeCtV1TdTs1+QTkXCgkJPIlVsCO1dC8/6OpvH+i8I1LicSHhRyErnySgELVYs6msblpmCMZliKhAuFnESu3Cm+206TTxJjPJp8IhJGFHISueLTIeOkIxZrHp+botOVImFCISeRLa/UF3LWdjSNz0tl+74mdu5rcrAwEQkEhZxEttypcGAn1G7paBqvHQlEwoZCTiLbwYvCO10vN25IsiafiIQJhZxEtsHjwBMLVYeul0uI8XBSVqK23REJAwo5iWzuKBgy+YiVT8Zr5RORsKCQE8krgeoKaDu0ysn4vBR27m9mhyafiIQ0hZxIbgm0N8P25R1NHZNPdMpSJKQp5ES62ZFg7JBkXAYqdMpSJKQp5ERSciEpp8u4XHy0h5GDklhWWetcXSLSZwo5EfDvSNDdtjv7sJ0uFBeR0KKQEwHfKcu9m+DAro6mCXkp7KpvZrsmn4iELIWcCPgmn0CXdSyLNflEJOQp5EQAhkwC4+46+SQnGbfL6Ho5kRCmkBMBiE6AwWO7jMvFRbsZOShRIScSwhRyIgfllfo2UPV6O5rG56awrLJOk09EQpRCTuSgvFJo3ge71nY0TchLYfeBFrbVafKJSChSyIkc1DH55NApS00+EQltCjmRgzJOgtiULpNPxuQk43EZllXVOleXiJwwhZzIQS6XbxPVTpcRxEa5GTk4iWVV+xwsTEROlEJOpLO8Uti5EprrO5om5KawrLJWk09EQpBCTqSzvFKwXti2uKNpfF4Kextaqdzb6GBhInIiFHIineVO9d12mnxycNud5bpeTiTkKOREOotPh/QRULWwo2l0ThJRbqNtd0RCkEJO5HAHdyTwj8HFeNyMGpyknpxICFLIiRwurxTqd0BdZUfThLwUKrTyiUjIUciJHC7vyIvCJ+enUdfYyqc19T08SUSCkUJO5HCDi8ET2+V6uZLCNAAWbNrrVFUicgIUciKHc0dBzqQuK58My0wgMzGaBRv3OFeXiBw3hZxId/JKYNsSaGsBwBhDydB0FmxWyImEEoWcSHfySqC9GXYs72gqHZbO1j2NbNeOBCIhw+N0ASJBqWNHgnLInQJAace43B4umDjEqcokhLS2tlJZWUlTk/5jFAixsbHk5eURFRXV6+co5ES6k5IHidm+GZYn3wLA2Jxk4qPdCjnptcrKSpKSkigsLMQY43Q5Ic1ay+7du6msrGTYsGG9fp5OV4p0xxjfKctOk088bhdTCtI0w1J6rampiYyMDAVcABhjyMjIOO5esUJOpCd5JbBnAxzY3dFUWpjO6u37qGtsdbAwCSUKuMA5kT9LhZxIT/JKfbed1rEsLUzDWli0Rb05CX61tbU88MADx/28c889l9ra2qMe88Mf/pA33njjBCsbOAo5kZ7kTALj6rLyyaSCVDwuo+vlJCT0FHJtbW1Hfd68efNITU096jH33HMPZ511Vl/KGxAKOZGexCTCoHFdQi4+2sO43BTKNS4nIeCuu+7i008/ZdKkSZSWljJz5kwuvPBCxo4dC8DFF1/M1KlTGTduHA899FDH8woLC9m1axebNm1izJgx3HzzzYwbN47PfvazNDb69lW84YYbePrppzuOv/vuu5kyZQrjx49n9erVANTU1HD22Wczbtw4brrpJoYOHcquXbsG9M9AsytFjiavBJY/A14vuHz/J5xWmMZfPtpMc1s7MR63wwVKqPjRiytYuW1fQF9z7JBk7r5gXI+P//SnP2X58uUsWbKEd955h/POO4/ly5d3zE585JFHSE9Pp7GxkdLSUi677DIyMjK6vMa6det44okn+OMf/8iVV17JP//5T6677roj3iszM5NFixbxwAMP8Mtf/pKHH36YH/3oR3zmM5/he9/7Hq+++ip/+tOfAvr5e0M9OZGjySuB5jrYva6jqaQwnZY2L8sqtfWOhJZp06Z1mX5/3333MXHiRMrKyti6dSvr1q074jnDhg1j0qRJAEydOpVNmzZ1+9qXXnrpEcd88MEHXH311QDMnj2btLS0wH2YXlJPTuRoDk4+qSyHrCIASob6/qLO37SHksJ0pyqTEHO0HtdASUhI6Pj+nXfe4Y033uCjjz4iPj6e008/vdvp+TExMR3fu93ujtOVPR3ndruPOeY3kNSTEzmajJEQk9JlXC4jMYYRWQkal5Ogl5SUxP79+7t9rK6ujrS0NOLj41m9ejUff/xxwN9/xowZzJ07F4DXX3+dvXsH/u+MenIiR+Ny+Zb16nRROMC0Yem8XFGN12txuXQdlASnjIwMZsyYQXFxMXFxcQwePLjjsdmzZ/Pggw8yZswYioqKKCsrC/j733333VxzzTX87W9/Y/r06WRnZ5OUlBTw9zka49ROxyUlJba8vPzYB4o47a2fwPu/hO9VQrTvdM8/F1byzaeW8uqdMxmdnexwgRKsVq1axZgxY5wuwzHNzc243W48Hg8fffQRt956K0uWLOnTa3b3Z2qMWWitLenuePXkRI4lrwSsF7YthsJTAV9PDmDBxj0KOZEebNmyhSuvvBKv10t0dDR//OMfB7wGhZzIsXTekcAfcnlpcQxOjmHBpr1cP73QudpEgtjIkSNZvHixozVo4onIsSRkQNqwLpNPjDGUFqazYNMenDrlLyLHppAT6Y28Ul9PrlOglRamU13XRFVt91OqRcR5CjmR3iicAfXbYefKjqZS/zVyCzZpHUuRYKWQE+mNUbN9t2te6Wgqyk4iKcaj/eVEgphCTqQ3krJhyJQuIed2GaYWpmlHAgkbiYmJAGzbto3LL7+822NOP/10jnX517333ktDQ0PH/d5s3dNfFHIivVV0ru+i8P07OppKC9NZt7OevQdaHCxMJLCGDBnSscPAiTg85HqzdU9/UciJ9FbRHN/t2lc7mg6Oy5Vv1ilLCT533XUX999/f8f9//7v/+bHP/4xZ555Zse2OM8///wRz9u0aRPFxcUANDY2cvXVVzNmzBguueSSLmtX3nrrrZSUlDBu3DjuvvtuwLfo87Zt2zjjjDM444wzgENb9wD8+te/pri4mOLiYu69996O9+tpS5++0nVyIr01eBykFPhOWU79AgAT8lKIdrso37SHs8cOPsYLSER75S7Yviywr5k9Hub8tMeHr7rqKu68805uu+02AObOnctrr73GHXfcQXJyMrt27aKsrIwLL7wQY7pfnu73v/898fHxrFq1ioqKCqZMmdLx2E9+8hPS09Npb2/nzDPPpKKigjvuuINf//rXvP3222RmZnZ5rYULF/Loo4/yySefYK3l5JNPZtasWaSlpfV6S5/jpZ6cSG8Z4+vNbXgHWnynYmKj3IzPS2G+ZlhKEJo8eTI7d+5k27ZtLF26lLS0NLKzs/n+97/PhAkTOOuss6iqqmLHjh09vsZ7773XETYTJkxgwoQJHY/NnTuXKVOmMHnyZFasWMHKlSt7ehnAt/XOJZdcQkJCAomJiVx66aW8//77QO+39Dle6smJHI+i2TD/D7Dx3Y7Tl6WF6fzpgw00trQTF61NVKUHR+lx9acrrriCp59+mu3bt3PVVVfx2GOPUVNTw8KFC4mKiqKwsLDbLXaOZePGjfzyl79kwYIFpKWlccMNN5zQ6xzU2y19jpd6ciLHY+ipEJ0Ea+Z1NJUWptHablmytda5ukR6cNVVV/Hkk0/y9NNPc8UVV1BXV8egQYOIiori7bffZvPmzUd9/mmnncbjjz8OwPLly6moqABg3759JCQkkJKSwo4dO3jllUMzj3va4mfmzJk899xzNDQ0cODAAZ599llmzpwZwE97JPXkRI6HJxpGngVrXgWvF1wuSob6J59s2sP0ERkOFyjS1bhx49i/fz+5ubnk5ORw7bXXcsEFFzB+/HhKSkoYPXr0UZ9/6623cuONNzJmzBjGjBnD1KlTAZg4cSKTJ09m9OjR5OfnM2PGjI7n3HLLLcyePZshQ4bw9ttvd7RPmTKFG264gWnTpgFw0003MXny5ICdmuxOr7baMcbMBn4LuIGHrbXd9ruNMZcBTwOl1tqjXkihrXYkZFXMhWduhpvegjzfX/hzfvMeg5Jj+NuXTna4OAkmkb7VTn843q12jnm60hjjBu4H5gBjgWuMMWO7OS4J+DrwyQnULRI6TjoLjLvrKcthaSzavJe2dq+DhYnI4XozJjcNWG+t3WCtbQGeBC7q5rj/AX4GnPjIo0goiE+HguldVj8pLUznQEs7q7cfOQ4hIs7pTcjlAls73a/0t3UwxkwB8q21LwewNpHgVTQHdq6AvZsALdYsEqz6PLvSGOMCfg18sxfH3mKMKTfGlNfU1PT1rUWcc3D1kzW+1U+GpMaRmxqnkJMjaL/BwDmRP8vehFwVkN/pfp6/7aAkoBh4xxizCSgDXjDGHDEIaK19yFpbYq0tycrKOu5iRYJGxgjILIK1nU9ZprFg0179oyYdYmNj2b17t34nAsBay+7du4mNjT2u5/XmEoIFwEhjzDB84XY18LlOb1wHdKzdYox5B/jWsWZXioS8ojnw0f+DpjqITaGkMJ3nlmxj8+4GCjMTnK5OgkBeXh6VlZXozFVgxMbGkpeXd1zPOWbIWWvbjDG3A6/hu4TgEWvtCmPMPUC5tfaFE6pWJNQVzYF/3wvr34Diy5g27NC4nEJOAKKiohg2bJjTZUS0Xo3JWWvnWWtHWWtHWGt/4m/7YXcBZ609Xb04iQh5pRCf0THL8qSsRFLiojQuJxJEtKyXyIlyuX07hq97HdpbcbkMpYVplGuncJGgoZAT6YuiOb4xuS0fA1BSmM6GXQeo2d/scGEiAgo5kb4Zfga4oztOWR68Xm7hZp2yFAkGCjmRvohJhGGzfEt8Wcv43BRiPC7mb9QpS5FgoJAT6auiObB3I9SsIdrjYlJ+KuXqyYkEBYWcSF+Nmu279S/YXFqYzopt+zjQ3OZgUSICCjmRvkvJhZxJsNa3xFfpsHTavZbFW2odLUtEFHIigVE0B7bOh/oaphSk4jIwX9fLiThOIScSCEVzAAvrXiMpNooxOcmUK+REHKeQEwmE7AmQnNvlUoLFW2pp1SaqIo5SyIkEgjG+3tynb0FrE6WF6TS2trNi2z6nKxOJaAo5kUApmgOtDbDxPUoL0wBYsFGnLEWcpJATCZTCmRCdCGvmMSg5lqEZ8VqsWcRhCjmRQPHEwIjP+C4l8HopGZpO+WZtoiriJIWcSCAVnQv7q6F6CdOGpbHnQAuf1hxwuiqRiKWQEwmkkZ8F44K1r1JSeGgTVRFxhkJOJJASMiD/ZFgzj+GZCWQkRCvkRBykkBMJtKI5sH0Zpq6SksI0hZyIgxRyIoFWdK7vdu2rlBams3VPI9vrmpytSSRCKeREAi1zJGScBGvmdWyiqt6ciDMUciL9oWgObHyfcRkQH+3WOpYiDlHIifSHUXPA24pn4ztMLkhl/ibtFC7iBIWcSH/IPxni0mDNK5QWprN6+z72NbU6XZVIxFHIifQHtwdGngPrXqO0IBlrYeFm9eZEBppCTqS/FM2Bxr1Mda3F7TIalxNxgEJOpL+cdCa4o4nd8DrFQ5JZsFE9OZGBppAT6S8xSVB4ase43JLKWprb2p2uSiSiKORE+lPRubB7PbMyamlp87Ksss7pikQiikJOpD+Nmg3AlMaPAFigSwlEBpRCTqQ/peZD9ngSNr3B8KwErXwiMsAUciL9rehc2Poxp+e5KN+0B69Xm6iKDBSFnEh/GzUbrJfZMcvY19TG2p37na5IJGIo5ET6W84kSMph3P5/AxqXExlICjmR/uZywajZxG99l9xEFws2alxOZKAo5EQGQtG5mJZ6rhm0WSufiAwghZzIQBh2GkTFc6ZrIdvqmqjc2+B0RSIRQSEnMhCiYmHEZzhp7/uApVzjciIDQiEnMlCK5hB1oJqSmErm65SlyIBQyIkMlJHnAIbPpa7QuJzIAFHIiQyUxCzIn8ap7QtYu6OevQdanK5IJOwp5EQG0qjZDKpfRTa7+USXEoj0O4WcyEAqOheAC+IqeKlim8PFiIQ/hZzIQMoqgrRhXJ64jH+t3MH+planKxIJawo5kYFkDBSdy8gDi3C3NfDaih1OVyQS1hRyIgOtaA4ubwvXJC/l+SVVTlcjEtYUciIDbegMyBrNre4X+Gj9Tnbub3K6IpGwpZATGWguF8z6LpmNG5ltPuHFpdVOVyQSthRyIk4YezFkjebbcc/z4uItTlcjErYUciJOcLlg1ncY2r6FvOp/8WlNvdMViYQlhZyIU8ZeTFtGEXd4nuGFRerNifQHhZyIU1xuPGd8l1GuKuoW/hNrrdMViYQdhZyIk8ZeTF3iCD7X9ASLt2iZL5FAU8iJOMnlJurMuxjlqmL92393uhqRsKOQE3FY/MTL2BY9lKmbHqK1rc3pckTCikJOxGkuNzWTv84IKlmj3pxIQCnkRILA6DOv51NyyVjwG/B6nS5HJGwo5ESCQEx0NPMLbianZRNNFc84XY5I2FDIiQSJ4addyzpvLi1v/p96cyIBopATCRKlw7P4W/RVJO9fD6ued7ockbCgkBMJEi6XIWHK5ayzubS99VP15kQCQCEnEkQumpLPfa2X4Nm9Wr05kQBQyIkEkdHZyXyadRZb3fnwzs/UmxPpI4WcSJC5cEoBv2i8CGpWqTcn0kcKOZEgc+HEIbxsy9gTV6jenEgfKeREgsyQ1DhKCjN5wHuZvzf3gtMliYQshZxIELp4ci6P1E2mKWUEvKvenMiJUsiJBKFzi3PwuD28nP552LlSvTmRE6SQEwlCKfFRnDE6i59tGYPNGKXenMgJUsiJBKmLJ+Wy80Abq4u+ot6cyAlSyIkEqTNGDyIp1sPDeydBxkj15kROgEJOJEjFRrmZU5zNqytqaDn12+rNiZyAXoWcMWa2MWaNMWa9Meaubh7/ijFmmTFmiTHmA2PM2MCXKhJ5Lp6Uy4GWdl4zp/h7cz9Xb07kOBwz5IwxbuB+YA4wFrimmxB73Fo73lo7Cfg58OtAFyoSiU4enkF2cizPL90Os74LO1fA6hedLkskZPSmJzcNWG+t3WCtbQGeBC7qfIC1dl+nuwmADVyJIpHL7TJcOGkI76ypYc+w8329Oa2CItJrvQm5XGBrp/uV/rYujDG3GWM+xdeTu6O7FzLG3GKMKTfGlNfU1JxIvSIR56JJQ2jzWl5esRNmfUe9OZHjELCJJ9ba+621I4DvAv/ZwzEPWWtLrLUlWVlZgXprkbA2NieZkYMSeX5xFRRfBhknqTcn0ku9CbkqIL/T/Tx/W0+eBC7uQ00i0okxhosn51K+eS9ba5s1NidyHHoTcguAkcaYYcaYaOBqoMs8ZmPMyE53zwPWBa5EEblw4hAAXli67VBvTjMtRY7pmCFnrW0DbgdeA1YBc621K4wx9xhjLvQfdrsxZoUxZgnwDeAL/VWwSCTKT4+ntDCN5xZXYY3L15vbsRxWv+R0aSJBrVdjctbaedbaUdbaEdban/jbfmitfcH//detteOstZOstWdYa1f0Z9EikeiiSbms21nPyup9nXpzGpsTORqteCISIs4bn4PHZXhucRW43HDad9SbEzkGhZxIiEhLiOb0oixeWLqNdq9Vb06kFxRyIiHkokm57NjXzCcbdoPb06k3p5mWIt1RyImEkLPGDCYxxsNzS/xX8RRfBpmjYN53oK7S2eJEgpBCTiSExEW7OWdcNq8s205Ta7uvN3flX6G1AR67AprqnC5RJKgo5ERCzMWTh7C/uY23V+/0NQwa4wu6XWth7hegvdXZAkWCiEJOJMScMiKTrKQYnl3caeGhEWfABb+FDW/DS/8BVmuki4BCTiTkuF2GCyb4diaoa+jUa5t8nW8iyuK/wfu/cq5AkSCikBMJQRdPHkJLu5d5y6u7PnDG92HCVfDW/8Cyp50pTiSIKOREQtD43BSGZyX4LgzvzBi48HcwdAY8dyts/tCZAkWChEJOJAQZY7h4Ui6fbNzDttrGrg96YuCqv0PqUHjyc7BL66VL5FLIiYSoiyZ12pngcPHpcO1TYNzw2OVwYNcAVycSHBRyIiFqaEYCkwtSjzxleVD6MPjcP2D/dnjiamht7P44kTCmkBMJYRdPymX19v2s3r6v+wPySuDSP0JlOTxzi9a4lIijkBMJYedNyMHtMjy7qIfeHMDYC+GzP4ZVL8Abdw9ccSJBQCEnEsIyE2OYXZzN3z7ezPa6pp4PnH4blN4MH94HCx4euAJFHKaQEwlx3z1nNG1ey89fXd3zQcbA7J/CqNkw79uw9vWBK1DEQQo5kRBXkBHPzTOH8cziKhZt2dvzgW4PXPYnyB4PT90A1UsHrEYRpyjkRMLAV08/iUFJMfzoxZV4vUdZtzImET43F+LS4LErtT2PhD2FnEgYSIjx8N3Zo1m6tbbrws3dScqGa+f6t+e5Epp6mJkpEgYUciJh4pLJuUzMT+Vnr67mQHPb0Q8ePM6/Pc8aeErb80j4UsiJhAmXy3D3BWPZub+ZB95Zf+wnjDgDzr8XPn0LXv6GtueRsKSQEwkjUwrSuHRyLn98fyNbdjf04gnXw2nfhkV/hQ9+3f8FigwwhZxImPnO7NF4XIb/nbeqd0844wcw/gp48x5tzyNhRyEnEmayU2L56ukjeHXFdj5c34uFmY2Bi+6HglP82/N81P9FigwQhZxIGLpp5nDy0uK456WVtLX3Yr1KTwxc/Zhve57Hr4KKpzRGJ2FBIScShmKj3Pzg3DGs3r6fJxZs7d2T4tPh+mcgaxQ8cxPMvR7qa/q3UJF+ppATCVOzi7MpG57Or19fQ11DLy8RSC2AL74GZ/0I1r4GD5wMK5/v30JF+pFCTiRMGWP44fnjqGts5d431/b+iS43nHonfPk9SMmHuZ+Hp78EDXv6rVaR/qKQEwljY4ckc820Av760WbW7dh/fE8eNAZuegNO/z6sfA4eKIM1r/ZLnSL9RSEnEua+cfYo4qPd3PPSSuzxTiZxR8Hp34Wb34L4THjiKnjuq9BU1z/FigSYQk4kzGUkxnDnWaN4f90u3lq988ReJGci3PI2zPwmLH0CHpgO698MbKEi/UAhJxIBPj99KCOyEvjxy6toaevFJQXd8cTAmT+EL70B0Qnw90vhxTuh+ThPg4oMIIWcSASIcrv4r/PHsnHXAf7y4aa+vVjeVN+klOm3w8I/w+9PgY3vB6JMkYBTyIlEiNOLBvGZ0YO478111Oxv7tuLRcXBOT+BG18B44a/nA+vfBdaerFepsgAUsiJRJD/PG8Mja3t/Or1NYF5waHT4dZ/w7Rb4JMH4cFTYcsngXltkQBQyIlEkOFZidxwSiH/KN/K8qoAzZCMToBzfwGff8G3L92js+H1/4LWpsC8vkgfKOREIszXzhxJenw0P3pxxfFfUnA0w2f5enWTr4cP74OHZkHVosC9vsgJUMiJRJiUuCi+dU4RCzbt5aWK6sC+eGwyXHgfXPu071q6h8+CuV/wXW7gPcFZnSJ9oJATiUBXluQzNieZn76ymsaW9sC/wciz4asfQdmtsPFd3+UGv50I7/wUanu5YLRIACjkRCKQ22W4+4KxVNU28tB7G/rnTeLSfDMwv7kGLn8EMkbAO/8H946Hv10KK56Dtpb+eW8RP4WcSIQ6eXgG503I4ffvrmdbbWP/vZEnBoovg88/B19fCqd9G2pWw1NfgF+Phtd+ADtX99/7S0RTyIlEsO/NGY218NNXBihk0grhMz+AO5fBtf+EoTN8lx48cDI8fDYs+hs01w9MLRIRFHIiESwvLZ4vnzacF5ZuY8GmAdxKx+WGkWfBVX+Db6yGz/4YmmrhhdvhV0Xwwtdg6wLtTi59ZgI6hfg4lJSU2PLyckfeW0QOaWhp4zO/fJespBiev20GLpdxphBrYet8WPRXWPEMtDZA1hiYcj1MuBoSMpypS4KeMWahtbaku8fUkxOJcPHRHr537miWVdXx9KJK5woxBgpOhovv901WueC3vgvNX/u+r3c39wu+XcrrT3AnBYlI6smJCNZaLn/wIzbvbuDtb80iKTbK6ZIO2bHCN1ZX8SQ07vW1pY+AgulQUAZDT4H04b6QlIh0tJ6cQk5EAKiorOXC//dvvjxrON+bM8bpco7U1gLVS2DLR7DlY9/twdBLyPIF3sHgy57g2/BVIsLRQs4z0MWISHCakJfKFVPz+NP7G5kxIpPTRmU5XVJXnmjIn+b7mvF13woqu9cdCr3NH8KqF33HRsVDXsmh0MsrhZgkZ+sXR6gnJyId9jW1ctUfPmbTrgM8fvPJTC5Ic7qk47Nvm7+X5+/p7VgO1uvbDih7/KHQKyiDpGynq5UA0elKEem1nfubuOLBj6hrbOWpL09n5OAQ7gE17YPKBYdCr7Ic2vwXvqcUQMZw37V7HV/DfLdxqY6VLMdPIScix2XL7gYue/BDPC7D07eeQm5qnNMlBUZ7K1RXwJYPYdti2LvJ99Wwu+txsamHhV8hpPsDMDkP3BrpCSYKORE5bquq93HlHz4iKymGp748nYzEGKdL6j9N+w4F3uFftVvA23roWOOG1PyuAZg6FBIyISbZtxNDTIrvVpNfjs7b7lsYoI8UciJyQuZv3MP1f/qEouwkHr+5jMSYCOzBeNt9Y317N8HejUeG4OG9wM48cf7QS+7mNuXQ7eGPRcX7eouuKF9QuqIOu+/p/0smrPV9dm8bWP+tt913kX5zPbQcgJaDtwe/767df7/58GMPwKAx8OV3+1yqQk5ETtibq3Zwy98WUjY8nUduKCXG0/f/eYeVpn2+3l7jHt/3zfs63dYddv+w29aGE39fl8f/1U0Adg5G4/aHVPuh0DoYWIcH2OHtJ8oT57uQPzoBohN9tzGJXe9HJ0BqAZTedOLv46dLCETkhJ05ZjA/v2wC33xqKf/xjyX87popuJ1a+isYxSZDdvGJPbe9FZr3+9bt7ByAbU2+x7yt/tu2TvfbOrW3+kKpp8fa/WHl8oBxdQrGg1+HtR1xjNv/dfBxN0THHxlW0Yld7wfgFGSgKORE5Jgum5rH3oYWfvzyKlLjl/OTi4sxWmGk79xREJ/u+5J+oZATkV65aeZwdh9o4ffvfEpGQjTf/GyR0yWJHJNCTkR67TvnFLH3QAu/e2s96QnR3DhjmNMliRyVQk5Ees0Yw48vLmZvQws/enElafHRXDw51+myRHqkrXZE5Lh43C5+e/Vkpg/P4FtPLeXtNdr6RoKXQk5EjltslJuHPj+V0TlJ3Pr3hSzcPIC7ioscB4WciJyQpNgo/nzjNHJS4rjx0QWs2b7f6ZJEjqCQE5ETlpkYw1+/OI24aDfX/+kTtu7pw8XNIv1AIScifZKfHs9fv3gyzW1erv/TJ9Tsb3a6JJEOCjkR6bOi7CQeuaGE7fuauOHR+exraj32k0QGgEJORAJi6tB0fn/dVNZs38/NfymnqbUPax+KBIhCTkQC5oyiQfzyiol8snEPdzyxmLZ2r9MlSYRTyIlIQF08OZe7LxjL6yt38INnl+PUTicioBVPRKQf3DhjGHv8y3/VN7fxf5eNJzlWG4jKwFPIiUi/+MbZo0iI8fCL19ZQUVXL766ZwqT8VKfLkgjTq9OVxpjZxpg1xpj1xpi7unn8G8aYlcaYCmPMm8aYoYEvVURCiTGGr8wawdwvT8frhct//yF/fG8DXq9OX8rAOWbIGWPcwP3AHGAscI0xZuxhhy0GSqy1E4CngZ8HulARCU1Th6Yx746ZnDlmED+Zt4qb/lrOngMtTpclEaI3PblpwHpr7QZrbQvwJHBR5wOstW9baw8udfAxkBfYMkUklKXER/HgdVP50YXj+GDdLs797ft8smG302VJBOhNyOUCWzvdr/S39eRLwCvdPWCMucUYU26MKa+pqel9lSIS8owxfOGUQp756inERbu55o8f89s31tGu05fSjwJ6CYEx5jqgBPhFd49bax+y1pZYa0uysrIC+dYiEiKKc1N48WunctGkXH7zxlque/gTduxrcrosCVO9CbkqIL/T/Tx/WxfGmLOAHwAXWmu1eJ2I9CgxxsOvr5zILy6fwJKttZz72/d5R/vSST/oTcgtAEYaY4YZY6KBq4EXOh9gjJkM/AFfwOk3VUSOyRjDFSX5vPi1GWQlxXDDowv4v1dW0apVUiSAjhly1to24HbgNWAVMNdau8IYc48x5kL/Yb8AEoGnjDFLjDEv9PByIiJdnDQoiedum8G1Jxfwh3c3cOUfPtKWPRIwxqkld0pKSmx5ebkj7y0iwenlimru+mcFxsDPL5/A7OIcp0uSEGCMWWitLenuMa1dKSJB47wJObx8x0yGZSbwlb8v4ofPL9duBtInCjkRCSoFGfE89ZVTuHnmMP760WYueeBDNtTUO12WhCiFnIgEnWiPix+cN9a3EWtdI+f/7gOeWVTpdFkSghRyIhK0PjN6MPO+PpPi3BS+MXcptz2+SJNS5Lgo5EQkqOWkxPH4TSfzjbNH8eaqHZz5q3f58UsrqW3Q+pdybAo5EQl6HreLO84cyTvfOoNLJufyyL83ctrP3+ah9z7VxBQ5KoWciISM7JRYfnb5BOZ9fSZTh6bxv/NWc+av3uXZxZXawke6pZATkZAzOjuZR2+cxuM3nUxaQhT/8Y+lXPD/PuCDdbucLk2CjEJORELWKSdl8sJtp/LbqydR29DKdX/6hC88Mp9V1fucLk2ChEJOREKay2W4aFIub31rFv953hjfgs/3vc+3nlpKdV2j0+WJw7Ssl4iElbqGVu5/Zz1//vcmjIEvnTqMr5w+guTYKKdLk35ytGW9FHIiEpa27mngV6+v4bkl20iLj+KOM0dy7clDifboBFa40dqVIhJx8tPjuffqybz0tVMZk5PMj15cydm/eZeXK6px6j/3MvAUciIS1opzU3jsppP5842lxEW5ue3xRVzywId8vGG306XJANDpShGJGO1eyz8XVfLr19eyfV8TkwtSuXnmcM4Zl43bZZwuT06QxuRERDppbGlnbvlW/vTBRrbsaSA/PY4bTxnGlaX5JMZ4nC5PjpNCTkSkG+1ey79W7uDh9zdQvnkvSbEePjetgBtmFJKTEud0edJLCjkRkWNYvGUvD3+wkVeWVeMyhvMm5HDzzOEU56Y4XZocg0JORKSXtu5p4M8fbuIfC7ZS39zGycPSuXnmcD4zehAujdsFJYWciMhx2tfUyj/mb+XRf29kW10TwzMT+OKpw7hsSh5x0W6ny5NOFHIiIieotd3LK8u38/D7G6iorCMtPorryoZy/fShDEqKdbo8QSEnItJn1loWbNrLH9/fwBurdhDlcnHRpCHcNHM4RdlJTpcX0Y4WcporKyLSC8YYpg1LZ9qwdDbuOsAjH2zkqYVbeWphJTNHZvL56YWcUZSFx601NoKJenIiIido74EWHp+/hb98uImd+5sZnBzDlSX5XFmST356vNPlRQydrhQR6Uet7V7eWr2TJ+dv4Z21NQCcelIm10wr4Kwxg7UodD9TyImIDJCq2kbmLtjKU+Vb2VbXRGZiNJdNzePq0gKGZSY4XV5YUsiJiAywdq/lvbU1PDF/C2+u3km711I2PJ1rphVwzrhsYqN0GUKgKORERBy0c18TTy2s5MkFW9i6p5HU+CgunZzHNdPyGTlYMzP7SiEnIhIEvF7Lh5/u5okFW3h9xXZa2y1Th6ZxzbQCzhufo4vMT5BCTkQkyOyub+afiyp5cv5WNuw6QFKsh4sn5XL1tHzGDdF6mcdDISciEqSstczfuIcnF2zl5WXVtLR5Kc5N5pLJeVw4cQhZSTFOlxj0FHIiIiGgtqGFZxdX8cyiKpZV1eF2GWaOzOSSybl8dmy2Tmf2QCEnIhJi1u3Yz7OLq3h+yTaqahtJiHZzTnE2l07OY/qIDO1k3olCTkQkRHm9lvmb9vDsoirmLatmf3Mbg5NjuGhSLpdMzmVMTrLTJTpOISciEgaaWtt5c9VOnl1cyTtramjzWkZnJ3HJ5FwumpRLdkpk7oqgkBMRCTO765t5qaKaZxdXsWRrLcbAjBGZXDw5l9nF2STGRM76+wo5EZEwtqGmnucWV/Hskiq27mkkNsrFOeOyuXhyLjNPygz7nREUciIiEcBay8LNe3lmcRUvV1RT19hKZmI0547P4YKJQ5hakIYrDCesKORERCJMc1s7b6+u4fklVby1eifNbV5yUmI5f4Iv8MbnpmBMeASeQk5EJILVN7fxxsodvLh0G++tq6G13TI0I74j8IoGJ4V04CnkREQE8F1w/tqK7bxUUc2/1+/Ca2HkoEQumDiE8yfkMDwr0ekSj5tCTkREjlCzv5lXl1fz4tJq5m/aA0BxbjLnT/AFXl5aaOxurpATEZGjqq5r5OWKal6sqGbp1loAphSkcsHEIZw3PodBycF7DZ5CTkREem3L7gZeWraNF5dWs6p6H8ZA2bAMzp+Yw+xx2WQkBtei0Qo5ERE5Iet37ufFpdW8WLGNDTUHcBmYNiydOcU5nDMuOyhWWVHIiYhIn1hrWVW9n1eXV/PK8u2s21kP+E5pzinOYXZxNvnpzozhKeRERCSg1u/cz6vLt/PK8u2s2LYP8E1aORh4IwZwlqZCTkRE+s2W3Q28usLXw1u8pRaAUYMTmV2cw5zibEZn9+91eAo5EREZENV1jbzm7+Et2LQHr4XCjPiOwJuQF/iVVhRyIiIy4HbVN/P6ih28sryajz7dTZvXMiQllnOKs5lTnMPUoWkB2fxVISciIo6qbWjhjVU7eXV5Ne+t20VLm5fR2Um88vWZfe7ZHS3kImfDIRERcUxqfDSXT83j8ql51De38dbqnRxobuv3NTMVciIiMqASYzxcOHHIgLxXeO+kJyIiEU0hJyIiYUshJyIiYUshJyIiYUshJyIiYUshJyIiYUshJyIiYUshJyIiYUshJyIiYUshJyIiYUshJyIiYUshJyIiYUshJyIiYUshJyIiYUshJyIiYUshJyIiYUshJyIiYUshJyIiYUshJyIiYUshJyIiYUshJyIiYctYa515Y2NqgM0BeKlMYFcAXsdJof4ZQr1+CP3PoPqdF+qfIZTrH2qtzeruAcdCLlCMMeXW2hKn6+iLUP8MoV4/hP5nUP3OC/XPEOr190SnK0VEJGwp5EREJGyFQ8g95HQBARDqnyHU64fQ/wyq33mh/hlCvf5uhfyYnIiISE/CoScnIiLSLYWciIiErZAJOWPMbGPMGmPMemPMXd08HmOM+Yf/8U+MMYUOlNkjY0y+MeZtY8xKY8wKY8zXuznmdGNMnTFmif/rh07U2hNjzCZjzDJ/beXdPG6MMff5fwYVxpgpTtTZHWNMUac/1yXGmH3GmDsPOybo/vyNMY8YY3YaY5Z3aks3xvzLGLPOf5vWw3O/4D9mnTHmCwNXdZcauqv/F8aY1f7fkWeNMak9PPeov28DpYfP8N/GmKpOvyvn9vDco/67NRB6qP8fnWrfZIxZ0sNzg+Jn0CfW2qD/AtzAp8BwIBpYCow97JivAg/6v78a+IfTdR9WXw4wxf99ErC2m89wOvCS07Ue5TNsAjKP8vi5wCuAAcqAT5yu+Si/T9vxXUAa1H/+wGnAFGB5p7afA3f5v78L+Fk3z0sHNvhv0/zfpwVJ/Z8FPP7vf9Zd/b35fXP4M/w38K1e/J4d9d8tp+o/7PFfAT8M5p9BX75CpSc3DVhvrd1grW0BngQuOuyYi4C/+L9/GjjTGGMGsMajstZWW2sX+b/fD6wCcp2tKuAuAv5qfT4GUo0xOU4X1Y0zgU+ttYFYcadfWWvfA/Yc1tz5d/0vwMXdPPUc4F/W2j3W2r3Av4DZ/VVnT7qr31r7urW2zX/3YyBvoOs6Hj38DHqjN/9u9buj1e//N/JK4IkBLWoAhUrI5QJbO92v5MiA6DjG/xeoDsgYkOqOk/9U6mTgk24enm6MWWqMecUYM25gKzsmC7xujFlojLmlm8d783MKBlfT81/qYP7zP2iwtbba//12YHA3x4TKz+KL+Hr/3TnW75vTbvefcn2kh1PGofAzmAnssNau6+HxYP8ZHFOohFzYMMYkAv8E7rTW7jvs4UX4TqFNBH4HPDfA5R3LqdbaKcAc4DZjzGlOF3S8jDHRwIXAU908HOx//kewvnNKIXkdkDHmB0Ab8FgPhwTz79vvgRHAJKAa3ym/UHQNR+/FBfPPoFdCJeSqgPxO9/P8bd0eY4zxACnA7gGprpeMMVH4Au4xa+0zhz9urd1nra33fz8PiDLGZA5wmT2y1lb5b3cCz+I7HdNZb35OTpsDLLLW7jj8gWD/8+9kx8HTwP7bnd0cE9Q/C2PMDcD5wLX+oD5CL37fHGOt3WGtbbfWeoE/0n1twf4z8ACXAv/o6Zhg/hn0VqiE3AJgpDFmmP9/4lcDLxx2zAvAwRlklwNv9fSXxwn+c99/AlZZa3/dwzHZB8cRjTHT8P18giKojTEJxpikg9/jmzyw/LDDXgA+759lWQbUdTqtFix6/J9rMP/5H6bz7/oXgOe7OeY14LPGmDT/qbTP+tscZ4yZDXwHuNBa29DDMb35fXPMYWPNl9B9bb35d8tJZwGrrbWV3T0Y7D+DXnN65ktvv/DN3FuLb7bSD/xt9+D7iwIQi+8U1HpgPjDc6ZoPq/9UfKeVKoAl/q9zga8AX/EfczuwAt8srI+BU5yuu1P9w/11LfXXePBn0Ll+A9zv/xktA0qcrvuwz5CAL7RSOrUF9Z8/vkCuBlrxjel8Cd9Y85vAOuANIN1/bAnwcKfnftH/92E9cGMQ1b8e31jVwb8HB2dFDwHmHe33LYg+w9/8v+MV+IIr5/DP4L9/xL9bwVC/v/3PB3/3Ox0blD+DvnxpWS8REQlboXK6UkRE5Lgp5EREJGwp5EREJGwp5EREJGwp5EREJGwp5ERCnH/3hJecrkMkGCnkREQkbCnkRAaIMeY6Y8x8/95cfzDGuI0x9caY3xjfHoNvGmOy/MdOMsZ83GnPtTR/+0nGmDf8i0gvMsaM8L98ojHmaf8+bY8F0w4cIk5SyIkMAGPMGOAqYIa1dhLQDlyLbxWWcmvtOOBd4G7/U/4KfNdaOwHfyhoH2x8D7re+RaRPwbeSBfh2tbgTGItvpYoZ/fyRREKCx+kCRCLEmcBUYIG/kxWHb2FlL4cWyP078IwxJgVItda+62//C/CUfx3BXGvtswDW2iYA/+vNt/41CP27PBcCH/T7pxIJcgo5kYFhgL9Ya7/XpdGY/zrsuBNdZ6+50/ft6O+2CKDTlSID5U3gcmPMIABjTLoxZii+v4OX+4/5HPCBtbYO2GuMmelvvx541/p2lK80xlzsf40YY0z8QH4IkVCj/+2JDABr7UpjzH/i22XZhW9F+NuAA8A0/2M78Y3bgW8LnQf9IbYBuNHffj3wB2PMPf7XuGIAP4ZIyNEuBCIOMsbUW2sTna5DJFzpdKWIiIQt9eRERCRsqScnIiJhSyEnIiJhSyEnIiJhSyEnIiJhSyEnIiJh6/8Ddzpqxbnz44gAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Loss\n", "\ttraining \t (min: 0.130, max: 0.706, cur: 0.130)\n", "\tvalidation \t (min: 0.224, max: 0.696, cur: 0.226)\n" ] } ], "source": [ "gmf_recommender = GMFRecommender(print_type='live', n_neg_per_pos=10, batch_size=16, \n", " embedding_dim=6, lr=0.001, weight_decay=0.0001, n_epochs=20, seed=1)\n", "gmf_recommender.fit(ml_ratings_df, None, ml_movies_df)" ] }, { "cell_type": "markdown", "id": "incorporated-messaging", "metadata": {}, "source": [ "## Quick test of the recommender (recommending)" ] }, { "cell_type": "code", "execution_count": 5, "id": "accessible-value", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Recommendations\n" ] }, { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
user_iditem_idscoretitlegenres
0148960.768898Harry Potter and the Sorcerer's Stone (a.k.a. Harry Potter and the Philosopher's Stone) (2001)Adventure|Children|Fantasy
114350.650600Coneheads (1993)Comedy|Sci-Fi
21415660.609373Chronicles of Narnia: The Lion, the Witch and the Wardrobe, The (2005)Adventure|Children|Fantasy
3165020.53533228 Days Later (2002)Action|Horror|Sci-Fi
411450.441272Bad Boys (1995)Action|Comedy|Crime|Drama|Thriller
5165370.432268Terminator 3: Rise of the Machines (2003)Action|Adventure|Sci-Fi
613550.421626Flintstones, The (1994)Children|Comedy|Fantasy
7156730.242538Punch-Drunk Love (2002)Comedy|Drama|Romance
814810.218651Kalifornia (1993)Drama|Thriller
912670.213728Major Payne (1995)Comedy
1047800.858898Independence Day (a.k.a. ID4) (1996)Action|Adventure|Sci-Fi|Thriller
1144350.634766Coneheads (1993)Comedy|Sci-Fi
124415660.597829Chronicles of Narnia: The Lion, the Witch and the Wardrobe, The (2005)Adventure|Children|Fantasy
13465020.53141728 Days Later (2002)Action|Horror|Sci-Fi
1441450.447853Bad Boys (1995)Action|Comedy|Crime|Drama|Thriller
15465370.439573Terminator 3: Rise of the Machines (2003)Action|Adventure|Sci-Fi
1643550.430258Flintstones, The (1994)Children|Comedy|Fantasy
17456730.266561Punch-Drunk Love (2002)Comedy|Drama|Romance
1844810.243838Kalifornia (1993)Drama|Thriller
1942670.239114Major Payne (1995)Comedy
20648960.687780Harry Potter and the Sorcerer's Stone (a.k.a. Harry Potter and the Philosopher's Stone) (2001)Adventure|Children|Fantasy
216415660.572620Chronicles of Narnia: The Lion, the Witch and the Wardrobe, The (2005)Adventure|Children|Fantasy
22615000.572483Grosse Pointe Blank (1997)Comedy|Crime|Romance
23665020.52322028 Days Later (2002)Action|Horror|Sci-Fi
24665370.455307Terminator 3: Rise of the Machines (2003)Action|Adventure|Sci-Fi
25656730.321320Punch-Drunk Love (2002)Comedy|Drama|Romance
2664810.302354Kalifornia (1993)Drama|Thriller
27648900.270704Shallow Hal (2001)Comedy|Fantasy|Romance
28659540.26198125th Hour (2002)Crime|Drama
29634680.239384Hustler, The (1961)Drama
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "recommendations = gmf_recommender.recommend(pd.DataFrame([[1], [4], [6]], columns=['user_id']), ml_movies_df, 10)\n", "\n", "recommendations = pd.merge(recommendations, ml_movies_df, on='item_id', how='left')\n", "print(\"Recommendations\")\n", "display(HTML(recommendations.to_html()))" ] }, { "cell_type": "markdown", "id": "documentary-barcelona", "metadata": {}, "source": [ "## User and item representations" ] }, { "cell_type": "code", "execution_count": 8, "id": "balanced-detective", "metadata": { "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "User id=1\n", "[ 8.8694301e-03 -1.1293894e-09 7.6482260e-01 6.5688614e-06\n", " 6.1402158e-03 -3.4989858e-10 3.0581679e-05 1.6342730e-05]\n", "\n", "User watched\n", "['Independence Day (a.k.a. ID4) (1996)', 'Grosse Pointe Blank (1997)', 'Ladyhawke (1985)']\n", "\n", "User history item representations\n", "Item id = 780\titem title = Independence Day (a.k.a. ID4) (1996)\n", "[-2.0800237e-01 -3.2530998e-08 -7.2467870e-01 -7.6390163e-04\n", " 6.0946174e-02 -1.0309565e-09 -1.6934791e-03 -3.3520073e-02]\n", "Scalar product=-0.555722\n", "Score=0.884161\n", "\n", "Item id = 1500\titem title = Grosse Pointe Blank (1997)\n", "[-4.7350328e-02 -1.4992246e-09 -1.5850608e-01 -2.9982104e-05\n", " 6.0663655e-02 4.1064720e-08 1.5929480e-04 1.2831817e-03]\n", "Scalar product=-0.121276\n", "Score=0.609364\n", "\n", "Item id = 3479\titem title = Ladyhawke (1985)\n", "[-2.8682781e-02 6.1106755e-09 6.3241005e-01 -3.3657509e-06\n", " 9.6770316e-02 9.6757424e-10 -6.0637249e-05 1.5274031e-03]\n", "Scalar product=0.484021\n", "Score=0.145174\n", "\n", "===============\n", "Item id = 145\titem title = Bad Boys (1995)\n", "[-9.6727222e-02 1.2952676e-09 8.4303088e-02 1.5707446e-05\n", " 9.7245917e-02 -9.5372132e-10 -9.6978983e-05 1.0601738e-02]\n", "Scalar product=0.064216\n", "Score=0.441272\n", "\n", "Item id = 171\titem title = Jeffrey (1995)\n", "[ 7.6405336e-03 -6.6923184e-10 9.0268552e-01 -5.7306852e-06\n", " -1.5152089e-02 -9.7515729e-10 -1.3149886e-04 4.9494698e-08]\n", "Scalar product=0.690369\n", "Score=0.073709\n" ] } ], "source": [ "user_id = 1\n", "user_repr = gmf_recommender.get_user_repr(user_id=user_id)\n", "print(\"User id={}\".format(user_id))\n", "print(user_repr)\n", "print()\n", "\n", "print(\"User watched\")\n", "print(ml_df.loc[ml_df['user_id'] == user_id, 'title'].tolist())\n", "print()\n", "\n", "print('User history item representations')\n", "for item_id in ml_df.loc[ml_df['user_id'] == user_id, 'item_id'].tolist():\n", " item_repr = gmf_recommender.get_item_repr(item_id=item_id)\n", " print(\"Item id = {}\\titem title = {}\".format(\n", " item_id, ml_movies_df.loc[ml_movies_df['item_id'] == item_id, 'title'].iloc[0]))\n", " print(item_repr)\n", " scalar_product = np.dot(user_repr, item_repr)\n", " print(\"Scalar product={:.6f}\".format(scalar_product))\n", " score = gmf_recommender.nn_model(\n", " torch.tensor([[gmf_recommender.user_id_mapping[user_id], \n", " gmf_recommender.item_id_mapping[item_id]]]).to(gmf_recommender.device)).flatten().detach().cpu().item()\n", " print(\"Score={:.6f}\".format(score))\n", " print()\n", "\n", "print(\"===============\")\n", " \n", "item_id = 145\n", "item_repr = gmf_recommender.get_item_repr(item_id=item_id)\n", "print(\"Item id = {}\\titem title = {}\".format(item_id, ml_movies_df.loc[ml_movies_df['item_id'] == item_id, 'title'].iloc[0]))\n", "print(item_repr)\n", "score = np.dot(user_repr, item_repr)\n", "print(\"Scalar product={:.6f}\".format(score))\n", "score = gmf_recommender.nn_model(\n", " torch.tensor([[gmf_recommender.user_id_mapping[user_id], \n", " gmf_recommender.item_id_mapping[item_id]]]).to(gmf_recommender.device)).flatten().detach().cpu().item()\n", "print(\"Score={:.6f}\".format(score))\n", "print()\n", "\n", "item_id = 171\n", "item_repr = gmf_recommender.get_item_repr(item_id=item_id)\n", "print(\"Item id = {}\\titem title = {}\".format(item_id, ml_movies_df.loc[ml_movies_df['item_id'] == item_id, 'title'].iloc[0]))\n", "print(item_repr)\n", "score = np.dot(user_repr, item_repr)\n", "print(\"Scalar product={:.6f}\".format(score))\n", "score = gmf_recommender.nn_model(\n", " torch.tensor([[gmf_recommender.user_id_mapping[user_id], \n", " gmf_recommender.item_id_mapping[item_id]]]).to(gmf_recommender.device)).flatten().detach().cpu().item()\n", "print(\"Score={:.6f}\".format(score))" ] }, { "cell_type": "markdown", "id": "framed-negative", "metadata": {}, "source": [ "# Training-test split evaluation" ] }, { "cell_type": "code", "execution_count": 9, "id": "amended-future", "metadata": {}, "outputs": [], "source": [ "from evaluation_and_testing.testing import evaluate_train_test_split_implicit" ] }, { "cell_type": "code", "execution_count": 43, "id": "unsigned-video", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
RecommenderHR@1HR@3HR@5HR@10NDCG@1NDCG@3NDCG@5NDCG@10
0GMFRecommender0.2922080.4870130.6623380.8051950.2922080.4049140.4772920.52351
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "gmf_recommender = GMFRecommender(n_neg_per_pos=10, batch_size=16, \n", " embedding_dim=6, lr=0.001, weight_decay=0.0001, n_epochs=20)\n", "\n", "gmf_tts_results = [['GMFRecommender'] + list(evaluate_train_test_split_implicit(\n", " gmf_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df))]\n", "\n", "gmf_tts_results = pd.DataFrame(\n", " gmf_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n", "\n", "display(HTML(gmf_tts_results.to_html()))" ] }, { "cell_type": "code", "execution_count": 14, "id": "romantic-music", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
RecommenderHR@1HR@3HR@5HR@10NDCG@1NDCG@3NDCG@5NDCG@10
0NetflixRecommender0.2922080.5389610.7337660.9480520.2922080.4342890.5142030.583217
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "from recommenders.netflix_recommender import NetflixRecommender\n", "\n", "netflix_recommender = NetflixRecommender(n_epochs=150)\n", "\n", "netflix_tts_results = [['NetflixRecommender'] + list(evaluate_train_test_split_implicit(\n", " netflix_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df))]\n", "\n", "netflix_tts_results = pd.DataFrame(\n", " netflix_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n", "\n", "display(HTML(netflix_tts_results.to_html()))" ] }, { "cell_type": "code", "execution_count": 15, "id": "standing-tiffany", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
RecommenderHR@1HR@3HR@5HR@10NDCG@1NDCG@3NDCG@5NDCG@10
0AmazonRecommender0.1818180.3116880.4025970.5519480.1818180.2578060.2946820.34147
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "from recommenders.amazon_recommender import AmazonRecommender\n", "\n", "amazon_recommender = AmazonRecommender()\n", "\n", "amazon_tts_results = [['AmazonRecommender'] + list(evaluate_train_test_split_implicit(\n", " amazon_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df))]\n", "\n", "amazon_tts_results = pd.DataFrame(\n", " amazon_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n", "\n", "display(HTML(amazon_tts_results.to_html()))" ] }, { "cell_type": "code", "execution_count": 16, "id": "saving-harrison", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
RecommenderHR@1HR@3HR@5HR@10NDCG@1NDCG@3NDCG@5NDCG@10
0TFIDFRecommender0.0259740.0909090.1363640.3181820.0259740.0643930.0836850.140799
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "from recommenders.tfidf_recommender import TFIDFRecommender\n", "\n", "tfidf_recommender = TFIDFRecommender()\n", "\n", "tfidf_tts_results = [['TFIDFRecommender'] + list(evaluate_train_test_split_implicit(\n", " tfidf_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df))]\n", "\n", "tfidf_tts_results = pd.DataFrame(\n", " tfidf_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n", "\n", "display(HTML(tfidf_tts_results.to_html()))" ] }, { "cell_type": "code", "execution_count": 44, "id": "random-source", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
RecommenderHR@1HR@3HR@5HR@10NDCG@1NDCG@3NDCG@5NDCG@10
0GMFRecommender0.2922080.4870130.6623380.8051950.2922080.4049140.4772920.523510
1NetflixRecommender0.2922080.5389610.7337660.9480520.2922080.4342890.5142030.583217
2AmazonRecommender0.1818180.3116880.4025970.5519480.1818180.2578060.2946820.341470
3TFIDFRecommender0.0259740.0909090.1363640.3181820.0259740.0643930.0836850.140799
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "tts_results = pd.concat([gmf_tts_results, netflix_tts_results, amazon_tts_results, tfidf_tts_results]).reset_index(drop=True)\n", "display(HTML(tts_results.to_html()))" ] }, { "cell_type": "markdown", "id": "continued-harassment", "metadata": {}, "source": [ "# Leave-one-out evaluation" ] }, { "cell_type": "code", "execution_count": 30, "id": "exact-stuff", "metadata": {}, "outputs": [], "source": [ "from evaluation_and_testing.testing import evaluate_leave_one_out_implicit" ] }, { "cell_type": "code", "execution_count": null, "id": "divided-resistance", "metadata": {}, "outputs": [], "source": [ "gmf_recommender = GMFRecommender(n_epochs=10)\n", "\n", "gmf_loo_results = [['NetflixRecommender'] + list(evaluate_leave_one_out_implicit(\n", " gmf_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df, max_evals=300, seed=6789))]\n", "\n", "gmf_loo_results = pd.DataFrame(\n", " gmf_loo_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n", "\n", "display(HTML(gmf_loo_results.to_html()))" ] }, { "cell_type": "code", "execution_count": 31, "id": "prerequisite-lounge", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
RecommenderHR@1HR@3HR@5HR@10NDCG@1NDCG@3NDCG@5NDCG@10
0UserBasedCosineNearestNeighborsRecommender0.0966670.1466670.1866670.3066670.0966670.1242850.1407820.178962
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "netflix_recommender = NetflixRecommender(n_epochs=10)\n", "\n", "netflix_loo_results = [['NetflixRecommender'] + list(evaluate_leave_one_out_implicit(\n", " netflix_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df, max_evals=300, seed=6789))]\n", "\n", "netflix_loo_results = pd.DataFrame(\n", " netflix_loo_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n", "\n", "display(HTML(netflix_loo_results.to_html()))" ] }, { "cell_type": "code", "execution_count": 35, "id": "social-escape", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
RecommenderHR@1HR@3HR@5HR@10NDCG@1NDCG@3NDCG@5NDCG@10
0AmazonRecommender0.1666670.2566670.320.4266670.1666670.2190860.2454860.279978
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "from recommenders.amazon_recommender import AmazonRecommender\n", "\n", "amazon_recommender = AmazonRecommender()\n", "\n", "amazon_loo_results = [['AmazonRecommender'] + list(evaluate_leave_one_out_implicit(\n", " amazon_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df, max_evals=300, seed=6789))]\n", "\n", "amazon_loo_results = pd.DataFrame(\n", " amazon_loo_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n", "\n", "display(HTML(amazon_loo_results.to_html()))" ] }, { "cell_type": "code", "execution_count": 36, "id": "behind-cambodia", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
RecommenderHR@1HR@3HR@5HR@10NDCG@1NDCG@3NDCG@5NDCG@10
0TFIDFRecommender0.0066670.0533330.1233330.2333330.0066670.0334910.0621780.096151
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "tfidf_recommender = TFIDFRecommender()\n", "\n", "tfidf_loo_results = [['TFIDFRecommender'] + list(evaluate_leave_one_out_implicit(\n", " tfidf_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df, max_evals=300, seed=6789))]\n", "\n", "tfidf_loo_results = pd.DataFrame(\n", " tfidf_loo_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n", "\n", "display(HTML(tfidf_loo_results.to_html()))" ] }, { "cell_type": "code", "execution_count": 37, "id": "lightweight-password", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
RecommenderHR@1HR@3HR@5HR@10NDCG@1NDCG@3NDCG@5NDCG@10
0UserBasedCosineNearestNeighborsRecommender0.0966670.1466670.1866670.3066670.0966670.1242850.1407820.178962
1UserBasedCosineNearestNeighborsRecommender0.1000000.1500000.1800000.3133330.1000000.1271820.1395180.181748
2UserBasedCosineNearestNeighborsRecommender0.2666670.4200000.5133330.6500000.2666670.3577360.3960330.440599
3UserBasedCosineNearestNeighborsRecommender0.1733330.2800000.3366670.4200000.1733330.2345220.2577590.284723
4AmazonRecommender0.1666670.2566670.3200000.4266670.1666670.2190860.2454860.279978
5TFIDFRecommender0.0066670.0533330.1233330.2333330.0066670.0334910.0621780.096151
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "loo_results = pd.concat([gmf_loo_results, netflix_loo_results, amazon_loo_results, tfidf_loo_results]).reset_index(drop=True)\n", "display(HTML(loo_results.to_html()))" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.9" } }, "nbformat": 4, "nbformat_minor": 5 }