339 lines
14 KiB
Plaintext
339 lines
14 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "alike-morgan",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"%matplotlib inline\n",
|
|
"%load_ext autoreload\n",
|
|
"%autoreload 2\n",
|
|
"\n",
|
|
"import numpy as np\n",
|
|
"import pandas as pd\n",
|
|
"import matplotlib.pyplot as plt\n",
|
|
"import seaborn as sns\n",
|
|
"from IPython.display import Markdown, display, HTML\n",
|
|
"from collections import defaultdict\n",
|
|
"\n",
|
|
"# Fix the dying kernel problem (only a problem in some installations - you can remove it, if it works without it)\n",
|
|
"import os\n",
|
|
"os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "friendly-herald",
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"from data_preprocessing.dataset_specification import DatasetSpecification\n",
|
|
"from data_preprocessing.data_preprocessing_toolkit import DataPreprocessingToolkit\n",
|
|
"from data_preprocessing.people_identifier import PeopleIdentifier"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "prepared-signal",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Load original data"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "solid-crisis",
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"data_path = os.path.join(\"data\", \"hotel_data\")\n",
|
|
"\n",
|
|
"original_data = pd.read_csv(os.path.join(data_path, \"hotel_data_original.csv\"), index_col=0)\n",
|
|
"\n",
|
|
"original_data = original_data.replace({\"\\\\N\": \"\"})\n",
|
|
"original_data = original_data.fillna(\"\")\n",
|
|
"\n",
|
|
"numeric_columns = [\"n_people\", \"n_children_1\", \"n_children_2\", \"n_children_3\",\n",
|
|
" \"discount\", \"accomodation_price\", \"meal_price\", \"service_price\",\n",
|
|
" \"paid\"]\n",
|
|
"\n",
|
|
"for column in numeric_columns:\n",
|
|
" original_data.loc[:, column] = pd.to_numeric(original_data.loc[:, column], errors=\"coerce\")\n",
|
|
"\n",
|
|
"original_data = original_data.astype(\n",
|
|
" {\n",
|
|
" \"date_from\": np.datetime64,\n",
|
|
" \"date_to\": np.datetime64,\n",
|
|
" \"booking_time\": np.datetime64,\n",
|
|
" \"booking_date\": np.datetime64,\n",
|
|
" \"n_people\": np.int64,\n",
|
|
" \"n_children_1\": np.int64,\n",
|
|
" \"n_children_2\": np.int64,\n",
|
|
" \"n_children_3\": np.int64,\n",
|
|
" \"discount\": np.float64,\n",
|
|
" \"accomodation_price\": np.float64,\n",
|
|
" \"meal_price\": np.float64,\n",
|
|
" \"service_price\": np.float64,\n",
|
|
" \"paid\": np.float64,\n",
|
|
" }\n",
|
|
" )\n",
|
|
"\n",
|
|
"display(HTML(original_data.head(15).to_html()))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "endangered-lingerie",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Preprocess the data\n",
|
|
"\n",
|
|
"- Identify users by client_id, name hash, phone hash, email hash.\n",
|
|
"- Fix date_to - originally it points to the last full day of stay, not the departure date.\n",
|
|
"- Add length of stay.\n",
|
|
"- Add book to arrival.\n",
|
|
"- Add number of rooms (important for group reservations).\n",
|
|
"- Add indicator for stays encompasing a weekend.\n",
|
|
"- Add night price.\n",
|
|
"- Fix book to arrival to be not smaller than 0.\n",
|
|
"- Filter out companies as recommendations for such clients should work differently.\n",
|
|
"- Aggregate group reservations into single interactions.\n",
|
|
"\n",
|
|
"<span style=\"color:red\"><font size=\"4\">**Task:**</font></span><br> \n",
|
|
"In the file data_preprocessing/data_preprocessing_toolkit write code for the add_length_of_stay and add_night_price methods:\n",
|
|
" - add_length_of_stay - should add 'length_of_stay' variable to the DataFrame, which counts the number of nights the customer stayed at the hotel,\n",
|
|
" - add_night_price - should add 'night_price' column to the dataset DataFrame, which shows the average accomodation price per night per room (there can be many rooms in group reservations - 'n_rooms' column).\n",
|
|
"You have to pass all assertions."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "swedish-iceland",
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"preprocessed_data = original_data.copy()\n",
|
|
"\n",
|
|
"dataset_specification = DatasetSpecification()\n",
|
|
"dp_toolkit = DataPreprocessingToolkit()\n",
|
|
"\n",
|
|
"id_column_names = dataset_specification.get_id_columns()\n",
|
|
"\n",
|
|
"people_identifier = PeopleIdentifier()\n",
|
|
"preprocessed_data = people_identifier.add_pid(preprocessed_data, id_column_names, \"user_id\")\n",
|
|
"\n",
|
|
"preprocessed_data = dp_toolkit.fix_date_to(preprocessed_data)\n",
|
|
"preprocessed_data = dp_toolkit.add_length_of_stay(preprocessed_data) # Code this method\n",
|
|
"preprocessed_data = dp_toolkit.add_book_to_arrival(preprocessed_data)\n",
|
|
"preprocessed_data = dp_toolkit.add_nrooms(preprocessed_data)\n",
|
|
"preprocessed_data = dp_toolkit.add_weekend_stay(preprocessed_data)\n",
|
|
"preprocessed_data = dp_toolkit.clip_book_to_arrival(preprocessed_data)\n",
|
|
"\n",
|
|
"preprocessed_data = dp_toolkit.sum_npeople(preprocessed_data)\n",
|
|
"\n",
|
|
"preprocessed_data = dp_toolkit.filter_out_company_clients(preprocessed_data)\n",
|
|
"preprocessed_data = dp_toolkit.filter_out_long_stays(preprocessed_data)\n",
|
|
"\n",
|
|
"preprocessed_data = dp_toolkit.aggregate_group_reservations(preprocessed_data)\n",
|
|
"\n",
|
|
"preprocessed_data = dp_toolkit.add_night_price(preprocessed_data) # Code this method (remember that there can be many rooms)\n",
|
|
"\n",
|
|
"preprocessed_data = preprocessed_data.reset_index(drop=True)\n",
|
|
"\n",
|
|
"assert preprocessed_data.iloc[1]['length_of_stay'] == 3\n",
|
|
"assert preprocessed_data.iloc[2]['length_of_stay'] == 2\n",
|
|
"assert preprocessed_data.iloc[3]['length_of_stay'] == 7\n",
|
|
"\n",
|
|
"assert preprocessed_data.iloc[0]['night_price'] == 330.76\n",
|
|
"assert preprocessed_data.iloc[1]['night_price'] == 231.13\n",
|
|
"assert preprocessed_data.iloc[2]['night_price'] == 183.40\n",
|
|
"\n",
|
|
"display(HTML(preprocessed_data.head(15).to_html()))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "coupled-river",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Bucket important features to reduce the offer space size\n",
|
|
"\n",
|
|
"Without this step every pair (user_id, item_id) would have at most a single interaction. The base item space has around $2^{25} \\sim 3.3 \\text{mln}$ elements. Therefore, values for selected features are aggregated into buckets:\n",
|
|
"\n",
|
|
"```python\n",
|
|
"column_values_dict = {\n",
|
|
" 'term': ['WinterVacation', 'Easter', 'OffSeason', 'HighSeason', 'LowSeason', 'MayLongWeekend', 'NewYear', 'Christmas'],\n",
|
|
" 'length_of_stay_bucket': ['[0-1]', '[2-3]', '[4-7]', '[8-inf]'],\n",
|
|
" 'rate_plan': ['Standard', 'Nonref'],\n",
|
|
" 'room_segment': ['[0-160]', '[160-260]', '[260-360]', '[360-500]', '[500-900]'],\n",
|
|
" 'n_people_bucket': ['[1-1]', '[2-2]', '[3-4]', '[5-inf]'],\n",
|
|
" 'weekend_stay': ['True', 'False']\n",
|
|
"}\n",
|
|
"```\n",
|
|
"\n",
|
|
"Explanation:\n",
|
|
" - term - the term of the arrival date,\n",
|
|
" - length_of_stay_bucket - aggregated length of stay,\n",
|
|
" - rate_plan - rate plan which distinguishes if a given booking was refundable or nonrefundable (in reality rate plans are much more complex, they define prices for all rooms for every date, they include features like free breakfast, wine in the room etc.),\n",
|
|
" - room_segment - for every room its average price is calculated, then every room assigned to an appropriate price range, which is a proxy for room quality,\n",
|
|
" - n_people_bucket - aggregated number of people in a reservation,\n",
|
|
" - weekend_stay - indicates if the stay encompassed a weekend.\n",
|
|
"\n",
|
|
"The buckets are chosen based on expert knowledge of people working in the hotel industry for many years. Alternatively, clustering techniques could be used, but on a relatively small dataset expert methods are significantly better.\n",
|
|
"\n",
|
|
"The above aggregations reduce the number of possible items to $8 * 4 * 2 * 5 * 4 * 2 = 2560$.\n",
|
|
"\n",
|
|
"### The recommenders will be trained and evaluated on such aggregated data. To get a proper offer for a user one would have to decode those buckets into specific values, but this is a much easier task and can be achieved based on simple rules.\n",
|
|
"\n",
|
|
"<span style=\"color:red\"><font size=\"4\">**Task:**</font></span><br> \n",
|
|
"In the file data_preprocessing/data_preprocessing_toolkit write code for the map_night_price_to_room_segment_buckets method. You must calculate average of night prices for every **room_group_id** and map those prices to buckets (you can apply the map_value_to_bucket method which is available in the data_preprocessing_toolkit, the buckets are available under self.room_segment_buckets). The new column should be named 'room_segment'. You have to pass all assertions."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "interracial-rendering",
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"preprocessed_data = dp_toolkit.map_date_to_term_datasets(preprocessed_data)\n",
|
|
"preprocessed_data = dp_toolkit.map_length_of_stay_to_nights_buckets(preprocessed_data)\n",
|
|
"preprocessed_data = dp_toolkit.map_night_price_to_room_segment_buckets(preprocessed_data) # Code this method\n",
|
|
"preprocessed_data = dp_toolkit.map_npeople_to_npeople_buckets(preprocessed_data)\n",
|
|
"\n",
|
|
"assert preprocessed_data.iloc[0]['room_segment'] == '[260-360]'\n",
|
|
"assert preprocessed_data.iloc[1]['room_segment'] == '[160-260]'\n",
|
|
"assert preprocessed_data.iloc[4]['room_segment'] == '[0-160]'\n",
|
|
"\n",
|
|
"preprocessed_data = dp_toolkit.map_item_to_item_id(preprocessed_data)\n",
|
|
"\n",
|
|
"preprocessed_data.to_csv(os.path.join(data_path, \"hotel_data_preprocessed.csv\"))\n",
|
|
"\n",
|
|
"display(HTML(preprocessed_data.head(15).to_html()))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "offshore-biography",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Base statistics"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "acknowledged-crime",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"print(\"Number of users: {}\".format(len(preprocessed_data['user_id'].unique())))\n",
|
|
"print()\n",
|
|
"print(\"Number of items: {}\".format(len(preprocessed_data['item_id'].unique())))\n",
|
|
"print()\n",
|
|
"print(\"Number of interactions: {}\".format(len(preprocessed_data)))\n",
|
|
"print()\n",
|
|
"\n",
|
|
"n_user = preprocessed_data.loc[:, ['user_id', 'item_id']].groupby('item_id').count().sort_values(by='user_id', ascending=False)\n",
|
|
"n_user = n_user.rename(columns={'user_id': 'n_users'})\n",
|
|
"display(HTML(n_user.head(10).to_html()))\n",
|
|
"\n",
|
|
"n_item = preprocessed_data.loc[:, ['user_id', 'item_id']].groupby('user_id').count().sort_values(by='item_id', ascending=False)\n",
|
|
"n_item = n_item.rename(columns={'item_id': 'n_items'})\n",
|
|
"display(HTML(n_item.head(10).to_html()))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "blessed-knitting",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Prepare the dataset for recommenders\n",
|
|
"\n",
|
|
"One could consider many features describing each interaction but from the business perspective term, length_of_stay_bucket, room_segment, weekend_stay are the most important."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "victorian-bottom",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"item_features = ['term', 'length_of_stay_bucket', 'rate_plan', 'room_segment', 'n_people_bucket', 'weekend_stay']\n",
|
|
"\n",
|
|
"interactions_df = preprocessed_data.loc[\n",
|
|
" :, ['user_id', 'item_id'] + item_features]\n",
|
|
"\n",
|
|
"column_values_dict = {\n",
|
|
" 'term': ['WinterVacation', 'Easter', 'OffSeason', 'HighSeason', 'LowSeason', 'MayLongWeekend', 'NewYear', 'Christmas'],\n",
|
|
" 'length_of_stay_bucket': ['[0-1]', '[2-3]', '[4-7]', '[8-inf]'],\n",
|
|
" 'rate_plan': ['Standard', 'Nonref'],\n",
|
|
" 'room_segment': ['[0-160]', '[160-260]', '[260-360]', '[360-500]', '[500-900]'],\n",
|
|
" 'n_people_bucket': ['[1-1]', '[2-2]', '[3-4]', '[5-inf]'],\n",
|
|
" 'weekend_stay': ['True', 'False']\n",
|
|
"}\n",
|
|
"\n",
|
|
"interactions_df.loc[:, 'term'] = pd.Categorical(\n",
|
|
" interactions_df['term'], categories=column_values_dict['term'])\n",
|
|
"interactions_df.loc[:, 'length_of_stay_bucket'] = pd.Categorical(\n",
|
|
" interactions_df['length_of_stay_bucket'], categories=column_values_dict['length_of_stay_bucket'])\n",
|
|
"interactions_df.loc[:, 'rate_plan'] = pd.Categorical(\n",
|
|
" interactions_df['rate_plan'], categories=column_values_dict['rate_plan'])\n",
|
|
"interactions_df.loc[:, 'room_segment'] = pd.Categorical(\n",
|
|
" interactions_df['room_segment'], categories=column_values_dict['room_segment'])\n",
|
|
"interactions_df.loc[:, 'n_people_bucket'] = pd.Categorical(\n",
|
|
" interactions_df['n_people_bucket'], categories=column_values_dict['n_people_bucket'])\n",
|
|
"interactions_df.loc[:, 'weekend_stay'] = pd.Categorical(\n",
|
|
" interactions_df['weekend_stay'], categories=column_values_dict['weekend_stay'])\n",
|
|
"\n",
|
|
"interactions_df.to_csv(os.path.join(data_path, \"hotel_data_interactions_df.csv\"))\n",
|
|
"\n",
|
|
"display(HTML(interactions_df.head(15).to_html()))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "incredible-feeling",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.6.9"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|