s416178/warsztaty-B
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
dad90fbfa1
warsztaty-B/.ipynb_checkpoints/P0. Data preparation-checkpoint.ipynb

683 lines
71 KiB
Plaintext
Raw Normal View History

Solutions
2020-06-13 15:34:33 +02:00
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Building train and test sets"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"# if you don't have some library installed try using pip or pip3 to install it - you can do it from the notebook\n",
"# example: !pip install tqdm\n",
"# also on labs it's better to use python3 kernel - ipython3 notebook\n",
"\n",
"import pandas as pd\n",
"import numpy as np\n",
"import scipy.sparse as sparse\n",
"import time\n",
"import random\n",
"import evaluation_measures as ev\n",
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
"\n",
"# df = pd.DataFrame(np.loadtxt( './Datasets/ml-1m.dat',delimiter='::'))\n",
"df=pd.read_csv('./Datasets/ml-100k/u.data',delimiter='\\t', header=None)\n",
"df.columns=['user', 'item', 'rating', 'timestamp']\n",
"\n",
"from sklearn.model_selection import train_test_split\n",
"\n",
"train, test = train_test_split(df, test_size=0.2, random_state=30)\n",
"\n",
"train.to_csv('./Datasets/ml-100k/train.csv', sep='\\t', header=None, index=False)\n",
"test.to_csv('./Datasets/ml-100k/test.csv', sep='\\t', header=None, index=False)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Interactions properties"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### How data looks like?"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>user</th>\n",
" <th>item</th>\n",
" <th>rating</th>\n",
" <th>timestamp</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>196</td>\n",
" <td>242</td>\n",
" <td>3</td>\n",
" <td>881250949</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>186</td>\n",
" <td>302</td>\n",
" <td>3</td>\n",
" <td>891717742</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>22</td>\n",
" <td>377</td>\n",
" <td>1</td>\n",
" <td>878887116</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>244</td>\n",
" <td>51</td>\n",
" <td>2</td>\n",
" <td>880606923</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>166</td>\n",
" <td>346</td>\n",
" <td>1</td>\n",
" <td>886397596</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" user item rating timestamp\n",
"0 196 242 3 881250949\n",
"1 186 302 3 891717742\n",
"2 22 377 1 878887116\n",
"3 244 51 2 880606923\n",
"4 166 346 1 886397596"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df[:5]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Sample properties"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"We have 943 users, 1682 items and 100000 ratings.\n",
"\n",
"Average number of ratings per user is 106.04. \n",
"\n",
"Average number of ratings per item is 59.453.\n",
"\n",
"Data sparsity (% of missing entries) is 6.3047%.\n"
]
}
],
"source": [
"users, items, ratings=len(set(df['user'])), len(set(df['item'])), len(df)\n",
"\n",
"print('We have {} users, {} items and {} ratings.\\n'.format(users, items, ratings))\n",
"\n",
"print('Average number of ratings per user is {}. \\n'.format(round(ratings/users,2)))\n",
"print('Average number of ratings per item is {}.\\n'.format(round(ratings/items,4)))\n",
"print('Data sparsity (% of missing entries) is {}%.'.format(round(100*ratings/(users*items),4)))"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA6UAAAHvCAYAAACsfXllAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdeZgcZbn+8fvJhiAqAQImARxElrCESabRIIpRDoJkEUQk/ARBNEGUo4B4jEZIWNQgcIToEZMIh+ACeuAAWRBZBIEgwkwIORijEBggCyHshIQl8Pz+qJpJ11RPTyczU29N6vu5rr6mu6q6+u5napJ+uqreMncXAAAAAAAh9AodAAAAAABQXDSlAAAAAIBgaEoBAAAAAMHQlAIAAAAAgqEpBQAAAAAEQ1MKAAAAAAiGphQAMmJmU8zM49vI0Hl6GjPb1sx+aGYPmdmrZvZOXMuXQmfrLmZ2Vdk2Uxc6DwAA3YGmFECXK/sQ3XL7SA3PGVe2/JQMYqIHMbMdJTVJ+r6keklbS7KgoTaBmY2Mv5yYQpMJAECkT+gAAArhR5IOCR0CPdokSXXx/fmSfiNplSSX9FagTJtipKTJ8f27JDWHCgIAQF7QlALIwqfM7N/c/fbQQdBjHRH/fFHSp919bcgwWXH3kySdFDgGAADdisN3AXSn8sbhx8FSYHOwc/zzn0VpSAEAKAqaUgDd6WlJN8T3S2b2uZBh0KP1i3++ETQFAADocjSlALrbDyS9E9+/wMx6b+qKygZCuquzy5rZXS3LxI97mdnJ8fRnzew1M/s/M5tkZu9p89z3m9n5ZrbIzF4xs5fN7G4zO3YT3tMRZnaTmS0zszfin9eY2YEbsY7t45z3mNkzZvamma2OH/9H2/wVnt8c16I5fvwuM/ummd1rZqviUW7v2tj3Vrb+rczsDDO7M873Rlzje83se2b2vnae1zpacdnkT1QYSGvkRuY5qey5J8XTSmb2KzN7LP7dJ9ZrkY/Ho//+2cxWxO/jNTN7wsyuNbMxZlZx8KWW96IN55NK0p0V3stdbZ5XdfTdeOCkxABhZraLmV1iZkvifC+Z2X1m9nUzq+m0HTM7yszmxb//1+Nt5DcWD1pWqYbtrOezZvYHM3vczNbG61puZg+b2a/N7Itm1r+WTO2sP1E3MxtgZufFf7svx3+fTWY20cy23Ij1Hh7X/lGLRnpea2ZL42kf6+C5G7191Zip5pGYa1nWor/zr5vZbWa2Mt6e18S/6wfM7DIz+4yZ9e3gtQ40s8vNbHG8rb1uZk+Z2e/NbFQHz620/e5pZpea2T/i31/VbQzAZsLduXHjxq1Lb4oGn3FJS+LHV5VNO6md54wrW2ZKB+u9ayMyVFxW0SAzLctsLen2ssdtbwsk9Y+fd6CkZ6sse3GVTFPKlhsp6b+qrOdtSZNreJ8nSXqlynpc0jOSDqyyjuZ4uWZJu0p6pMI6Oqx5O+seIWl5B/meU3SeaLV6VbuN3MhMJ5Vvj5ImSlpfbb2S/rvGLH+U9N5OvJe72jzvqrJ5dRXWO7Js/hRJhys677a99d8qaYsqtekr6Q9Vnr9e0rfb1rDCeraUNLfG93x6F/xbc5ekYZKWVXmdRyXt2sH6Bki6o4bMv5LUt6u2rxrfa9VtYSO3m93ietTy+6lv5zXeLemaGp4/V9J72llH2+33S4pO+2i7jtQ2xo0bt83rxkBHALIwWdJxig7BnGJmv3P3NwNnKvffikYHnq/oA/kzkj4g6Rvxz2GSLjWzyZL+pOh9/ErSvZLelPRxSeMVDR73bTP7k7vf1sFrfkvSkYoasl9JWiRpK0VNxdGKjmSZYmbPu/vPK63AzL4l6dL44RuSrpd0j6TnJW0br+uzknaUdLuZHeDui6tk2kLS/0raJ35v10taoeiD+o4dvJ9K+eol/VlRgyJJD0n6naSnJL1f0hckHSRpO0lzzezT7n5X2SqulbQwvt9yGPjfFe19L/fIxmYr8wVJn5H0sqRZii4787ak/eNpLbZUVOO/SHpA0lJJrymqzR6STtCGml+t6HdbruW9jJPUskf97ArZn+vEe6mX9B1Fl8qZLumvceaSpK8paiIOVTSS8TntrGOGpGPi+68ram7+qqgmJUlfkXSxpOs6yPIjSS17yVYqGi3575LWKPoS6EOKvuA5uPa3V9X7FG27gxU13jdKekHSnnHmXeLXvMPM6t39lbYrMLNtFb3X3eJJiyX9j6R/KTraYx9FTeZO8Tr7qONBqGrdvjIT783/H0X1kKLt8jpJjysaybq/pCGSPqlom6q0ji0UfZE3Ip70lKIG9e+KtrkPKWow91S0HdxoZoe6+zsVVtfiIEXb5tuSrlD07/Hr8Tqe2YS3CqAnCd0Vc+PGbfO7acO320vKpk0rm/7NCs8JuafUJX2/wjIDtGEv33pFH96elTS0wrInlK3r5nZec0qb13xE0g4VljtS0YdDV9T47FJhmYayZZZI2r2d1xylqHF2SX9rZ5nmNrnO6IJtoJeSe1wvldSrwnJnly3zlKR3dfZ3X0O2k9q8339IGtTBcz4uaZsq89+t5B7GT9SwDYysIetVZcvXVZg/ss17ebLStiDpw2XbywuqsLdU0RczLetZLWnfCsvUVdheTmqzTG9JL8Xzmitt423+xvbqxO+y7R61b1RYZmtJd5Yt87N21nVD2TI/aGd73VrRF1Mtyx3eFdtXje+16rZQ67KKvlxomTdHUu8q69lb0nYVpv+0bB2XS+pXYZm+ihrxluW+VsP2u1LS3p2tFTdu3HrejXNKAWTlAkUNliRNMrOtQ4Zp40/u/qO2E919taSWvZS9Fe3dOM3dF1VY9teKDoeTpENqOHdvvaRj3f3ZCuu6UdIl8cOtJJ1a4fmTFe2peUPSaHd/tMIycvd5kqbGDz9sZh/tINcN7v7TDpapxWhFe5Yk6X5FjW5qL4m7ny9pXvxwZ0nHd8FrbwyXNM7dV1RdyP0ed3+pyvzXFO09a9nGT+i6iBvl+Erbgrs/IOn38cP+iprUts4ou3+au6f2QLt7szreOzhA0Z5LSbqp0jZetr7V7r6kg/XV6lp3/68Kr7FG0ZdeLXtHv2Jm25QvY2bDtWHv9pXufkE722vLulr2cp7ZQaaatq+Mfajs/pXu/nZ7C7r7Ynd/vnyamQ2U9PX44R3ufqpXOPLF3d+S9FVFe2CljmslSad49aM5AGymaEoBZCL+YNpyqOkOkk4PGKetiofHxuaX3V+l6oct3hv/7KcNhwC250/u/vcq8y9VdBibJB1VPiMeGKbl0Mib3P2xDl7rN2X3P93Bsj/rYH6tykdavsjdvcqyU8vuZz1C8z3u/nBXrMjdX5X0f/HDj3TFOjfSQ+5+T5X5fy67v3f5DDN7lzZsGysVHd5ZkUeHWKe+mCmzrr3X6WaXtDfD3Vdpw9/BlooOsy5X/iXCxdVexN1flHRz/PDg+FDW9nTZ9tWFyi+ptE+7S7XvC9owGna7NZdaG9OWL0N272CApicV7bkFUECcUwogSxcp2uu3raSzzOwX7v5C4EyS9Lcq81aV3W+qtPeknWU7GlH0jmoz3f0ZM/uHpH0l7WFm73P3lr0zB2nDl4qvm1nb8xfbKh89c0iV5d5WdE5dV2jZE+eSOjq/9j5tONcw62auWhOXEDcfX1B0nu7+is6z3VrROZxt7dQl6TbO/R3MX152v+32ub82bCd/6WA7l6LD34dWmuHuL5vZA4q2gX8zsxsUfdlxT9ykdIeXFZ2vWc2ftWEP3wGKzvNt8fH455uS9jSzPTtY1xZlPz+o6BDdSmrevjJ0r6IvDraUNDn+kmtWpSNA2vHxsvs71PDvT/m2NkTRId0Vc3Xw5RWAzRhNKYDMxB9WL5R0oaLD+yZK+o+wqSRFAwO1p/y6mNWWa7vsuzpYtqO9my3L7Kuo6Xm/NhwyWFe2zJfiW62qNcvPu/vrG7GuagbGP5+J9yC2y93fMbOlihqjbc2sX6XDAbvJ8o4XkcxsP0UDP+1e43rfu8mJNl1HgyRV2z4Hld1/XB3raJlvKPri5b2KDos9UtJrZvY
"text/plain": [
"<Figure size 1152x576 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"items_per_user=df.groupby(['user']).count()['rating']\n",
"\n",
"plt.figure(figsize=(16,8))\n",
"plt.hist(items_per_user, bins=100)\n",
"\n",
"# Let's add median\n",
"t=items_per_user.median()\n",
"plt.axvline(t, color='k', linestyle='dashed', linewidth=1)\n",
"plt.text(t*1.1, plt.ylim()[1]*0.9, 'Median: {:.0f}'.format(t))\n",
"\n",
"# Let's add also some percentiles\n",
"t=items_per_user.quantile(0.25)\n",
"plt.axvline(t, color='k', linestyle='dashed', linewidth=1)\n",
"plt.text(t*1.1, plt.ylim()[1]*0.95, '25% quantile: {:.0f}'.format(t))\n",
"\n",
"t=items_per_user.quantile(0.75)\n",
"plt.axvline(t, color='k', linestyle='dashed', linewidth=1)\n",
"plt.text(t*1.05, plt.ylim()[1]*0.95, '75% quantile: {:.0f}'.format(t))\n",
"\n",
"plt.title('Number of ratings per user', fontsize=30)\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA6UAAAHvCAYAAACsfXllAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdeZwU9Z3/8feHKx54oYgcKi6iMSiOzGgwJoaYdSEKeKAJbjyILkOiJBpzaOIPATVZ3ejGM67DqmDW1bgaEw4DnsQjXjM4GOMJiuEYEVEUBCXo5/dH1UAz9HQ3UHyrCl/Px6Mf3V317fp8+9PF0J/+Vn3L3F0AAAAAAKShTdodAAAAAAB8dlGUAgAAAABSQ1EKAAAAAEgNRSkAAAAAIDUUpQAAAACA1FCUAgAAAABSQ1EKAAGY2Tgz8/g2IO3+5I2ZdTKzX5jZc2a23Mw+jXO5LO2+bSlmNrFgn+mZdn+QHjMbULAvjEu7PwCQtHZpdwDA1sXMWl78uL+7P13mNcMl3RE/He/u47ZE35BPZtZF0lOSeqbclc0S/xgxIH460d3npdYZbHXM7DxJO0ta5u5Xp90fANgYFKUAtrRfSvp62p1Arl2kdQXpE5L+R9JiSS7pHyn1aVMMkDQ2fjxT0ry0OoKt0nmS9pb0piSKUgC5QlEKYEs7ysz+2d0fTLsjyK1j4vv3JP2Lu69MszOhuPsISSNS7gYywN1nSrK0+wEAWwrnlALYUgoLh39PrRfYGuwZ37/yWSlIAQD4LKEoBbClzJd0b/y4xsxOTLMzyLUO8f3HqfYCAABsERSlALak/yfp0/jxZWbWdlM3VDDz5MzNbWtmM5vbxM/bmNmZ8fK3zexDM/urmV1kZju0eO0eZnapmT1vZh+Y2ftm9qiZfWsT3tMxZvZHM1tgZh/H93eY2eEbsY3d4n4+ZmZvmdlqM1sSP/9py/4Xef28OBfz4ufbmNkPzOxxM1scz3I7c2PfW8H2tzOzH5rZI3H/Po5z/LiZ/czMdmrldWtnKy5Y/NWCz3aTZjI2sxEFrx0RL6sxs/82sznxZ7/edi3ylXj234fNbFH8Pj40szfM7E4zG2JmRQ+vbH4vWnc+qSQ9UuS9zGzxupKz7xabkdXM9jKzq8zs5bh/y8zsL2Z2tplVdMqOmZ1gZtPiz/+jeB/5HzP7Yms5bGU7x5nZXWb2upmtjLe10Mxmm9lvzezbZrZLJX1qZfvr5c3MOpvZJfG/3ffjf58NZnahmW27EdsdFOf+NYtmel5pZnPjZV8u89qN3r8q7FOrs+82/xtWdD6pJO1dZN9q9bMysw5mdpaZTTaz+fHntMyiv3FXFdv3Wrx+g/3UzIaZ2Z/ifyur4v3xV2bWucVrdzKzn8Sf03tmtsLMnjWz75oZ31GBzxJ358aNG7fEboomn3FJL8fPJxYsG9HKa4YXtBlXZrszN6IPRdsqmmSmuU1HSQ8WPG95myVpl/h1h0t6u0TbK0v0aVxBuwGSbiixnU8kja3gfY6Q9EGJ7biktyQdXmIb8+J28yTtI+mFItsom/NWtt1f0sIy/XtH0XmipfJV6jZgI/s0onB/lHShpDWltivp1gr78idJO27Ge5nZ4nUTC9b1LLLdAQXrx0kapOi829a2f7+kz5XITXtJd5V4/RpJP2qZwyLb2VbS1Arf83kJ/K2ZKekQSQtKxHlN0j5lttdZ0kMV9Pm/JbVPav+q8L2u91m38m+43K3YZ1Uj6fUyr/tY0qgSfSvcT3spmoistW3Nk7R3/Lr9Jc0p0fYuSbap+wc3btzydWOiIwBb2lhJpyg6BHOcmf2vu69OuU+FblU0O/ATir4EvaVoxOGc+P4QSVeb2VhJMxS9j/+W9Lik1ZK+ImmkoonjfmRmM9z9gTIxz5V0vKKC7L8lPS9pO0VFxTBFR7GMM7Ol7n59sQ2Y2blaN8Pmx5LukfSYpKWSOsXbOk5SF0kPmtmh7v5iiT59TtLvJfWJ39s9khYp+qLepcz7Kda/KkkPKypQJOk5Sf8r6e+S9pD0TUlHSNpV0lQz+xePJnNpdqekxvhx82Hgf1M0+l7ohY3tW4FvSvqGpPclTZLUoOgHgYPjZc22VZTjP0t6RtJcSR8qys1+kk7TupzfpuizLdT8XoZLah5RH1Ok7+9sxnupkvQTRZPh3CTpybjPNZK+K2l7SUcrmsn44la2USfp5PjxR4qKjScV5aRG0lmSrpR0d5m+/FLSsfHjJkVFyt8krVD0I9C+in7gObLyt1fSTor23e6KCu8/SHpXUdFzlqS94pgPmVmVu3/QcgNm1knRe+0VL3pR0v9JelXR0R59FBWZPeJttlP5Sagq3b82V62ivx91ivbJJfGylmYVPrHoiIwH49dKUUH+J0WnXmyj6DM6PV7/X2b2sbtPLNOXyyWdpOhv2v8omgl4D0V/Iw9U9Df1NjM7TtIDij6z38X9WC6pn6K/vdsr2hfvV/Q3EsDWLu2qmBs3blvXTet+5X65YNm1Bct/UOQ1aY6UuqSfF2nTWetG+dYoKireltS3SNvTCrZ1Xysxx7WI+YKk3Yu0O17RZU5cUeGzV5E21QVtXpbUu5WYxyoqnF3S0620mdeiXz9MYB9oo/VHXK+W1KZIuzEFbf4uaZvN/ewr6NuIFu/3JUndyrzmK5J2LrF+e60/wvjVCvaBARX0dWJB+55F1g9o8V7eLLYvSDqsYH95V0VGSxX9MNO8nSWSDizSpmeR/WVEizZtJS3TulGxDfbxFv/GPr8Zn2XLkbVzirTpKOmRgjbXtbKtewva/L9W9teOin6Yam43KIn9q8L3WvhZj2ulTfNnM6+C7e0Q/5tzRT8WfKOVdvvG+1Vzu93K7Kcu6b9a5k/RDzuzC9rUK5oM76gi2ztS0Q8BLunFzc0dN27c8nHjeH0AIVymqMCSpIvMrGOanWlhhrv/suVCd18iqXmUsq2i0Y3R7v58kba/VXR4oCR9vYJz99ZI+pa7v11kW3+QdFX8dDtJ3yvy+rGKRmo+ljTY3V8r0kbuPk3RyIUkHWZmXyrTr3vd/ddl2lRisKKRJUl6SlGh+2nLRu5+qaRp8dM9JZ2aQOyN4ZKGu/uiko3cH3P3ZSXWf6ho9Kx5Hz8tuS5ulFOL7Qvu/oyi0ShJ2kVRkdrSDwsej3b3DUag3X2eyo8OdlY0cilJfyy2jxdsb4m7v1xme5W6091vKBJjhaIfvZpHR88ys50L25hZP60b3b7F3S9rZX9t3lbzKOf5ZfpU0f6VkpFaN6v199z9T8UaufscSd+Jn26v4iOwhV5QtP+slz93X6V1f4uk6Ie1ce7+cJGYjyoaOZWkA8xsz5ZtAGx9KEoBbHHxF9PmQ013V3SR96woenhs7ImCx4tV+rDFx+P7Dlp3CGBrZrj730qsv1rRYX6SdELhinhimOZDI/8Yf2ks5X8KHv9LmbbXlVlfqcKZln/l7l6ibeEX1dAzND/m7rOT2JC7L5f01/jpF5PY5kZ6zt0fK7G+8Mv/FwpXmNk2WrdvNCk6bLUojw6x3uCHmQKrWouzhV3V2gp3X6x1/w62VXSYdaHCHxGuLBXE3d+TdF/89Egz+1yJ5ontX1tA83tuknR7qYZx4dhcWJf7G3KTu69pZV3h39NPFB1m3prHCx6H3I8ApIRzSgGE8itFo36dJP3YzH7j7u+m3CdJerrEusUFjxuKjZ600rbcjKIPlVrp7m+Z2UuKzsHaz8x2cvfm0ZkjtO4HxY/MrOX5iy21L3h8QIl2nyg6py4JzSNxrui8sVL+onXnGoYu5koVceuJi49vKjpP92BF59l2VHQOZ0s9EundxnmqzPqFBY9b7p8Ha91+8ucy+7kUHf7et9gKd3/fzJ5RtA/8s5ndq+jHjsfc/R9ltrup3ld0vmYpD0s6O358qKLzfJt9Jb5fLWl/M9u/zLY+V3D/T4oO0S2m4v0rJItmvG7+/JokDbXiE0cXWhHfl/obIlX+9/SVgr9p5dpu8gzNAPKDohRAEPGX1SskXaHo8L4LJf003V5JiiYGak3hdTFLtWvZdpsybcuNbja
"text/plain": [
"<Figure size 1152x576 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"items_per_user=df.groupby(['item']).count()['rating']\n",
"\n",
"plt.figure(figsize=(16,8))\n",
"plt.hist(items_per_user, bins=100)\n",
"\n",
"# Let's add median\n",
"t=items_per_user.median()\n",
"plt.axvline(t, color='k', linestyle='dashed', linewidth=1)\n",
"plt.text(t*1.1, plt.ylim()[1]*0.9, 'Median: {:.0f}'.format(t))\n",
"\n",
"# Let's add also some percentiles\n",
"t=items_per_user.quantile(0.25)\n",
"plt.axvline(t, color='k', linestyle='dashed', linewidth=1)\n",
"plt.text(t*1.1, plt.ylim()[1]*0.95, '25% quantile: {:.0f}'.format(t))\n",
"\n",
"t=items_per_user.quantile(0.75)\n",
"plt.axvline(t, color='k', linestyle='dashed', linewidth=1)\n",
"plt.text(t*1.05, plt.ylim()[1]*0.95, '75% quantile: {:.0f}'.format(t))\n",
"\n",
"plt.title('Number of ratings per item', fontsize=30)\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"rating\n",
"1 0.06110\n",
"2 0.11370\n",
"3 0.27145\n",
"4 0.34174\n",
"5 0.21201\n",
"Name: user, dtype: float64"
]
},
"execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.groupby(['rating']).count()['user']/len(df)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Item attributes"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
"genres = pd.read_csv('./Datasets/ml-100k/u.genre', sep='|', header=None,\n",
" encoding='latin-1')\n",
"genres=dict(zip(genres[1], genres[0]))"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{0: 'unknown',\n",
" 1: 'Action',\n",
" 2: 'Adventure',\n",
" 3: 'Animation',\n",
" 4: \"Children's\",\n",
" 5: 'Comedy',\n",
" 6: 'Crime',\n",
" 7: 'Documentary',\n",
" 8: 'Drama',\n",
" 9: 'Fantasy',\n",
" 10: 'Film-Noir',\n",
" 11: 'Horror',\n",
" 12: 'Musical',\n",
" 13: 'Mystery',\n",
" 14: 'Romance',\n",
" 15: 'Sci-Fi',\n",
" 16: 'Thriller',\n",
" 17: 'War',\n",
" 18: 'Western'}"
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"genres"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [],
"source": [
"movies = pd.read_csv('./Datasets/ml-100k/u.item', sep='|', encoding='latin-1', header=None)"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>0</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" <th>3</th>\n",
" <th>4</th>\n",
" <th>5</th>\n",
" <th>6</th>\n",
" <th>7</th>\n",
" <th>8</th>\n",
" <th>9</th>\n",
" <th>...</th>\n",
" <th>14</th>\n",
" <th>15</th>\n",
" <th>16</th>\n",
" <th>17</th>\n",
" <th>18</th>\n",
" <th>19</th>\n",
" <th>20</th>\n",
" <th>21</th>\n",
" <th>22</th>\n",
" <th>23</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1</td>\n",
" <td>Toy Story (1995)</td>\n",
" <td>01-Jan-1995</td>\n",
" <td>NaN</td>\n",
" <td>http://us.imdb.com/M/title-exact?Toy%20Story%2...</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>...</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2</td>\n",
" <td>GoldenEye (1995)</td>\n",
" <td>01-Jan-1995</td>\n",
" <td>NaN</td>\n",
" <td>http://us.imdb.com/M/title-exact?GoldenEye%20(...</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>...</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>3</td>\n",
" <td>Four Rooms (1995)</td>\n",
" <td>01-Jan-1995</td>\n",
" <td>NaN</td>\n",
" <td>http://us.imdb.com/M/title-exact?Four%20Rooms%...</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>...</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>3 rows × 24 columns</p>\n",
"</div>"
],
"text/plain": [
" 0 1 2 3 \\\n",
"0 1 Toy Story (1995) 01-Jan-1995 NaN \n",
"1 2 GoldenEye (1995) 01-Jan-1995 NaN \n",
"2 3 Four Rooms (1995) 01-Jan-1995 NaN \n",
"\n",
" 4 5 6 7 8 9 ... \\\n",
"0 http://us.imdb.com/M/title-exact?Toy%20Story%2... 0 0 0 1 1 ... \n",
"1 http://us.imdb.com/M/title-exact?GoldenEye%20(... 0 1 1 0 0 ... \n",
"2 http://us.imdb.com/M/title-exact?Four%20Rooms%... 0 0 0 0 0 ... \n",
"\n",
" 14 15 16 17 18 19 20 21 22 23 \n",
"0 0 0 0 0 0 0 0 0 0 0 \n",
"1 0 0 0 0 0 0 0 1 0 0 \n",
"2 0 0 0 0 0 0 0 1 0 0 \n",
"\n",
"[3 rows x 24 columns]"
]
},
"execution_count": 26,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"movies[:3]"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [],
"source": [
"for i in range(19):\n",
" movies[i+5]=movies[i+5].apply(lambda x: genres[i] if x==1 else '')"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [],
"source": [
"movies['genre']=movies.iloc[:, 5:].apply(lambda x: ', '.join(x[x!='']), axis = 1)"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [],
"source": [
"movies=movies[[0,1,'genre']]\n",
"movies.columns=['id', 'title', 'genres']"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>id</th>\n",
" <th>title</th>\n",
" <th>genres</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1</td>\n",
" <td>Toy Story (1995)</td>\n",
" <td>Animation, Children's, Comedy</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2</td>\n",
" <td>GoldenEye (1995)</td>\n",
" <td>Action, Adventure, Thriller</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>3</td>\n",
" <td>Four Rooms (1995)</td>\n",
" <td>Thriller</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>4</td>\n",
" <td>Get Shorty (1995)</td>\n",
" <td>Action, Comedy, Drama</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>5</td>\n",
" <td>Copycat (1995)</td>\n",
" <td>Crime, Drama, Thriller</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" id title genres\n",
"0 1 Toy Story (1995) Animation, Children's, Comedy\n",
"1 2 GoldenEye (1995) Action, Adventure, Thriller\n",
"2 3 Four Rooms (1995) Thriller\n",
"3 4 Get Shorty (1995) Action, Comedy, Drama\n",
"4 5 Copycat (1995) Crime, Drama, Thriller"
]
},
"execution_count": 30,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"movies.to_csv('./Datasets/ml-100k/movies.csv', index=False)\n",
"movies[:5]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Toy example"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"if not os.path.exists('./Datasets/toy-example/'):\n",
" os.mkdir('./Datasets/toy-example/')"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [],
"source": [
"toy_train=pd.DataFrame([[0,0,3,0], [0,10,4,0], [0,40,5,0], [0,70,4,0],\n",
" [10,10,1,0], [10,20,2,0], [10,30,3,0],\n",
" [20,30,5,0], [20,50,3,0], [20,60,4,0]])\n",
"toy_test=pd.DataFrame([[0,60,3,0],\n",
" [10,40,5,0],\n",
" [20,0,5,0], [20,20,4,0], [20,70,2,0]])\n",
"\n",
"toy_train.to_csv('./Datasets/toy-example/train.csv', sep='\\t', header=None, index=False)\n",
"toy_test.to_csv('./Datasets/toy-example/test.csv', sep='\\t', header=None, index=False)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.9"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Reference in New Issue Copy Permalink