{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "CNN_on_FashionMNIST.ipynb",
"provenance": [],
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"accelerator": "GPU"
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
""
]
},
{
"cell_type": "code",
"metadata": {
"id": "lKXHmN72oSr-"
},
"source": [
"from torchvision import datasets\n",
"import torch\n",
"data_folder = '/content/' # This can be any directory you want to download FMNIST to\n",
"fmnist = datasets.FashionMNIST(data_folder, download=True, train=True)"
],
"execution_count": 15,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "8quMVIspoXAc"
},
"source": [
"tr_images = fmnist.data\n",
"tr_targets = fmnist.targets"
],
"execution_count": 16,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "pCybp42UoYfD"
},
"source": [
"val_fmnist = datasets.FashionMNIST(data_folder, download=True, train=False)\n",
"val_images = val_fmnist.data\n",
"val_targets = val_fmnist.targets"
],
"execution_count": 17,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "_wf7B5v_oZpV"
},
"source": [
"import matplotlib.pyplot as plt\n",
"%matplotlib inline\n",
"import numpy as np\n",
"from torch.utils.data import Dataset, DataLoader\n",
"import torch\n",
"import torch.nn as nn\n",
"device = 'cuda' if torch.cuda.is_available() else 'cpu'"
],
"execution_count": 18,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "DeG0gLx4oavL"
},
"source": [
"class FMNISTDataset(Dataset):\n",
" def __init__(self, x, y):\n",
" x = x.float()/255\n",
" x = x.view(-1,1,28,28)\n",
" self.x, self.y = x, y \n",
" def __getitem__(self, ix):\n",
" x, y = self.x[ix], self.y[ix] \n",
" return x.to(device), y.to(device)\n",
" def __len__(self): \n",
" return len(self.x)\n",
"\n",
"from torch.optim import SGD, Adam\n",
"def get_model():\n",
" model = nn.Sequential(\n",
" nn.Conv2d(1, 64, kernel_size=3),\n",
" nn.MaxPool2d(2),\n",
" nn.ReLU(),\n",
" nn.Conv2d(64, 128, kernel_size=3),\n",
" nn.MaxPool2d(2),\n",
" nn.ReLU(),\n",
" nn.Flatten(),\n",
" nn.Linear(3200, 256),\n",
" nn.ReLU(),\n",
" nn.Linear(256, 10)\n",
" ).to(device)\n",
"\n",
" loss_fn = nn.CrossEntropyLoss()\n",
" optimizer = Adam(model.parameters(), lr=1e-3)\n",
" return model, loss_fn, optimizer\n",
"\n",
"def train_batch(x, y, model, opt, loss_fn):\n",
" prediction = model(x)\n",
" batch_loss = loss_fn(prediction, y)\n",
" batch_loss.backward()\n",
" optimizer.step()\n",
" optimizer.zero_grad()\n",
" return batch_loss.item()\n",
"\n",
"@torch.no_grad()\n",
"def accuracy(x, y, model):\n",
" model.eval()\n",
" prediction = model(x)\n",
" max_values, argmaxes = prediction.max(-1)\n",
" is_correct = argmaxes == y\n",
" return is_correct.cpu().numpy().tolist()\n"
],
"execution_count": 19,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "-VxMySqHoyUc"
},
"source": [
"def get_data(): \n",
" train = FMNISTDataset(tr_images, tr_targets) \n",
" trn_dl = DataLoader(train, batch_size=32, shuffle=True)\n",
" val = FMNISTDataset(val_images, val_targets) \n",
" val_dl = DataLoader(val, batch_size=len(val_images), shuffle=True)\n",
" return trn_dl, val_dl"
],
"execution_count": 20,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "MKIE_sjtpRP6"
},
"source": [
"@torch.no_grad()\n",
"def val_loss(x, y, model):\n",
" model.eval()\n",
" prediction = model(x)\n",
" val_loss = loss_fn(prediction, y)\n",
" return val_loss.item()"
],
"execution_count": 21,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "r2hKhLHQpSqx"
},
"source": [
"trn_dl, val_dl = get_data()\n",
"model, loss_fn, optimizer = get_model()"
],
"execution_count": 22,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "WQmmjw70pUe9",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "e8836519-77c7-4b75-f009-d7fd3cd03e14"
},
"source": [
"!pip install torch_summary\n",
"from torchsummary import summary\n",
"model, loss_fn, optimizer = get_model()\n",
"summary(model, torch.zeros(1,1,28,28));"
],
"execution_count": 23,
"outputs": [
{
"output_type": "stream",
"text": [
"Requirement already satisfied: torch_summary in /usr/local/lib/python3.6/dist-packages (1.4.3)\n",
"==========================================================================================\n",
"Layer (type:depth-idx) Output Shape Param #\n",
"==========================================================================================\n",
"├─Conv2d: 1-1 [-1, 64, 26, 26] 640\n",
"├─MaxPool2d: 1-2 [-1, 64, 13, 13] --\n",
"├─ReLU: 1-3 [-1, 64, 13, 13] --\n",
"├─Conv2d: 1-4 [-1, 128, 11, 11] 73,856\n",
"├─MaxPool2d: 1-5 [-1, 128, 5, 5] --\n",
"├─ReLU: 1-6 [-1, 128, 5, 5] --\n",
"├─Flatten: 1-7 [-1, 3200] --\n",
"├─Linear: 1-8 [-1, 256] 819,456\n",
"├─ReLU: 1-9 [-1, 256] --\n",
"├─Linear: 1-10 [-1, 10] 2,570\n",
"==========================================================================================\n",
"Total params: 896,522\n",
"Trainable params: 896,522\n",
"Non-trainable params: 0\n",
"Total mult-adds (M): 10.13\n",
"==========================================================================================\n",
"Input size (MB): 0.00\n",
"Forward/backward pass size (MB): 0.45\n",
"Params size (MB): 3.42\n",
"Estimated Total Size (MB): 3.87\n",
"==========================================================================================\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "97CcIWOBpXuw",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "8b16304b-81dd-4a4b-ea15-bab0508a8cc2"
},
"source": [
"train_losses, train_accuracies = [], []\n",
"val_losses, val_accuracies = [], []\n",
"for epoch in range(5):\n",
" print(epoch)\n",
" train_epoch_losses, train_epoch_accuracies = [], []\n",
" for ix, batch in enumerate(iter(trn_dl)):\n",
" x, y = batch\n",
" batch_loss = train_batch(x, y, model, optimizer, loss_fn)\n",
" train_epoch_losses.append(batch_loss) \n",
" train_epoch_loss = np.array(train_epoch_losses).mean()\n",
"\n",
" for ix, batch in enumerate(iter(trn_dl)):\n",
" x, y = batch\n",
" is_correct = accuracy(x, y, model)\n",
" train_epoch_accuracies.extend(is_correct)\n",
" train_epoch_accuracy = np.mean(train_epoch_accuracies)\n",
"\n",
" for ix, batch in enumerate(iter(val_dl)):\n",
" x, y = batch\n",
" val_is_correct = accuracy(x, y, model)\n",
" validation_loss = val_loss(x, y, model)\n",
" val_epoch_accuracy = np.mean(val_is_correct)\n",
"\n",
" train_losses.append(train_epoch_loss)\n",
" train_accuracies.append(train_epoch_accuracy)\n",
" val_losses.append(validation_loss)\n",
" val_accuracies.append(val_epoch_accuracy)"
],
"execution_count": 24,
"outputs": [
{
"output_type": "stream",
"text": [
"0\n",
"1\n",
"2\n",
"3\n",
"4\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "l9N0n1k0paJx",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 337
},
"outputId": "2b7a2e49-476c-45b5-f57c-9327eb98a662"
},
"source": [
"epochs = np.arange(5)+1\n",
"import matplotlib.ticker as mtick\n",
"import matplotlib.pyplot as plt\n",
"import matplotlib.ticker as mticker\n",
"%matplotlib inline\n",
"plt.subplot(211)\n",
"plt.plot(epochs, train_losses, 'bo', label='Training loss')\n",
"plt.plot(epochs, val_losses, 'r', label='Validation loss')\n",
"plt.gca().xaxis.set_major_locator(mticker.MultipleLocator(1))\n",
"plt.title('Training and validation loss with CNN')\n",
"plt.xlabel('Epochs')\n",
"plt.ylabel('Loss')\n",
"plt.legend()\n",
"plt.grid('off')\n",
"plt.show()\n",
"plt.subplot(212)\n",
"plt.plot(epochs, train_accuracies, 'bo', label='Training accuracy')\n",
"plt.plot(epochs, val_accuracies, 'r', label='Validation accuracy')\n",
"plt.gca().xaxis.set_major_locator(mticker.MultipleLocator(1))\n",
"plt.title('Training and validation accuracy with CNN')\n",
"plt.xlabel('Epochs')\n",
"plt.ylabel('Accuracy')\n",
"#plt.ylim(0.8,1)\n",
"plt.gca().set_yticklabels(['{:.0f}%'.format(x*100) for x in plt.gca().get_yticks()]) \n",
"plt.legend()\n",
"plt.grid('off')\n",
"plt.show()"
],
"execution_count": 25,
"outputs": [
{
"output_type": "display_data",
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAACgCAYAAAAB6WsAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3QV1b3A8e8vIZCEBJQA4RFIgIJWBBJIQI0i+LhaoaiID5qr5lJFaNUqvqhUoVp6763UZamPim8Um2u1ZalgtQrhIaI8RORZBQNGokIUSEwCIfzuHzMJh+Sck3OSHHKS8/usdVZm5uzZs2cnmd/M3jN7RFUxxhgTuaKauwDGGGOalwUCY4yJcBYIjDEmwlkgMMaYCGeBwBhjIpwFAmOMiXAWCFopEXlLRK5v6rTNSUQKROSCEOSrIvIjd/ovInJfIGkbsJ0cEXmnoeX0k+8oESls6nwbQ0R6i0ipiET7SdPgujRNywJBGHH/cao/R0Wk3GM+J5i8VPUnqvpCU6dt7VR1iqo+2Nh8RCTNPdC18ch7gar+R2PzbglUdbeqJqhqFYCI5IvIDY3JU0QGiMjfRGSfiBwQkY0iMk1Eoj3qe3GtdV4SkVnu9Cg3zeO10qwUkdzGlK2ls0AQRtx/nARVTQB2Az/1WLagOp3nwcWYSCAi/YAPgS+BQaraEbgSyAQSPZKOEJGz/GT1A3CtiKSFqKgtkgWCFqD60l9E7hGRr4HnRORkEXlTRPaKyPfudIrHOjVnYCKS6571zHHTfiEiP2lg2j4islxESkTkXRF5TERe8lHuQMr4oIi87+b3joh09vj+WhHZJSLFIjLDT/2MEJGvPZshRORyEdnoTg8XkQ9EZL+IFInIoyLS1kdez4vI7zzm73LX2SMik2qlHSMiH4vIQRH5svrM07Xc/bnfvaI7s7puPdY/S0TWuGe3azwPYPXVjT8i8mN3/f0isllExnl8d4mIbHHz/EpE7nSXd3Z/P/tF5DsRWSEidY4PIvJbEfmzOx0jIj+IyEPufJyIVIhIJ48z9DYiMhs4B3jUrYtHPbK8QEQ+c7f7mIiIj936LbBKVaepahGAqm5X1Z+p6n6PdH8AZvupnv3A88DMeqoxolggaDm6AZ2AVGAyzu/uOXe+N1AOPOpzbRgBbAc64/yzPOPnn85f2peBj4AkYBZwrZ9tBlLGnwH/BXQF2gLVB6bTgCfc/Hu420vBC1X9EOdM77xa+b7sTlcBt7v7cyZwPvALP+XGLcPFbnkuBPoDtfsnfgCuA04CxgBTReQy97uR7s+T3Cu6D2rl3QlYBMx19+1hYJGIJNXahzp1U0+ZY4A3gHfc9W4BFojIKW6SZ4CbVDUROB1Y4i6/AygEugDJwL2At/FnlgGj3Oks4GuPfT0T2K6q33muoKozgBXAzW5d3Ozx9Vg3n8HAVcBFPnbtAuBVf/vuehwYIP77kmYDV3jUScSzQNByHAVmquohVS1X1WJVfU1Vy1S1BOeP+1w/6+9S1afcNtsXgO44//ABpxWR3jj/tPer6mFVXQm87muDAZbxOVX9t6qWA68A6e7yCcCbqrpcVQ8B97l14MtfgYkAIpIIXOIuQ1XXqepqVT2iqgXAk17K4c1Vbvk2qeoPOIHPc//yVfVTVT2qqhvd7QWSLziB4zNVfdEt11+BbcBPPdL4qht/zgASgP9xf0dLgDdx6waoBE4TkQ6q+r2qrvdY3h1IVdVKVV2h3gci+wDo7waskTiBpaeIJLj7vizA/a/2P6q6X1V3A0v97GMSUBRAfuU4f2e/85VAVb8G/gI8EGRZWy0LBC3HXlWtqJ4RkXgRedJtOjmI0xRxkvi+S+Pr6glVLXMnE4JM2wP4zmMZOG22XgVYxq89pss8ytTDM2/3QFzsa1s4Z//jRaQdMB5Yr6q73HIMcJs9vnbL8Xucq4P6HFcGYFet/RshIkvdpq8DwJQA863Oe1etZbuAnh7zvuqm3jKrqmfQ9Mz3CpwguUtElonIme7yh4DPgXdEZKeITPeWuRuU1uIc9EfiHPhXAdk0LBAEuo/FOIEqEE/jnLj81E+a/wUuEpEhAebZqlkgaDlqn53dAZwCjFDVDhy7PPfV3NMUioBOIhLvsayXn/SNKWORZ97uNpN8JVbVLTgHvJ9wfLMQOE1M24D+bjnubUgZcJq3PL2Mc0XUy+28/ItHvvUN67sHp8nMU2/gqwDKVV++vWq179fkq6prVPVSnGajhThXGqhqiareoap9gXHANBE538c2luE0w2UAa9z5i4DhHOsbqa2xwxy/ixPE6qWqh3H6FB7Ex+9ZVYuBR9w0Ec8CQcuViHMZvN9tbw5555d7hr0WmCUibd2zSX9nXY0p46vAWBE52+3YfYD6/15fBn6FE3D+VqscB4FSETkVmBpgGV4BckXkNDcQ1S5/Is4VUoWIDMcJQNX24jRl9fWR92KctuyfuR2qVwOn4TTjNMaHOGfWd7uduaNwfkd57u8sR0Q6qmolTp0cBRCRsSLyI7cv6ABOv4qvprhlOH0jW9yDbj5wA/CFqu71sc43+K6LQMwEzhKRh0Skm1vmH4lze+hJXtK/CMQCF/vJ82HgLODHjShXq2CBoOV6BIgD9gGrgX+eoO3m4HQKFuO0w/4fcMhH2gaXUVU3A7/EObgXAd/jdGb6U91Gv0RV93ksvxPnIF0CPOWWOZAyvOXuwxKcZpMltZL8AnhAREqA+3HPrt11y3Daqt9374g5o1bexTgdpXfg1OXdwNha5Q6ae2D+Kc6V0T6cztPrVHWbm+RaoMBtIpuC8/sEpzP8XaAUpx/gcVVd6mMzq3B+r9Vn/1uACnxfDQD8CZggzt1jcxuwXztw/u7SgM1uU9xrOCcmJV7SV+H8Tjr5yfMgzs0QPtNECrEX05jGEJH/A7apqt2OZ0wLZVcEJigikiUi/UQkyr298lKctmZjTAtlT6iaYHUD/o7TcVsITFXVj5u3SMaYxrCmIWOMiXDWNGSMMRHOAoExxkS4FtdH0LlzZ01LS2vQuj/88APt27dv2gK1YlZfwbH6Cp7VWXAaU1/r1q3bp6pdvH3X4gJBWloaa9euDWqdBQtgxgzYvVvp3VuYPRtyghrdPzLl5+czatSo5i5Gi2H1FTyrs+A0pr5EpPaQJjVaXCAI1oIFMHkylJUBCLt2OfNgwcAYYyAC+ghmzKgOAseUlTnLjTHGREAg2L07uOXGGBNpWn3TUO/esMtLy1jv2uNIGmN8qqyspLCwkIqKivoTB6Fjx45s3bq1SfNszQKpr9jYWFJSUoiJiQk431YfCGbP9uwjcMTHO8uNMYEpLCwkMTGRtLQ0fL/YLnglJSUkJibWn9AA9deXqlJcXExhYSF9+vQJON9W3zSUkwPz5kFqKogoqanOvHUUGxO4iooKkpKSmjQImKYnIiQlJQV95dbqAwE4B/2CAliyZBkFBRYEjGkICwItQ0N+TxERCIwxLVtxcTHp6emkp6fTrVs3evbsWTN/+PBhv+uuXbuWW2+9td5tnHXWWU1S1vz8fMaOHdskeZ0orb6PwBhz4h17iNO5MaOxD3EmJSWxYcMGAGbNmkVCQgJ33nlnzfdHjhyhTRvvh7PMzEwyMzPr3caqVasaXsAWLqRXBCJysYhsF5HPfb0M2013hYioiNT/2zLGhLXqhzh37QJVah7iXLCgabeTm5vLlClTGDFiBHfffTcfffQRZ555JhkZGZx11lls374dOP4MfdasWUyaNIlRo0bRt29f5s499rK0hISEmvSjRo1iwoQJnHrqqeTk5FA9SvPixYs59dRTGTZsGLfeemu9Z/7fffcdl112GYMHD+aMM85g48aNACxbtqzmiiYjI4OSkhKKiooYOXIk6enpnH766axYsaJpK8yPkF0RiEg08BhwIc649WtE5HX3JeOe6RJx3jP7YajKYow5cfw9xNnU/XOFhYWsWrWK6OhoDh48yIoVK2jTpg3vvvsu9957L6+99lqddbZt28bSpUspKSnhlFNOYerUqXVutfz444/ZvHkzPXr0IDs7m/fff5/MzExuuukmli9fTp8+fZg4cWK95Zs5cyYZGRksXLiQJUuWcN1117FhwwbmzJnDY489RnZ2NqWlpcTGxjJv3jwuuugiZsyYQVVVFWW1KzGEQtk0NBz4XFV3AohIHs7brLbUSvcg8L/AXSEsizHmBDmRD3FeeeWVREdHA3DgwAGuv/56PvvsM0SEyspKr+uMGTOGdu3a0a5dO7p27co333xDSkrKcWmGDx9esyw9PZ2CggISEhLo27dvzW2ZEydOZN68eX7Lt3LlyppgdN5551FcXMzBgwfJzs5m2rRp5OTkMH78eFJSUsjKymLSpElUVlZy2WWXkZ6e3qi6CUYom4Z6Al96zBe6y2qIyFCgl6ouCmE5jDEnkK+HNUPxEKfnSJz33Xcfo0ePZtOmTbzxxhs+b6Fs165dzXR0dDRHjhxpUJrGmD59Ok8//TTl5eVkZ2ezbds2Ro4cyfLly+nZsye5ubnMnz+/SbfpT7N1FotIFPAwkBtA2snAZIDk5GTy8/MbtM3S0tIGrxuJrL6C05rrq2PHjpSUlASU9r772nDLLbGUlx+7jTEuTrnvvgpKSo4/oFZVVQWcb7VDhw4RExNDZWUl5eXlNesXFxfTqVMnSkpKePLJJ1FVSkpKKCsr48iRI5SUlNSsW73O0aNHKS0trZmvnR7g8OHDVFRU0KNHD3bs2MGmTZtITU3lpZdeOi5dNc/1R4wYwbPPPss999zDihUr6NSpEyLCJ598Qt++ffnFL37BBx98wMcff0xVVRU9e/bkmmuu4cCBA6xevZrLL7+8QfVVUVER1N9iKAPBV0Avj/kUd1m1ROB0IN+977Ub8LqIjFPV48aZVtV5wDyAzMxMbegwrDbkbXCsvoLTmutr69atAT8B/POfQ2xs7buGhJycuDppG/JkcXWzTkxMDHFxcTXr33vvvVx//fX88Y9/ZMyYMYgIiYmJxMfH06ZNGxITE2vWrV4nKiqKhISEmvna6QHatm1LbGwsXbt25YknnmDChAm0b9+erKwsYmJi6pTfc/3f//73TJo0iezsbOLj43nxxRdJTEzk6aefZunSpURFRTFw4EDGjx9PXl4eV199NTExMSQkJDB//vw6eQdaX7GxsWRkZAReqaoakg9OkNkJ9AHaAp8AA/2kzwcy68t32LBh2lBLly5t8LqRyOorOK25vrZs2RKSfA8ePBiSfEOlpKREVVWPHj2qU6dO1YcffviEbj/Q+vL2+wLWqo/jasj6CFT1CHAz8DawFXhFVTeLyAMiMi5U2zXGmFB56qmnSE9PZ+DAgRw4cICbbrqpuYvUJELaR6Cqi4HFtZbd7yPtqFCWxRhjGuv222/n9ttvb+5iNDkbYsIYYyKcBQJjjIlwFgiMMSbCWSAwxpgIZ4HAGBP2Ro8ezdtvv33cskceeYSpU6f6XGfUqFGsXes8knTJJZewf//+OmlmzZrFnDlz/G574cKFbNlybGSc+++/n3fffTeY4nsVTsNVWyAwxoS9iRMnkpeXd9yyvLy8gAZ+A2fU0JNOOqlB264dCB544AEuuOCCBuUVriwQGGPC3oQJE1i0aFHNS2gKCgrYs2cP55xzDlOnTiUzM5OBAwcyc+ZMr+unpaWxb98+AGbPns2AAQM4++yza4aqBucZgaysLIYMGcIVV1xBWVkZq1at4vXXX+euu+4iPT2dHTt2kJuby6uvvgrAe++9R0ZGBoMGDWLSpEkcOnSoZnszZ85k6NChDBo0iG3btvndv+YertpeTGOMCc5tt4H7kpjGiquqguhoSE+HRx7xma5Tp04MHz6ct956i0svvZS8vDyuuuoqRITZs2fTqVMnqqqqOP/889m4cSODBw/2ms+6devIy8tjw4YNHDlyhKFDhzJs2DAAxo8fz4033gjAb37zG5555hluueUWxo0bx9ixY5kwYcJxeVVUVJCbm8t7773HgAEDuO6663jiiSe47bbbAOjcuTPr16/n8ccfZ86cOTz99NM+9y/Q4aorKyt59tlnm3y4arsiMMa0CJ7NQ57NQq+88gpDhw4lIyODzZs3H9eMU9uKFSu4/PLLiY+Pp0OHDowbd2yQg02bNnHOOecwaNAgFixYwObNm/2WZ/v27fTp04cBAwYAcP3117N8+fKa78ePHw/AsGHDKCgo8JvXypUrufbaawHvw1XPnTuX/fv306ZNG7KysnjuueeYNWsWn376adBjNXljVwTGmOD4OXMPVnkQg85deuml3H777axfv56ysjKGDRvGF198wZw5c1izZg0nn3wyubm5Poefrk9ubi4LFy5kyJAhPP/8840eSbZ6KOvGDGM9ffp0xowZw+LFi8nOzubvf/97zXDVixYtIjc3l2nTpnHdddc1qqx2RWCMaRESEhIYPXo0kyZNqrkaOHjwIO3bt6djx4588803vPXWW37zGDlyJAsXLqwZvvqNN96o+a6kpITu3btTWVnJAo/3aiYmJnod+vmUU06hoKCAzz//HIAXX3yRc889t0H7ds4559RsMz8/n86dO9OhQwd27NjBoEGDuOeee8jKyuLf//43u3btIjk5mRtvvJEbbriB9evXN2ibnuyKwBjTYkycOJHLL7+8poloyJAhZGRkcOqpp9KrVy+ys7P9rj906FCuvvpqhgwZQteuXcnKyqr57sEHH2TEiBF06dKFESNG1Bz8r7nmGm688Ubmzp1b00kMzlDPzz33HFdeeSVHjhwhKyuLKVOmNGi/qt+lPHjwYOLj43nhhRcA5xZZz+GqL7zwQhYtWsRDDz103HDVjSXqvpS5pcjMzNTqe4OD1ZrHiw8Fq6/gtOb62rp1Kz/+8Y+bPN+GvI8gkgVaX95+XyKyTlUzvaW3piFjjIlwFgiMMSbCWSAwxpgIZ4HAGBOQltafGKka8nuyQGCMqVdsbCzFxcUWDMKcqlJcXExsbGxQ69nto8aYeqWkpFBYWMjevXubNN+KioqgD1qRLJD6io2NJSUlJah8LRAYY+oVExNDnz59mjzf/Px8MjIymjzf1ipU9WVNQ8YYE+ECCgQi0l5EotzpASIyTkRiQls0Y4wxJ0KgVwTLgVgR6Qm8A1wLPB+qQhljjDlxAg0EoqplwHjgcVW9EhgYumIZY4w5UQIOBCJyJpADLHKXRYemSMYYY06kQAPBbcCvgX+o6mYR6QssrW8lEblYRLaLyOciMt3L91NE5FMR2SAiK0XktOCKH4SVK0l7/nmYPx/efx+KisDuiTbGmMBuH1XVZcAyALfTeJ+q3upvHRGJBh4DLgQKgTUi8rqqer4+6GVV/YubfhzwMHBx0HsRiNWrSZ0/H9zhXQGIi4O+faFfP+dTPd23L6SlgftiCWOMac0CCgQi8jIwBagC1gAdRORPqvqQn9WGA5+r6k43jzzgUqAmEKjqQY/07YHQnaLfeSfLBw/m3NRU2LkTduw4/ue//gXl5cfSi0BKSt0AUR00Tj7ZSWOMMS1coA+UnaaqB0UkB3gLmA6sA/wFgp7Alx7zhcCI2olE5JfANKAtcF6A5WkQbdsWTjnF+dT5UuGbb5zAUDtILFrkfOepY0fvQaJvX+jVC9rYs3rGmJYhoBfTiMhmIB14GXhUVZeJyCeqOsTPOhOAi1X1Bnf+WmCEqt7sI/3PgItU9Xov300GJgMkJycPq347UbBKS0tJSEho0LpR5eXEFRURu2cPcXv2HJsuKiK2qIgoj3eSHo2O5lByMuU9elDeowcV7s/y7t2p6NGDqvj4BpXhRGtMfUUiq6/gWZ0FpzH1NXr0aJ8vpgn0tPVJoAD4BFguIqnAQb9rwFdAL4/5FHeZL3nAE96+UNV5wDxw3lDW0LdAhewNUlVVUFhYcxURtXMncTt2ELdzJ6xYAd9/f3z6Ll18X0107w5R4fHAd2t+41YoWH0Fz+osOKGqr0A7i+cCcz0W7RKR0fWstgboLyJ9cALANcDPPBOISH9V/cydHQN8RksUHQ2pqc5ntJdq+f57J0hUNzdVNzm9/z7k5cHRo8fSxsY6AcFbkOjTx/neGGOaUKCdxR2BmcBId9Ey4AHggK91VPWIiNwMvI3zzMGz7q2nDwBrVfV14GYRuQCoBL4H6jQLtQonnwzDhjmf2g4fhl27vHdgL10KP/xwfPqePet2XFdPJyVZB7YxJmiBNg09C2wCrnLnrwWew3nS2CdVXQwsrrXsfo/pXwVc0taqbVvo39/51KYKe/d678B++23nWQhPiYm+m5x694aYwIaHWrAAZsyA3bvPpXdvmD0bcnKaYF+NMWEp0EDQT1Wv8Jj/rYhsCEWBjAcR6NrV+Zx5Zt3vy8rgiy/qXk1s3gxvvulcbVSrbr7yFiT69YMOHQAnCEye7GQNwq5dzjxYMDCmtQo0EJSLyNmquhJARLKB8nrWMaEWHw8DBzqf2o4eha++8t7k9OqrUFx8fPqkJOjXj8RNffl1WT920pddpFJCImVl8fzl7nhyzo93thkfb7fHGtOKBPrfPAWY7/YVQGtuz28toqKc5xl69YJzz637/YEDXjuwTy/7kEv4G22oOj79HqC7x3xMjBMQ4uKOBYfGfLzlExfnXMkYY0Iq0LuGPgGGiEgHd/6giNwGbAxl4UwIdewIGRnOx8N5afDVrkp6s5tefEk8ZcRTRu+kMh7+XZnTZuT5KS+vu+zbb+suKys7/u6oQLVr17hgEsgnNtY62RtK1bl9uqrK+f1WTwfyAWK+/95ZL0xumY5UQV3f1xoSYhrwSNMWxzS32bNh8uQYdpb1Yyf9AOdYOe9POGPPNpSq02fhLUD4Cyr+Pt995z2PhmiCgJL02Wewf3/9B8BgD5jhnE8jB27MBqeZsVs35xmaHj2O/+k53aWLXSGGSGMaeu0UqhWq7hB27hpSeveWprlrSMQ5u2/XzrmdNlSOHoWKiuCDiq9Paan3K5xDh+pselBT7kd0dOCfqKjA0sXEOFc/jc2nqcoDfLZqFf3bt3fugNuz59jzNfv2ea+T5OT6A0bXrtaHFaTG1JaN4dxK5eQ4n/z8ZS3vqc+oqGNn6UlJodtOVVWdYLN2xQoyhw9v/AEzgppJvkpOpr+3v7HDh+Hrr53gUB0kioqOTe/eDatXO7dX1xYV5QSD+gJGcnLAt1S3dn4DgYiU4P2AL0BcSEpkTEsQHQ0JCc7HVfrtt3X6XEwDtW3rPPvSu7f/dJWVzoCQvgLGnj2wdq1zVVe7GUvEaW6qL2B06+aUpxXzGwhUNfFEFcQYY4IWE+MMF5+S4j/dkSNOMPAVMIqKYMMGJ6h4u6mhc2fvQcJzWffuLfYdJtaQZoxp/dq0cQ7aPXr4T1dV5TQ3+QsYmzY5zVZVVXXX79QpsIARF14NKhYIjDGmWnS00xTUrZv/dEePOgGjdpDwDBzbtjkBo7Ky7vonnRRYwGjfPjT7WYsFAmOMCVZUlNPZnJwM6em+0x096jzF7y9grFjh/PQcEqZahw7HBYeThg2D5hqG2hhjTANERTkd0l26wODBvtOpOs/F+AsYH3xA2z59QlJMCwTGGNPcRJzbnZOS4PTTfSb7Nj+f00Kw+ci5YdkYY4xXFgiMMSbCWSAwppEWLIC0NDjvvHNJS3PmjWlJrI/AmEawF/mY1sCuCIxphBkzqoPAMWVlznJjWgoLBMY0wu7dwS03JhxZIDCmEXyNiVbfWGnGhBMLBMY0wuzZzojXnuLjneXGtBQWCIxphJwcmDcPUlNBRElNdeato9i0JBYIjGmknBwoKIAlS5ZRUGBBwLQ8FgiMMSbChTQQiMjFIrJdRD4Xkelevp8mIltEZKOIvCciqaEsjzHGmLpCFghEJBp4DPgJcBowUURqj5f0MZCpqoOBV4E/hKo8xhhjvAvlFcFw4HNV3amqh4E84FLPBKq6VFWrH8dZDdTzvjljTGtgw3KEl1AOMdET+NJjvhAY4Sf9z4G3QlgeY0wYsGE5wk9YjDUkIv8JZALn+vh+MjAZIDk5mfz8/AZtp7S0tMHrRiKrr+BYfQXmjjvOoKws9rhlZWVwxx0V9Oy5uplK1TKE6m9MVLXJMwUQkTOBWap6kTv/awBV/e9a6S4A/gycq6rf1pdvZmamrl27tkFlys/PZ1QIXvPWWll9BcfqKzBRUc4LuWoTcd7saHxrzN+YiKxT1Uxv34Wyj2AN0F9E+ohIW+Aa4PVaBcsAngTGBRIEjDEtnw3LEX5CFghU9QhwM/A2sBV4RVU3i8gDIjLOTfYQkAD8TUQ2iMjrPrIzxrQSNixH+AlpH4GqLgYW11p2v8f0BaHcvjEm/FR3CM+YAbt3K717C7NnW0dxc7Ini40xJ5wNyxFeLBAYY0yEs0BgjDERzgKBMcaEuVA/iR0WD5QZY4zx7kQ8iW1XBMYYE8ZmzKgOAseUlTnLm4oFAmOMCWO7dwe3vCEsEBhjTBg7EU9iWyAwxpgwdiKexLZAYIwxYSwnB+bNg9RUEFFSU535pnwIzwKBMcaEuVA/iR2yYahDRUT2ArsauHpnYF8TFqe1s/oKjtVX8KzOgtOY+kpV1S7evmhxgaAxRGStr/G4TV1WX8Gx+gqe1VlwQlVf1jRkjDERzgKBMcZEuEgLBPOauwAtjNVXcKy+gmd1FpyQ1FdE9REYY4ypK9KuCIwxxtQSEYFARJ4VkW9FZFNzl6UlEJFeIrJURLaIyGYR+VVzlymciUisiHwkIp+49fXb5i5TSyAi0SLysYi82dxlCXciUiAin7rvdl/b5PlHQtOQiIwESoH5qnp6c5cn3IlId6C7qq4XkURgHXCZqm5p5qKFJRERoL2qlopIDLAS+JWqrm7mooU1EZkGZAIdVHVsc5cnnIlIAZCpqiF55iIirghUdTnwXXOXo6VQ1SJVXe9OlwBbgZ7NW6rwpY5SdzbG/bT+M6xGEJEUYAzwdHOXxURIIDANJyJpQAbwYfOWJLy5zRwbgG+Bf6mq1Zd/jwB3A0ebuyAthALviMg6EZnc1JlbIDA+iUgC8Bpwm6oebO7yhDNVrVLVdCAFGC4i1gTpg4iMBb5V1XXNXZYW5GxVHQr8BPil29zdZCwQGK/ctu7XgAWq+vfmLrCM0eUAAAKnSURBVE9Loar7gaXAxc1dljCWDYxz273zgPNE5KXmLVJ4U9Wv3J/fAv8Ahjdl/hYITB1u5+czwFZVfbi5yxPuRKSLiJzkTscBFwLbmrdU4UtVf62qKaqaBlwDLFHV/2zmYoUtEWnv3rSBiLQH/gNo0jsgIyIQiMhfgQ+AU0SkUER+3txlCnPZwLU4Z2ob3M8lzV2oMNYdWCoiG4E1OH0EdkukaSrJwEoR+QT4CFikqv9syg1ExO2jxhhjfIuIKwJjjDG+WSAwxpgIZ4HAGGMinAUCY4yJcBYIjDEmwlkgMMYlIlUet8tuEJHpTZh3mo1+a8JVm+YugDFhpNwdJsKYiGJXBMbUwx0L/g/uePAficiP3OVpIrJERDaKyHsi0ttdniwi/3DfT/CJiJzlZhUtIk+57yx4x30KGRG51X33w0YRyWum3TQRzAKBMcfE1WoautrjuwOqOgh4FGfkTIA/Ay+o6mBgATDXXT4XWKaqQ4ChwGZ3eX/gMVUdCOwHrnCXTwcy3HymhGrnjPHFniw2xiUipaqa4GV5AXCequ50B+P7WlWTRGQfzgt8Kt3lRaraWUT2AimqesgjjzScoSf6u/P3ADGq+jsR+SfOi5MWAgs93m1gzAlhVwTGBEZ9TAfjkMd0Fcf66MYAj+FcPawREeu7MyeUBQJjAnO1x88P3OlVOKNnAuQAK9zp94CpUPPCmo6+MhWRKKCXqi4F7gE6AnWuSowJJTvzMOaYOPctY9X+qarVt5Ce7I4uegiY6C67BXhORO4C9gL/5S7/FTDPHeW2CicoFPnYZjTwkhssBJjrvtPAmBPG+giMqUeoXxxuTHOzpiFjjIlwdkVgjDERzq4IjDEmwlkgMMaYCGeBwBhjIpwFAmOMiXAWCIwxJsJZIDDGmAj3/zCAgivvG+2nAAAAAElFTkSuQmCC\n",
"text/plain": [
"