zuma/w5/zumz5.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Uczenie maszynowe UMZ 2017/2018\n",
    "# 5. Sieci neuronowe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "slideshow": {
     "slide_type": "notes"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import math\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import random"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## 5.1. Perceptron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "image/jpeg": "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAUDBAQEAwUEBAQFBQUGBwwIBwcHBw8LCwkMEQ8SEhEP\nERETFhwXExQaFRERGCEYGh0dHx8fExciJCIeJBweHx7/2wBDAQUFBQcGBw4ICA4eFBEUHh4eHh4e\nHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh7/wAARCAFoAeADASIA\nAhEBAxEB/8QAHAAAAQUBAQEAAAAAAAAAAAAAAgABAwQFBgcI/8QAQBAAAQMCBAQEBAQEBQQCAwEA\nAQACEQMEBRIhMQZBUWETInGBMpGhsRRCwdEzcuHwFSM1UmIHJDTxFiVDc4KS/8QAFwEBAQEBAAAA\nAAAAAAAAAAAAAAECA//EAB8RAQEBAAMBAQEAAwAAAAAAAAABEQIhMUESA1Fhcf/aAAwDAQACEQMR\nAD8A+V+BwPxrpaCYOp9CureXaguJ3hclwU7LfuA3g/YrqLioZiYMla/nfUjncWotddkuEGdx6KKl\nUc3Y6LYr0W1CTlBPU6rKrsNN+XMPklrVS062sk6hTi4AEaKgXBoA0JPNO1rid1jEaBbSqfm0Ttsx\nqaen6qhlqU/hkI2XFdpHmIHRbgvi0uPyyVHUo1cvmaY5hFQxKq0QQDHdWqOJWzmEVKYLvX+i1RnP\nzZA12gGwUfPdBdVKlS4c5phpOgB5KE+INZJUE2cdULniRBUeQ/7k4bEyZ9lGUgcnB01MqMuA5FIe\nVNxYkZueSeVHm5z8k4Pcpi9pmu1UrHESoWMdzBU9KmNcxWrRIC7eVPRzuqNJEkmFEzIByWxw1a/j\nL6lSI8oPSeSzYOy4dwOnWwhr6wIJAI26Kapw9Qnyz8x+y37em23tKdu0A5QBI05IhHRTExzJwF7f\nNTkDpIRMw2uwEEH6LpQlDSdQCrjeOcNlWH5SVGKdRv5SupDW8mj5IatrTcPhHyTErmDUdTHmHoib\nWJZK234dSdyB9tlE/DaewMeyYjOY2TAUotyYJVwWZbrP0Sqt8JmZxEd9ExcV2UgBCMU2tOiYVKbi\nMtRuvdEGkiQUxQuAMRyTZQnynZI7wSgAU+cckgORGyIN0nMJ6SkQOsR7qgZhDB6Qj9pShSmonjbW\nU2VSmdNJQz/x+qJgA3RRVGjaSPRWS0BAaYkH9EwxULREAT6qM0SQW5dlcNITM/RJzRuNExZ0z6tm\nyo0B7ZjZUq+EUHknIdfRbRbqT1Q6piuYueG6ZHlza9x+yz6/DDmz4ebvJBXakb7+ijMtB826tjH5\nrga3D1y0GKZPuFn1cJumHzUyB7L0p7QRBbKrPotcZcwEdws4n5sedULGr+Lpscx0Eie2qbHfE/EC\ni1phgH2/qu/qWlGQ5tNoInUAKjc4XQqPLnMaSecJhY8883OU+c8l11bA6ZGjRPZqpV8BjVojf8qY\nrnvGe34SgEyPMtmpgdYCZ0/lVR2F3DdcrjH/ABKliVRqbnXkVj1Pjd6roKljXEkscBB5FYNZpbVc\n08j0Qje4K/1B3odfYrpq85yBrBOq53gj/wA2p6H7FdHXacwiVvh/xZFd5dOhhZeJ0XGqCFseGUxY\nHCDspyi2Obe1zSMx2Th4bzVjFAadYADQkqoHDnopDEza1TdyNlam4+bSFExw2KIU2vBPRVF+lTt6\ngGUwVI7D9Za4ALObTeD5dFM2tXpgDUkK7oa5piiYLpOqr5gTupbqo+4LS9sFsx/fsogyPi2VvQHx\nD1SLi7mUfl6pBzW891kwABPWFKKbolyA1nflCFniO3lTVzEzSwSHJFzOQSp27jJJUoo0xuVfTs4q\nE7KalTqVD5dhukH0W6t1KMV3uBFPbmkX8p6dvp5jBXacAWjTd+IANCfsuKp06lQg7L0jgXy03NHK\nfsru+n5dPUHnJTKQaDuUxTpcJgkqQMTtGqlpt0MrSYANEJy1TZQmLSh+UIbvCjdTM8lO5pQEIflA\nWnos7HQRYudzAMfJazws3Gmh1m6eh+yzhrh6l1cU3EtcYlGMZvKZnPI9B+yC7pNDzvuqjqYTMGpR\n4gqT5xr7K7Sx6lPmaZ9AucyhRua6dFMHYMxO0cRlcrdKvSqguYZ6rgTm3zJCpUafK6FTHoQjklmX\nB0r6uwaO+itUsbuGkFxB9ioWOxOuybKVzdLiI/maD6Aq7Rx+3Il5IPoUwjZAkpOZsqlPErWoYFQD\n1BVlteg5py1GmeS1pp8qiIRgtI0LT7ojAGYbpqq5aEBbqpy4uMnRCWyUnYgLVG9sx2U5TZU1rVYw\nEBAhWyFEW6dFEs1Ve0aQoiw9Fbe0yEDWnVGbNUTSHJA+l5DIk8lo+H6oalPbQoflk+DmHmbISNu1\n27WrT8P1QlpUp+dZbrGm5j8zBo0kfJeVYuzJidw0DQPK9oLf8qp/KfsV4zj3+sXX8/6LOM2Y2OBR\n/wB5UG+h+xXUVW6jTquZ4C81/U9D9iuvq0xA06rf8pe1ioGpvDEaKx4Y6IQpb2tZl5ZNrOB2ieUr\nDvKbaNUsIiOey657TIA5rFxbD31qudsDX9E0Ype2fKPqjzugZZ7wUFWj4Jh5gpmF0ExoUZT0q7hu\nPmVcpXlOTnYJPNZ7HCdVNT8N2k6pBbuKlAsDmRImQOaz31C53wkBXGWpcCQ7RQPa1ji0nUK3wRNo\nucdCfkpG2+X4nD3QsuCBA3QuNSoR5oUaTFtJuhcJTmu3XK0KJlAjVxlSBlNpkqUMK1R5Ia0iOikF\nOo8SSUIrU26NGpRis/8ALsrBPSotafO4DsrDKtKmCGtBncyqrbeo4y5xA9VZt7emXBuaT6rWKtUH\nvewva0gDkvQeAHZsxI6/ZcC53hsFICF3n/T7Z3v9lB1h3idkQahRt3ToSN3CmpDdQt3Cno7FaBZU\nRGiSMhBXqjZAWqZ42TZSrggLTCzsZbFo70K0jmWfjU/hHT0P2URw99/ELQPdVCQTGX6q7fD/ADlT\njSVKI35Z+FRE9BClq7D3UJUUHh/8pSyjl9VIUCtCEc2j5KF0O3aplHlPZZiVF4eUmJEpZT1KkLTo\nlCoHxntPlkehhXMOfVc8/wCY7lzKpOiVpYS0F59kE1e6uaJ8tVxA7lOzHbhplzSR0k/slfsGu/yW\ne9rXDQqUjVZxBqfEpkdPN/RWqOO2rvidlPrK5p9IAGXBRZN4Wortad/bOOlRvzCnFalUHle0xvqN\nFwYzt+Go75ohc3VH4arhKhruiW/7m/NAYXGNxS6b/wDkJ+Sm/wAfuG/Fr7BB1T3NBEqMPaJ5z1XO\nU+ImkxUaR7BWaeOWrh5pB9v3QbXit5uhMajTsQVlf4nbO2cUTLyk4nI4nqg1RlPID3URaOqqsuWx\nMpn3WiVdWnNHhVNd2n7LxXiNsY1dD/n+gXrgvPK+dRBXkXEL82NXLur/ANApjFra/wCnn+pv/l/Q\nrtq5jn1XIf8ATG2qXOKvbTBJykbdivQjhFVzocYg8wtcPGYxTBOqY06g0a1bzcLpU58R7QR1A/dO\n6pYW4zP8M+pCxybYLbe4cQG0ySVctcJuKzoecg9QrdbHMNps8vhyOjgsDEeKnUnk25BE8qn9EhLi\nlxlw+63q+NTJcABzHUrlB5SQ4QVv4jxBcXlMsdIB0nNKwqjSXZiTJM7KwO1zVIxrZl26hACdpdzk\nrfxKt0szT5dVFc03OfnKemX6kEgDklVqVHDRsrFZwNJzWndTuusoGXT2VdrQFICyPMAVWyNSrUOY\nbJNtzGYoxWgRTaD6JEVCeYHRA9NtFurjqpRVgRT+yha1gMuM9lMxzCIY0BA/nc8wMoVy08v+YdS3\nUKmxrnOlzp9lMamVnhgSDodUFku8Wqap1nT+/kvQP+n+hLeUn7Lzui7QQ2IXofAZEggyTOvsg7H8\nxHdSsURIzEbnqpGHsgVV7msJjmjtKjnEZuUJn+dpbtKKizLqD05LQuZ9ks3VQwXHeFI/zdk6+kIb\nk9U6ZrdDrKRTYqFyzsZ/8R3ofstBxWdjP/hu9D9kRxVzs71VR+war1yzQ+bc9Fnl2pMaKURvHlKj\nA0Uhe10gaxumDRG+nWFAJaUvDcpWAkZuSL2VsEGUhQ5R0Culp6BRGl/y+ikmCvlTFh5KYsg7omtH\nVBWewzotDCJFQ6dFWeBqeiuYZ5czomY0QSYi4kR3VIq5dnMfhA91TPyUoApkTghWofEZpjugNP19\n1OfUoXtJ/MVFRZFSqUzCvlp/3FQ1GnbKrLjNZdam6Rz3UDmFajqe8jbbRV6zQOSajNzPBJBOqB1a\ntTIOaFYfTEaE6KJ1MEHNr0lZaC3ELhpkO+isUsWujWbTcQQT0/qqj6bY2j2RWdPNesaBMlIldvht\nNrrQ1Hblk/ReV49/q91/+z9F7BZUQywaDzZ07LyLiVobjd00D8/6BSo6T/pTeCzxlzjsQeXYru8X\nxxzWOdTJ3J0AXl3BNQU8Vk9D9iusu7tjnOERuun851SRXveIr5z3QXR6BZFW+vLgQ6o6FfLml5JV\nd+WZlYs2txSNOpBzPJnugdQkdT6qxWgjyndCHNyQSkiYg8ODoEi0xqFIbhoCgq1gTIWvyBdI/KgG\naUznEpK3pKlaXAHoVI13lEbqsjZv7LNNIvnZFSaXSTsN0zhomDdTKmmrLXU2jQSUBqvcPLsoY9VI\nWtnoVTTjM4jqp2U6kSBCibLNQN0RqOI00Q1Yp+U6pjJeSlnzJDdXFWaWogL0LgJzZEmNT9l5zTXo\nP/T5zvEbJ6/ZLMV3JiTCJnJA/fedUgVExYbujZsoQ4qRjt1vBOzf2UihDk+YqWEShMVEXFKVMNA5\nZ2Nf+I4dQfstByzMb/8AEd6FUclcnzws8tlhI5BaF0qZdAJ017KUkc1eXlSjcuY10AmPqrvj1KeH\neM52v/tFd4f41YVDOhJUmIWwdhzaY5Hl7qfcXA4deCtSLqhIy8kbL+3qPc1riC0wVRsqdRtCq97d\nSIWd4OjHZTuZWrxqdOppFtQAhxhGQs6g97cEcToAdPqqvD1y59y6mTDYOvsVMxNbDmqMrJr4m9l5\nVotggOIGh6laVKXUmvcNwosguRV7DgPDPLZUntMFXbDy0XdwFcA3UZ91VKnrkZioUwC/kEKMpkkw\nCnCcKSm3eVDURaoXtMq7EKCq0phmqbm6mVUrtElaLxpqJVOvE7JlTFB7dFC4AA9eSuPGhVeoNQpD\nVZ22qPBKbn4jTnkf0KCsNStHhOkXXocBsf0VzC12tPS3y9G/ovHOJv8AXLr+f9AvZT5WkdQfsvG+\nJ9Mfu/5h9gs2M6bh6saN+HDWQeccitqrduc4+vVc/hM/iQR0P2K0XkzPddf5fWosm4dyP1Qmq4iJ\nj3VUkyn16lc71VqZz3aeZCXGd1ESeqbN3U1NO85j0QkJ0xXRTJwmRFSpSKNm/sowpGb+yiJXscGh\n2UweyHLC6TEaFF9hTdSaJWE61reISSADsrgrBziYyrXwDCa2JOc5oIa2dYmdJUNHD6jgDou84Go0\n6dtVaGyYO3ooPO7x7aN3VtyADTcW69iR+iiDxvE+6DiEPHEN6J08Z33KChMQUFkPPIKWk4ndQBSU\n9wtNLTHawvR+BaYa1r43nSOy81YdV6fwdDbZjp6/ZSkdS5/mOnPqia7RVmPzE6zBUzCoqw1w6Iw7\nsoWO1Rgrp4idp0Tyo2lFKlBylKCUpUCc7dZeO1MtoRlmQea0C4rMx7W2Haf0QcvX8xMiFULZMzA6\nK3W10CrvUvqW4jd5RtKB+VzAC3ZO/ZN+VSen6AaQDC2RB7KF1nTcC0QO8K0/khG5A3Wtq9K1S0z2\nLrUVMsmZA9eU91Uw+xFrVc7NIgjaOS1RzPMJoEajU8lLadMB9m78XUqEQHOkEjuVu02tFCm3KCQN\n0NWm1zgCIKZm5GbQKJonwQVathFE9wqYJIMq9RH/AGxPQLSqdd2p02KFnmEzCVT+KR1Sp6aIBO8d\nEinf8XqmO3sgJm6mp6mAomblS0806bc1kQXN9b278lRzZ7kJFwqNDm6g8xqua4jc0X5lxOo0nsug\nwZrzhzXOGpPP0C1JqWk/bQKnW1JjVXrhrmkjlrsqVXnlVw1UqbFV3xPRWanwlVnbrOYitXhbvBFP\nM9zzAgnTfksKu0yur4NpBts50Rr77BJUraqv3AbyK8c4n1x67/mH2C9hqfGZM6FePcT649d/zD7B\nS2H1WsDFcf3yV6qSI7kyqFi6K48s6H7K6955t5nmnDytTwjmbqELUwf2n3SzLNUfIJSgLp5Jx6qy\nBzun5oTunG5VZOnTJ5CBIgh9EQQdTaO8TDqUa5T+y0KVFrmF7tyBCy+H3GpaOblkA7/JbdvAZBEy\nPkgoiGg5d5Wpwlcto3j6JIgz9lmEf5zhrE7QgsKjrfF2zrM6zHIqUczxbTLeJrs8nVCR8yqtPcLS\n4xAdjZeOck/NZ1IeWZ2QTBG1A2Z6I42VkxpKw6LosIxx1vTFNwAE9CubZHVTAjdalNd7aY9RcTLw\n3bkVu4RiTKj5FVrhHNeUU6jmknX5q3bXtWiczHvHo4hXTXswrZenzUlO4Y6Qdx2XlttxBe0yC6q9\n46F5WtZcVO1FWnMRBD/6LA9Ca5pE5oRBzeq5O14msXNBqVAw9C5adDFLWpHh3DHejgrBtSE0tVFt\n03KCCHT0KNtcOGwnpKotl2ix+ITNJnv+i0GVmubM69Fm4yHVKbQ0TEz9FjBz1XmoniQrNak8Ey06\n9lCWu5tK1BXIQEKy6mev0URac2XbuqAhII8p6JFvst2pgUoT5e6I+i50qJ6FG+DHJNHXRRYBrfOB\n0VyqD4AO0hV2avbpzVy4pjwG+aNE6/wMxwMnzIUb25TvMoAgSSRSQFPdGx2h1kwokTHBs6T7oMil\nauusZc6oCGjn8lrvzNGWUm5WvLmtAJQVXbn9VYBfrLi5Ubjc+ZTVKmkeqqVSCTqqIn7qFw0d2Uh3\nkqOq4fNWsqVw05hHVd1w600rDN1P6BcWIqVxTA9139lTFGx667RHRZqw5AkkcwSvHeJP9cu/5/0C\n9gGznTOh06Lx/iX/AFy7/n/QLFZqrZz44jofsVbfKp2n8Yeiubkla4fWoFJJzRsj8qlUzAnKWnJJ\nJQk4TpBVMJPr0SUrBmEoYjHcQiATHc9ktER0XDziLepl2B/ZdBaQ6g10wY1XLcPO8jm+v2XRYc4O\npBp5FXFwNVsVjqNVWv2+Fd0qjTy/dXq7WtrAhQcR0xmpumRA+5UsMc3xG7xbum4jUg6/JUmUyWmF\noYzTaDScdAQffZR0A1zcuwUxEOXnCcB0+WFq4JhNW+rZmhwpzqdOi6mlgdtbsBcC50c40+i1i64Z\nlFzvyytQYPceCajgAB3K6hlu1pgNAA5rncbvn1KwpUXgsAg/VMw9Zb6ZpuLXdUbA2E7aYc2Xak6p\nxRIMhQxIHN/2pxUicoiUi3KIzICNdJPVDRFxd+aFLSuq1MSyq5p7FQBo5gpneXdDWtbY5f0IJrOc\nD1A/ZatrxXUYJqNc70AXJZp2SlXTXolrxTaVHDM11OeoH7rSoYlbXXw1A4fuvKZISbVc05g4tjpz\nTVet5aLz+UqJ1pTd+ULzO2xa7o/BUJC3LDiy78RrXNa49gf3TR1dXD27tj5qtVsHNEwEFpxJTc3/\nALimW7awfdaNtiuH1oNOs0HoZ/ZWDKdbkBROokQuncKNQEh4d6FQ1LOmQCNPRbprnfDURp6LfqWD\nZ0kBVq+HuiRJWOXFKxntDYndNH+7ZaD7So06tIULrd06tKmEqtQE1h6q3efwgOxQ0qWWo0kHdSYi\nNG9lbKrMe2SFCVbe1RFrZU9EISUgAGydMESRSITDSVApUNVx2UxKqVChekTnDUFV6hapKvL3Vept\n7q6mmfsoHuEHNy2TlRVfgTTBWEOvWxzK9CqQ2jlBlcTwtTLr8OHIH7hdtV23lPTEInK4ERAK8g4l\n/wBcvP5h9gvXx+bTkV49xJ/rl3/P+gWbEV7ODW35K27Q/F3VOy0q+ytkjMZWv5/VhjE/Enzf8Qkc\nqYmVm58UQjkITpmg6ynQEkEklYpSjzdNECdGacu1GiduUmDok0E6AaqQUXgZnBEaWBODa5aRE99t\nF0eHNPi5AQZIXKYcctw2TAXS4NWY69a0OJ1E/NaaXcRYab2k6TzRYoxtag3K2SAO/NWOJmMbQpua\nYP8AQqrd3VOnSBc6IGylpHO8RUXGhQqNzFrCQ6B6fsVnisxwAadI3la1fEKdembVzZDidYWLc020\n7l7G6CP2TUro8Hx4WNE02U2mTuHR+i0P/kTqoALB3Jd/RcVTJBEAj1VuhULXgEzPJVHR3eMVHeVg\nyzzBXPCWVoJLpOsq4yo1wkiYSo0RnzObJOylWJqVIkBwbuOimFvUcJa0geicPdIgQBoVq27m/hMw\nbqoqthljb1gS5zZB20K06eC0agim1oJ/47rmMDt7+6vH/h3FoG8nt6LpqQxSztn121GnwxJEdAe3\nZGVLEcJqW7S80CQOgXOVXNqiWEH0MrQxPia6vGmhnLSDBgBV7DD3fgzUG3c+iCtSpudAbMnlCerR\nqUhNTynkDpKsXA8Gk1zXQ8yR7KnXuH3T2tqEkjpogHM7uUz3CNQhc7KFE9xdsiakDhEAhFQuHUnh\nzZkd1WEypBl35KVqL7sSuHjK5xIPKVXFxUa4ua9zSehIUQjkkrKrVtsexCgAG16o15uK1bPjK6pg\ntqtNTaDmI/QrnKNnVqGAJ9Vcp4LdPgtZI5kkrpsHXWfGlu4Dx6Zae7p/RbNDiHDK0NFdknqQvPxg\ndSAKj2t9z+yN+H0aBn8TqOhKzaPShWtKwDm1qZnkCCnFKk7UNa4dQF5XVvq1qYp3Lh7zsrNpxXfU\nRl8QuA7D9kMeiOs2uIcAJHZVr2yqPgtbG/JYNtxs1zgKtMgegWpa8UYdcCHOc0iNDGs+6S6AqWNU\nflM+irOtncwR7LcoXlrca03ypKlCmRq0JZg5p9E8j9FGWxufouidZ03g5W7d1Wq4frpAQYT2nooz\nPTZatezcAdlRrW72nSFkU3vLT8M+6gfsrNRhBhyqPPJEqvWOUA7zKrOfP5Y91Pcan5qqd0Qx9VFX\n8rI3lHKiuT5R7o22+DKc13VJ1IOkdwuoqVBmygfVYHCDAykahGkH7rZLvOrEEHauHYryDiRv/wB7\ndCfz/oF6013ncOgK8m4lP/3t1/OPsFmsqln/ABfZWTGkiVWswfGjsfsrb5gQtfz+rA+WfhRyh15u\nATpVOCnTDmnUBSkhThGTqRjRmCjUjfi3+isFymRTpudzhBTdWvKTqTdY10/vuiqtDLMkuEvgNB/v\nutnhu1taNo6vUe0vg+UgDmqOca12YtG43U1KpUpvDmOIIgiOagDntrublJJ6c1bph9OHPomNwCP6\nLInrX95cMFN7nQOgTi3u7hwdUaQ3qdFYpXNDIHOotGm0D9kGL4nUuMtNjfDaOTTv8kWCtrWnTrBx\nqSQdZ2VLE8jbxzomR7bBD4znRJII77oasuOaZ7FFCApaO5a3UmICEAwTGo5LouFcIddP/EVWkMBB\nEtmd1pln2uG3lYhwpuiN1q0cFvtMzfqP3XZMpUKNIMp026c4CLKcoIEqVY5ZmD12kZxHTUK5a4TV\nymCde4W2/KYzN1GyVKplPw6eqiuapYfcYTdGo0Sw8/ogxLEjUoOph8FwIIXW3LaNVha5rSDyXPXu\nA06lx4rXEAEmAP6rQ463saj7lzgxzgdZXU3NjcUrPLcVWU2DcAj9yocSqVLCmG29uCRpMf0XN3t5\nfXBy3FSoQeUmE0Wb+tY0obTf4jtZMc/ks6rcFzxlEBCKQ7D2SdGmkQroKgab67Wl3lB17q9fut5a\nLcQI1PVX7fA6TsMdch4DhrAbPTnKwDULqtSmNAwx9/2WGUgb4hKkqNNPSdUdkxrhJdHstDCLP/EM\nSbTiWA6iJ5FWLFPD8LuLyoctNxEjX1XV4Twu2mA64JgjaQuktLO2sqDadOk0OAAJgCYClLjI5jom\nKouw+3osd4IId1K57HsZbYVW0KRa6qZB7HRbmP3zMPwyq9zofGhmJ1C8t8WrdXlS5qvcSTIBMxqV\nRfxK9uKz85dDjqqLA9zg7MVZGVw84k90JLWiBAPVShy3KAc0yoXog6AZMj7JEAtzAz7LMgAb6J3O\ndIMTGylrU6TaGZtQFx5Af1UIcAB5ZI1VnQno1KzRmDg0d1etuJL63cC2pmHcLHqPc4akgdFA2W9D\n6IldracbV2aVmMgxsD+61rTjCxqCanlPcFebaOEluo21SBJMHZEeuU8YsK5ytuQSeoI/RSGrbuHl\ne0k7wd15G2rUb8L3D0JClp4jfUyPDr1QB/yJW9a16hUt6dQn4RKzbiwpkeV0lchQ4kvqQh5Lo6kr\nStuJs5AqU4HUu/os0XbmwdEN7rOr4fVDjDSfktBuNWrgXEtJ9VVucbozDQ3/AP0oKj7N7ASWqlcs\nLSGluhV12OUX+U0wJ/5f0Va7vKVVmZjQCAdJRlq4bilOzsMmkqtS4kDjLmyfQrm61Sq4kZiBKibL\nTo7TorB21vjdOoyo6QCBzB5grzzFqnjYlXqiIc6fotFtVzWuhxMjaVi1TNRx7rNE1nm8X5/ZWX5j\nGsKrZfxx6H7FXVrh5V1HlHNyJM5JS1RBOmanU0JOECJk6wYVTBKRkyDG5UevVExpzNnaVYYuXrM1\nGhlJJEmPkhpXNRpNMEgRsnu6zQ5rW8hqq3xO1VMT0qrqdVry2SN9Apby+fcBuZuUNkCAB0VQebUp\noLiB0WUS5nEQMyMbSRqgY2ArNINDQ0O0RYiDQ4SRCNlNztlbt6FSu8MptLpIBPRdjgGB06DRWrNJ\ncRtI6oqhw9w14zG1bqIB2BOui66lQpWtIUqbA1sQQOyf8S1jCGlU6115hJHNaTFrKSDl2UghtMSF\nDY1m1H5Q4KUKerIjqCSI5oSNMo3RFuYlLKmGADiTone4kAdE3h+qRZHVawV6tuyofNTB9Vh4xg9J\n1Img0AgdSujyhNUAkuClg8uuaTqNUsduCQhAldxjOGMvWbEuAMHTSVyOIYbcWry1zCROh0UwPb4l\ncUaJpeISw8jCoHWo+o0RmMlO7MT5hCfKIg6T0UZTWlSJlb/CuIUcPvpqtIBP6LmDMQDstaztHOo5\n3GCrKPQxjFjXcXCpE9YSOJ2TQSawMei8+NuQR5tkz2csyurq1xji344+BRccum8dVzzaZpAS6ZGy\n1MrUL2tcNRMKaqi2oUzabnayrjfDB+FDKaK4YcwaNSVJk8Pdsk8kectIKI1MwSXBUAc0lzhI6JVK\ngBAyxoptOsoHUw4HlCoqvcXEgBKnSdKtNa2m3MN0RcpOKVXNItieaRpZUb3DRAXKIRaED27JnHRR\nEq6mpMreYB90/iN5EAKGUiosqdjyQYMjsme7XUFViYhOHIuidlB0KIOdGh0G6GUzidERI9wLB1UU\npZioDsromJ8jvRZT/jd6rRp7O9Fnv5+qzaJbP+N7H7K4522m6p2f8ce/2Vt/Ja4fQzikmKdZqwhP\nISizDnoh83JPpzRTxoE7QATzS5nsnGyrGn06IgSB1PIoU6sagiS45junLhIIGyEJ1dUYGY6GB1hT\nUKLnfC0meYCvYRhjriu1rvK0nUFd1YYPZU2Motpg1CBJkrLLibPB7q4g5XNB5lpWv/8AHvBPmeD/\nAPz/AFXT3lChZ0i3QuHILA/xBzhmc4kIsW8Ms7ezGYgEmPyxsrr75tMaO9iVhVsSdENMbzMLMucS\nqOeWySiusN9Rk+YR1lZWK4pTHwOBInQO3WVZW99dAZXQD/fRaA4fqeGH1HSTqdSmiTh3GC67yuJA\nM6l3cLs25Y0cCuItsNNOsPCaBB0K6q28Ro8xVhFobnREIURdJ7omu0VUUIS3b9kScLaI8qCoCdmk\nBTwk7LGylIpGkZ3j2UNxZUqwio0O9Qr5hCVlK5mvw5QcMzdO2X+qjbw9TgDbuWrp80mITOYXb6Qs\no5M4Ax1fK3Qdcigq1qdF76TWjymN4ldlQojMXGNF55ePc/Ea5B0BCC0ajcnwwT3VdztZlJ5MADkg\nIJ5oAzEbmUxqaEZfqhcUA5o1DnedQlm/uUkKFFOYaoQC07ynCElAwEbOSe52kOgdOqDMhc5aBPqG\nNNPdCanZRvcgzqxKkfUiNPqgNTt9UD3bIZWag3O5RCAlOUKjJJFySZFhn6wmCcpkURdrt9ULjIGs\nIRqJSkT3QPrz0QHYp5P5kBcgJroBETIVB51PqrgVN/xH1WaJbP8AjD0Vt0wIKrWrf80c9FeYBElX\nj5ViEtd1TKxA6qEtclUwTpNbvqiyoHThLy9U2nIykZFCdDKeVqh9OaWvJIR0S15GFBr4diT6Lg0x\n7retMaLAXtIkRpB1XGh3nzfRTsrObDgDA5ArWj0bDrpl1TNSuQ0kKClh9Co57S/zEaFc1hdzXuA1\nrn+G0Ded9VpXeMWlmw+AfFqjnmj901YKpZttHVHVW5m6wTGiyPxFjRunAN8Qn6KrieLXd4C1z3Ma\nZkAnmqDG5TmJJd1RXp2FZHsa5o/uFove0jfbRcpwtijXUhTqODXTpJ7LbdUnM4OkdldZWmu10KRK\npUrjzluWe8q2YyyFLVhTqIU1NwgKueWsyiaTt9VNVaDtVIwyqrXidvqpqTxqqJlGiDmoZExK1xCI\nSyoXu6apNMqg6bRCdzW5TKAODQNZlNUqCInQrAdkhj8vReWfFd1XN2n9l6fRqZm1Q3kP2Xl1MkPq\nuHN37KVKmYRmd1nVNVKFuhLuuqZ5lQ02sKJ5Mo8xhRPMndFPJ7IMyRKj90SpQ/dA4yh90xdHKUQ5\nKB0Hfkk5yje4yFAZd0QIQ7tKfMkCKUIXHUAJsxgnoqCTFNm7fVLN2Ro6FIO1Qh2p02RORzyTQkHT\nGiWbQ+XZENHokNNkteqF0jmpGTk6oTumLinO61fFh3fCVSO5V3TKR2VJ259Vzonstaw9P3VpxMCO\npVS1/jj0Vt4JAjuun8/GobM7onDtfMo2kNOqlU5Xat6Ly8kkklE0oCWnJEmKRDIkKk0Ws0CE6Xoi\nCYEFIJ01gc073CFHOYnss4LAuHMAbTcR6KLUklwcT1QKRJywEJ57ckxQwCR2TuaIWpy1dT0Kz6Lg\n5jiCOYWvaY9VawscSYAEmFitDQPiQPidDKuM66W3xoF5LtCugs8Sp1GauP0XnKnpVnMEghTlFlek\n07hlQnKZhSh+mi8+tMXqW7gAAQYmR0W1bY6xwGdwB9CsrrpM5RtrRMrB/wASocnA/NQ18UY0DKQJ\nla010n4kdVF+NZm+IwuSrYrqYI+SqHEDnJlP1hrum3jZMEomXRJ3XEUMSygkka9lYbiw6j5FP1/o\n12X4hska6KOtcAtBB1C5P/GD1HyKYYoKjhLmiOxTTXW2tceFXnTT9l5s1zoqd3fsulpYg1rSSQVz\nmk6dVN1L2LMdkxKF06qMomHLkBKZ26jKuNaMlBKSjS9FuiqOiO6EFIplnUIuQFOmKJpJpTqNIS6L\nmEhsUJTKqdPOiFC5F0TTqUh8TkCSIIfCCnO4HVAkgL2TO9IS8vf5pjHKVJ0mAKUop9UC1+jBs2Po\nqjvid6qw7Y+iqu3WL2YsWn8cDt+isvJiBpqVWtP/ACPn9laG5WuH1qBG2rZRZu31Two0vq8hhx5D\n6p5PT6oWkaynzDsoyOUt0ycJAoRadEyKFsMPSEQTAJ0DGYTsShOAOZhYoSLMmhvVMsgi4yOSRcfR\nCktQEHDmJ90i4ch9UxTLoyLMjkRESokYELPJYcRpyjuiDiDIKAuCafVZVYFVwMhx+aTqrnRmcfmo\nZHdMXTtK1jOrAbm5k+yA89PqnoOy7ynMLNWB5BoH1SzGD2TAyfRIbFFKe6Rc5uoOvZDKYkQlMS0r\nqo3QkkeqmkbgKm3cqcO0TiYIv3kIC5CXaoSVQi5CXDmnTGeQlTQ2YIJ1Ry7/AGqKdJSgikgzaJT3\nQOmPJNKcbFaxMNKHTqk8wlp1UzCGKUJJIoSYQlwO+iYItRylA3l/sJeX+wnk/wC1MgYxy1SlI8tI\nTIJNOn1QujSAmlIqBT2QwnSUoaPKfRVXb+6uDY+ipu+I+qyJrT+KPRXGTB9VTtCfHECZB+xWgwN1\nzECSt8fqxE6ZT6qY0SRLSCoiMu6lzVpjKbVOfRMp0CSSSW3MkYKBIGEFikM0t3mFMyzqVHkNMCOo\nVRrnTLTEKYXNVs5XEH1QL/LpiGmUxOYztCh9dUTT00Qg8o7IUXvCCVK0dJNKUoHThMkEDokJKSlX\nTpkTWzOsJ8vf6KM4cNRDmmLkhPRaUiUuSYg+g6pckiUjskkdgkqhJinQuKiZCUjToopTtfpt9Uiy\nCedCoipDME9VG7Qqh3boEi5NKn5dDpkpSSQzTGeSb1Sd2Kb2VTDJJpSlEOh0/NunlDm6iUBawR12\nTCACCPbqhJ8wPIck5drPMICSQ5ki5IySZLN2TSrasI8kx1TlMNFFLypJadExjogdCnlIqUIbFV3f\nEVOHaERuFXduVMZWMOY59yA3kD9irjvKYMSCUfCWV2Msa7QEH7FdTjGFMqMz0ZkAkjRa4z1uOaoV\nNhojdb5xIUFenUt6zmvEEc0dK4I0KzZ216huKZpkDrKi1V59TNACgqUzMhMTEaScNSK3jGGSSRay\noYTeadMJkynQwyQSSQwYKECEyNSqGUk5TICSRNbI7qanRMa80wVwpfD7qfKKeyHVTl0Iw3Lz3SUh\nDRqd+SVOm6oYptc49gpO1iJExrnAhq6fBeFq908OuM1NoOokLrbXDsPwygXQNAJLiOQ9FrEeUOZU\nByzqnXfYrY4Zi+Z1CoDUB2Hy5hcdiGE3Vi8udTcWDnoVRSSTDLpGhG4SOUjaUTDoXoUpIB6KNYZJ\nu4T0qdSqT4YJPZdBhOCubFW6Di0H4dEiWMI5o0UL2ukrrsTrWtrRDdAIIAHZYVb8NctLqbsruiqY\nzUlLXo1GulwlQHeFrpsSCU6Dzc1KbgimTNyzonG6hpkkKfkrYydAnTadFnQkkkk0JMU6YmIV1MOh\nT7plNISSSSapJinTGeSaGTlPBSZsfRNSgG6ru3crPNVn/GU0jT4czNxOm5rtROg9Cu+FUOphp0IG\nsndcRwg1rsYYHajX7FdZibhRqjIDGsq8ONaipilrSuCSWtB6wsC5tXUTuT7QuhbUFQhoBJQVaLaj\nSHgSpZ205xjso1E+qmpvBPwz7q1dWJ1c2ABKoFrqbiJQO5qjLTHoiD5UjdiFthW1nUIpMyiq00Cl\nBA7pSmCdQLTqlolr0S16QgSNAia0qUPvsjZRc7/0jotaJza7Kb4fhQMGNaIBCfMYQ5kjyVCO8lyJ\nzmjbZBKt4dh13eyGNBA5ys8hdwHBLjFH5oLaYIklpMg/+l22GYDY4fTBLKbnAalwAXOcP4lWwet+\nHuIDSQNANI0/VdY+tSxC0Pg1PMdd0go4vxFY2FNzWOa6pGgBAXC4zj99iQLWufTpyZAcdQfkrePY\nBdUXOuHOL2xzJ6rNw6xq3dUMpN1kAg+q0LfDFviDrgG1FV7oMxJ5r0V+EuqWTvxbWg7QR+6PhLDa\neE2TXPa01CDJ3590eM4kyjSIqvho1MAdUHF4vwo6Kla2cAACSA3fmOa495FGu6k8AOaYIXV8Q8VO\ncw21m45SCCYC5Slb17u5Lg3NUfrJ5oBEES0yr2EYdVv6oDZDQROk7rTZwtdCy8ZxbmHKT+yp2F5W\nwmu5rmwCQDHaVGnR22G2tiwFzGucNyQAs7Gcdp06TmUHAOI0IOyvW2KUL1gaZk6rExHB2kGowyB1\nJSJXP161e4qOfVqucTqJnn7pUWuaZEn00Ur6DmVS0tGmis0LckDkqgaTqjh59VHWptkubp+qsVct\nMdFTrVt+SmtInODTB1UQceaTiXGU5EKpTkhpBASDtZhAE6IfMnzDnohS9Ul0Pmam06JvKnSwJJJJ\nQDJ6JT2+qkyoXeWJ5oTsGvolKKUCLYeUpTJIghqOiWxhO3ZD+ZA+bQmNkIdCQ2KZEpxzMbBVXfES\nrQOh9FWJ1OnNCNfhZxGJtPPX7FdpcNzEF20arieGNMVZzifsV3deMoJ0ESpx8rXFkMmjfFo2I0+i\nmc5BSaal250aDY/JGWGN/on1aAxBBWddW7KjpEytE0yd3R7ITRbyMH0WpBzZovp+XLp1SBMkERC3\nq9sxzCIjvCzLq1dT8wJI15JqVXBSfTHLVM3npCOkddTKbrMQmm5pmEjK0G0hUaYMR2UFS3c09R6I\nYrwkG6gQpvBH+76ImNDZMg+ylULKOYy4ypXR+VDMHRL6LKUzp5pwkW6TKQiVr9NQZc3MQNUVKjUr\nvDabXOMwAAtPAsBusRqhzGOFMmC4tJ5LvsG4fscPYHVGsqVAAZLQII+a2RzPDvCVa4AqXrXMYRoJ\nHX3XaMp2eGWjg0hrW6yVSxrH7LD2QKjZGkBwHJeeYvxDd4kS0VH02nSA4mfspWnaV7jDcYNSm2oM\n40A2gn1HZY1bDb7C65r2oLm8tQf1XK2NWvQqh9Nzw6RME6r0DhqveXVFv4mm4sjdwJG/cKIWG41S\nvB+Hu2+G46RB/qtW3t7O0Dq1N4AdBJnp/wC1UxPBbeqDUptaxwEhwELisfxK6sqotBcOc0yDBPKB\n1WpMHTY3xTRtSadFwc4GJg9FxF7e3eIVDUru30hVgRUcXO8xOpkyV2XC3DP4louLgwwHUFs8vVLR\nhYJw/cX7xlY4NkSdNiu/wvAbTC6AcZNSIJJH7LRt22tjSNOixgAABIAGyzr28dWdlDpAPIomFiF5\nTaCxplxC57EadtdFrKjocZj3haNRoc/M7eN1zfELnMrN8IuDpMEH0U1VTEMJuKJLqAzCdNQoLTE7\ni3llUSO4K28Cbd1WA3BcWx+YHr3QYrhtN1NznZWEDkAn6xKoeNa3Rzkw7oqd/cNojK0ghZdw4Url\nzKdQw08tJUVao6oYLifUqfqoavWNY6oQNExbrI0TyQBKuh4Qp839ym91OV1YYpk5ShRTaZdN04nK\nZTaRATgmCCqycfAkmnSEggR2SjSE+jSDulm1nL7KYB16pjPqlpvH1S0gaQqGSSSlFmfSSSlKUNhJ\nJSlKtqkmTyhlRORxzVd3xlWBsVXPxFZqNXhgxibSeh+xXYXtR9TKxhjSD9FxvDX+pN7g/Yrsg0D3\nV4/V4wNNopMLY8x5pi4KQjyFATCS91q9AKYqRup9VKfDpgGdVZyRVc10whfQbUb5hIUznyU4GYjq\ntJYxL+zFHzgaSsxjjK6u9pt/BvLtey4+k8ugdFLMSdNKlULY7qYu8QaBUqZ3KKVF0ZaUJBRhwhC5\nwkaSEsZ0xdBCQ3RNl2jWyOgXR4BwtdX1Vr6zHU6U6yR0UnS5rDtLSveVW06DZJIB7TsrmIYHfWIa\n+qxpB6H+i9Rw/DLbDKAbTbJAEuMcgq13f2NaqbWrUbm6f2O6ZqyYzeDMXs/wYt8uSpO8DoO62cUb\nVrWzjQd5iDB9lzWM8Nii417CZGsSP6KphmO3mHVRQu6fkBAJIOw35ra4wMdw+/ZcVKlw7O3NoJ7e\nioWlq+tXFKm3UmAV6rRuLDFqIa14JOpG0fMIMMwe3s6viNaJGs+yZpWLwzwq2mBcXbQQQDBJ9V0d\narQs6MACnTGmgVLHcct7CmXOe0kAw0AnZeeY3jt1idc5SfDnRoGvRMGxxPxW+pNGxe5jCBrA6+65\n3DsPusYuhLs7iR5idpPotfhrhu4vqratxTLKQdqDHReh4fa2uHUQymAIA1PYJqa5xnBtCjYtBcPH\nI1OY91JYYlXws/hbkE0ydxH9OiPiDiilY1Cym4OfMaA9FWw7HLLFGeHdZQ4mNQUNbfj0LyiPBJEj\nn3VZ9F1IkblUrjC6lAi4sXQBqBI15hSWmLmfAvqfhkfmgwfuiygqNfnAJ3QHDadar4lQA5dRqef/\nAKWzRbRc+ab/AK6LPx7FKOH0HZqjS+DA1OyxejENxUt7RgLhlaNNFw2P4866Lqdu5zWkbwOqrYzi\n1XEK5DvgnSFRDQFUqNgc85i6SdyjAh2qd3JNCID3SPLWU6YpiaGUUpklGvBApZkBTJhoktUtOiWn\nSE1CSSTkrUoEGU6FxTJ0Q/JJJIqBJkk7t0PTJJzshKJh0kkxRrToUkSmYlphzVd25VnkfRVzupU1\nr8KtacVaDrofsV2VUBhBGq4zhSRirT2P2K6q/rlrwBK1w+t8UxeMkRv3UQIiS4BVhUc5u6cuMarM\n9q8huq5SQ1R+I5x823coczXfCNRugfUY349EiLTXCVMK1OmM1QgADTXdY9fEG0wW0wT3VGrc1avx\nOMa6LcK1MQxBpa6myDPdYLGAOkKZoBMk6pg1W/GSZ5REohmPVM9uyNjHxosgjEeVwKmsbK4vqoZR\nY4yYJAJiUJt6lF8VRHovSOA/8Pbh8NYPFIEkjnB7q1cQcN8H06DBWui1xI2LI59yumubi2s6Pmc2\nm0a8gmv6z6VAuZ5gNgOS8w4mxbELq5NN7yG6bQOfoskb3EvFjpNCzdrBBIdv02XGi6ujdOuHVHl5\n6klOyn8L3mZWhZYVWxC4a2gAARMlaiuo4b4grXLhRrUy+TvM8vRbmJ4Pa39AuNJrXEGCGgmShwLA\n7fDGA1GBz53k9FdxK+o2dE1KjsrQCYVVweJWGIYG816TnCnMAiR3/RHT4zfUtnMLfNtOaP0VPibi\nCvilY2trJZO5A6f1WFWwu6tSDWZAOuhTtKK/qXGI3ZOd0vOgkncrsOFuEWtY26vSRI0BZ37lcbY1\nm290ysTOUgx1gr0jB+Ire8o06Tpa5o2gfuptGtc3FtZ0i1uSkwCYEBcTxPxVLXW9q4ayC5tTblyX\nR45YuvrZ1Njy0kbg9151i+BXFlVh5BAJJMkqsqD6lS4eX1Huc4mZJlS2rajaw8Nzi4nQAmQntqDq\nzxTpiSF3vDnDtOzivdta542EnoiwfDLb/wDDD8VUcWkCA4HT5q9iNjb3DP8AMIaP90BWLm5o0WFz\nobTaDt2XCcT8UVKj3W9q8tYDvA6f1RQYnjVXDLo07WuarSOTiBz9Vg399Wv6hfWc6SSYJJGqqHNU\ncKjnFx6FOfKdoB2Wb6mna1rTO6Y1G9ISJAQZndEzSiLtRAlDmMxH1TOLomITSYlVCzJTKFOEqQ6a\nU6FZjVPuCeiUaSkN4T84VQ2YpA9UyRUDlyRcPRIIMw7pIlF7ptOZhMJOyQHVXGoKUxSSRCTlMk5A\n4MgoSnbsUxQOmKQSKBk8pkkSnnQjsoDupuqhO5WaNHh3/U2eh+xXT1yGiYmVzGAuazEWudoIP2K2\nLu+Y0EEjnGq1w8ani14gyKCrdNbJLgSsutfPdLWE+xVZ1VxBOp91nMq60a9+RAaPTRVH16lY+bZQ\njzQS6OkhPqPzCFuGjyjujYIlCHCNdD0RskgnQR1KbXM6YAl4aRv0VqwsLu+qZaFBzu4BI+y7nAeD\nKdEipfOa5wOxp9u5Ta1HLYNgN3iDw1tNwZI1MDQ+q73A+G7PD6YFXMakanT9lqsZa2NENphjWgCY\ngbLFx3ii0syadIhzwf8Ad29E7aTcS4NbYnQ8oioBoQR1XC5b3A7zLrlBgE66A9lp4HxO915Fw4uY\nTGr9tF097ZWmLW4c1rXZhMgAkSFrsRcP8RW91SFK4cGuiNj1VjHMDt8Qti5pl0aEEdVw2NYTeYXW\nNSkXFk6QCI0lafDfE9SkRRudWk7udtp3WbqU9hwvX/GtbcAikCOY1Eru7GjQsLYU6IEARJVajfWt\naiKzajCIkwRouX4l4mFF5o2rszgeR7dlEbGM49b2dJ7W1GuqAaaHquHc/E+IL3K1hLM0SBAAJ7ns\npsAwO/xm4FW48QUZ1zNJ5dSvQcKw+zwy2FKlTYXACXAAHRGlDh/hu2w2k19Uk1SNRI6+iuYzTs7h\npoVA0EjQKtxBjVvYUSZD3zEB8EaLgbnHbq5v21hUcI5SY5oLGN8N3FGq6pbAvaSSRI0HJYY8S3qZ\nX5mOGhHNd3hWO0a1PwrnK10AS5wk/NR4rgdrfUnVqYbmJkFoBJ+SGs3BuJ6lOoGXGUtJ1MHp6rqK\ndazxKgWsc1xcNjpErzi9wy9s3nxGuyjmAYSw3Eq9rVJp1HCIkAnktSYz09BsMLt7Co6qPiKLFcTo\n2lsaldzRA0Gplc3R4q/y8tSnJ6l39FzGKYlc3zyalQhv+0kkJyurOvFjH8fuL5/hsIbTBIAAOoKy\nGt1JcJKTMrQQQD3IT5tDBAlZNGHaKOoZKHNl0TF2o7paadRos2h0TadVGadvNEgBjknzIgUtesJJ\nGOiNH16pktOn1SlAkkpSlAteafQCRukfVMNigMbZkGwTh3l2QEz2QIbk9UUoeRPRITqXaDrKBk4T\nJ9FQ6FPKaR1V7+BJJackk2hZkp7poTj1We2Taf2EyPXko02rBDf2KgUzdzryUTt95TVW7djm1BUa\nYifspXS+ZMkdUkljjemip0iToPktOjgV9VAFNgM9/wCiSS1Eq5bcK4jUIFSmKYJ3Jn9Fefwpa0mZ\nri+cDzAA/ZJJbiK9XC8Ipgg37p7gaqi5mH0agLXOrNBnKdJj0KSSyy38J4qt7Olkt8La0xEkyfqS\ntqnxJiVzLqVi0jbdJJVqK9y3GsQIFSn+HYTuDvPsqF3w3atBfdX9QE77fskkprSi/CcPpsLrfEGl\n3IOMD6BbPDn+I24Ph3VnUaIEGo7v/wAUklrRuVvxF2w0rqjakby17j+gXNYtgmH0H523FZo6NAP3\nSSSpVJ1S3t6Ja28uoiDIEfdZ1tdUqN2a1Sm24B2zEg/RJJRHX2XFjqgFva2VGkP5j/VWbnFcSqUi\nynSt6cgiS8kn6JJI0wL/AAp9Qm4vrqqZOzACPrCyzRwyk4guux3DWn9UkkSqFe4pUq4dQL3AGTng\nHfsVt2nGYoW4Z+CnLoTKSSIq4jxVVuS7w7RmRwiCdQuaqVHur1HggZjMDlukkt1cN4rtJJMJi73P\nIpJLCUwdunlJJGSbqJQnc9kklmhuY7p9QkkkCJlMkkqEkUkkaMkkkgSSSSAQ6dwU5iQIiUkkCnWE\nySSCzYUres9zbit4TREHqnq0bUPLW3GZuwPVJJBBlp8nyhIHIykkqG06pa9EklaFrzEJJJLOhpTO\ndEJJKaybM7khzO6JJJqw2YzsmGqSSiv/2Q==\n",
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"800\"\n",
       "            height=\"600\"\n",
       "            src=\"https://www.youtube.com/embed/cNxadbrN_aI\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.YouTubeVideo at 0x7f8808605e10>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import YouTubeVideo\n",
    "YouTubeVideo('cNxadbrN_aI', width=800, height=600)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "<img style=\"margin: auto\" width=\"80%\" src=\"http://m.natemat.pl/b94a41cd7322e1b8793e4644e5f82683,641,0,0,0.png\" alt=\"Frank Rosenblatt\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "<img style=\"margin: auto\" src=\"http://m.natemat.pl/02943a7dc0f638d786b78cd5c9e75742,641,0,0,0.png\" width=\"70%\" alt=\"Frank Rosenblatt\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "<img style=\"margin: auto\" width=\"50%\" src=\"https://upload.wikimedia.org/wikipedia/en/5/52/Mark_I_perceptron.jpeg\" alt=\"perceptron\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Pierwszy perceptron liniowy\n",
    "\n",
    "* Frank Rosenblatt, 1957\n",
    "* aparat fotograficzny podłączony do 400 fotokomórek (rozdzielczość obrazu: 20 x 20)\n",
    "* wagi – potencjometry aktualizowane za pomocą silniczków"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Uczenie perceptronu\n",
    "\n",
    "Cykl uczenia perceptronu Rosenblatta:\n",
    "\n",
    "1. Sfotografuj planszę z kolejnym obiektem.\n",
    "1. Zaobserwuj, która lampka zapaliła się na wyjściu.\n",
    "1. Sprawdź, czy to jest właściwa lampka.\n",
    "1. Wyślij sygnał „nagrody” lub „kary”."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Funkcja aktywacji\n",
    "\n",
    "Funkcja bipolarna:\n",
    "\n",
    "$$ g(z) = \\left\\{ \n",
    "\\begin{array}{rl}\n",
    "1  & \\textrm{gdy $z > \\theta_0$} \\\\\n",
    "-1 & \\textrm{wpp.}\n",
    "\\end{array}\n",
    "\\right. $$\n",
    "\n",
    "gdzie $z = \\theta_0x_0 + \\ldots + \\theta_nx_n$,<br/>\n",
    "$\\theta_0$ to próg aktywacji,<br/>\n",
    "$x_0 = 1$. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "slideshow": {
     "slide_type": "notes"
    }
   },
   "outputs": [],
   "source": [
    "def bipolar_plot():\n",
    "    matplotlib.rcParams.update({'font.size': 16})\n",
    "\n",
    "    plt.figure(figsize=(8,5))\n",
    "    x = [-1,-.23,1] \n",
    "    y = [-1, -1, 1]\n",
    "    plt.ylim(-1.2,1.2)\n",
    "    plt.xlim(-1.2,1.2)\n",
    "    plt.plot([-2,2],[1,1], color='black', ls=\"dashed\")\n",
    "    plt.plot([-2,2],[-1,-1], color='black', ls=\"dashed\")\n",
    "    plt.step(x, y, lw=3)\n",
    "    ax = plt.gca()\n",
    "    ax.spines['right'].set_color('none')\n",
    "    ax.spines['top'].set_color('none')\n",
    "    ax.xaxis.set_ticks_position('bottom')\n",
    "    ax.spines['bottom'].set_position(('data',0))\n",
    "    ax.yaxis.set_ticks_position('left')\n",
    "    ax.spines['left'].set_position(('data',0))\n",
    "\n",
    "    plt.annotate(r'$\\theta_0$',\n",
    "                 xy=(-.23,0), xycoords='data',\n",
    "                 xytext=(-50, +50), textcoords='offset points', fontsize=26,\n",
    "                 arrowprops=dict(arrowstyle=\"->\"))\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdMAAAElCAYAAAC/NQipAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XlYlPX+//HXKDBCkEJq7slRsYSK\no+QleEwjLXJtcT1hYouaWRHlUTu4lBqW2sFzyrUFU7sS7CqJXMoMy8qSOqZBmSaQlqZmHksEBe7f\nH32ZX9OoDNwMN8vzcV1c4WeZec/dcL+4l/lgMwxDAACg8hpYXQAAALUdYQoAgEmEKQAAJhGmAACY\nRJgCAGASYQoAgEmEKQAAJhGmAACYRJgCAGASYQrUAjabbZPVNQC4MK8KjmftQcAC3bp1k/j5A6xg\nc2cQR6YAAJhEmAIAYBJhCgCASYQpAAAmEaYAAJhEmAIAYBJhCgCASYQpAAAmEaYAAJhEmAIAYBJh\nCgCASYQpAAAmEaYAAJhEmAIAYBJhCgCASYQp4KZDhw7pwQcfVGRkpPz8/GSz2ZSXl+fW3NLSUiUl\nJal9+/Zq1KiRrr32Wr3++uueLRhAtSFMATft379fqampCgwMVK9evSo0d/r06Zo1a5YmTZqkjRs3\nqkePHho2bJg2bNjgoWoBVCebYRgVGV+hwUBdUlpaqgYNfv/984UXXtB9992n3NxctW/f/qLzjh49\nqrZt22rq1Kl64oknHO033nijjh07pt27d5f73BEREcrKyjJVP4BKsbkziCNTwE1lQVpRmzdv1tmz\nZxUbG+vUHhsbqz179ig3N7cqygNgIcIU8LDs7GzZ7XZ17NjRqT00NFSSlJOTY0VZAKpQhU7z9unT\nx4iLi1NcXJyOHz+uoUOHuoy5//77NWLECB08eFCjR4926X/00Uc1aNAg7d27V+PHj3fpT0xMVN++\nfbVr1y7Fx8e79D/11FOKiorSxx9/rMcff9ylPzk5WeHh4dqyZYvmzJnj0r9s2TJ17txZb731lhYu\nXOjSv2rVKrVt21Zr167VkiVLXPrXrVunpk2bKiUlRSkpKS79GzZskJ+fnxYvXqzU1FSX/szMTEnS\nggULlJGR4dTn6+urjRs3SpJmz56t9957z6n/sssuc9y0Mm3aNH3yySdO/W3atNHq1aslSfHx8dq1\na5dTf0hIiJYvXy5JGjdunL799lun/vDwcCUnJ0v6/ajp0KFDTv2RkZFKSkqSJN1xxx36+eefnfpv\nvPFGTZ8+XZJ0yy236MyZM079AwcO1GOPPSZJ6tOnj/5s+PDhmjhxogoKCtS/f3+X/qp8742ctUIn\n2/SU0dDHZVxNdHhlvFqOSba6DKBatN8x3/G91fu9zMxMTvMCF1KbghRAzccNSKiX2k992+oSKoQj\nU9QnefMGWF3CH7l1ZOrl6SqAmq4yP7gVuZv3lVde0ZgxY7Rv3z6n66YpKSkaO3asDhw4oODg4Is+\nRsSWmcqqWTsYAH/AaV7Aw2JiYuTt7a01a9Y4ta9evVphYWHlBimAmo8jU6AC1q1bJ0n6/PPPJUkb\nN25Us2bN1KxZM/Xu3VuS5OXlpTFjxujFF1+UJDVv3lwJCQlKSkpSQECAunbtqrVr12rr1q1KT0+3\n5oUAqFKEKVABw4YNc/r3xIkTJUm9e/d23KldUlKikpISp3Fz586Vv7+/Fi1apCNHjqhz585KTU3V\nwIEDq6VuAJ5FmAIV4M4Ne+cb07BhQyUmJioxMdETZQGwGNdMAQAwiTAFAMAkwhQAAJMIUwAATCJM\nAQAwiTAFAMAkwhQAAJMIUwAATCJMAQAwiTAFAMAkwhQAAJMIUwAATCJMAQAwiTAFAMAkwhQAAJMI\nUwAATCJMAQAwiTAFAMAkwhQAAJMIUwAATCJMAQAwiTAFAMAkwhQAAJMIUwAATCJMAQAwiTAFAMAk\nwhQAAJMIUwAATCJMAQAwiTAFAMAkwhQAAJMIUwAATCJMAQAwiTAFAMAkwhQAAJMIUwAATCJMUaVO\nnjyp2bNnKzw8XAEBAQoKClJ0dLQ2bNhgdWkA4DFeVheAuiMzM1N///vfdfjwYaf2999/X5mZmVqy\nZInGjx9vUXUA4DkcmaJKZGZmqn///jp8+LBiY2O1c+dO/fLLL/rss88UGRkpwzCUkJCgQ4cOWV0q\nAFQ5whSm/fTTTxoxYoTOnDmjZ555RqtWrVJERISaNGmi6667TuvXr5e/v78KCgr06quvWl0uAFQ5\nwhSmTZ48WUePHtXAgQM1efJkl/5mzZqpZ8+ekqRt27ZVd3kA4HGEKUz55ptvtGbNGtlsNj3zzDMX\nHNesWTNJUn5+fnWVBgDVhjCFKcuWLVNpaan69u2rq6666oLjzp075/RfAKhLCFNUWmlpqV577TVJ\n0p133nnRsSdOnJAk+fr6erwuAKhuhCkqbdeuXTpy5IgkKS4uTjab7YJf7777riSpbdu2VpZs2sGD\nBzV06FA1btxYl156qW6//XZ9//33bs290LbZtWuXh6sG4Gl8zhSVVpmbiTp16uSBSqpHQUGBoqOj\nZbfbtXLlStlsNiUmJuqGG27Q7t27dckll5T7GHFxcS6ftQ0JCfFUyQCqCWGKSvviiy8kSTfffLPe\nfPPNC47btGmTbrvtNklS165dq6U2T1ixYoUOHDigvXv3qmPHjpKka665Rp06ddKyZcuUkJBQ7mO0\nbt1aPXr08HSpAKoZp3lRad9++60kqV27dmrUqNEFv3bs2OGYc/311zs9xtq1axURESFfX181bdpU\nI0eOVF5eXnW+DLelp6erR48ejiCVpODgYPXs2VPr16+3sDIAViNMUWllqxkFBQVddNzbb78tSbrq\nqqvUrl07R/vSpUs1cuRINWrUSMnJyYqPj9eWLVsUGRlZI1dKys7OVlhYmEt7aGiocnJy3HqMJUuW\nyG63y8/PT9HR0frwww+rukwAFuA0LyrtzJkzkqRGjRpdcMw333yjr776SpI0evRoR/uJEyf0j3/8\nQ127dlVmZqa8vH5/K8bExKh79+5KTExUSkqK54qvhBMnTigwMNClPSgoSL/88ku582NjYzVw4EC1\natVK+fn5mj9/vqKjo/Xuu++qT58+HqgYQHXhyBSV5uPjI0k6ffr0BccsXrxY0u8fiRk7dqyj/c03\n39Svv/6qhx56yBGkkhQREaHrr79eaWlpKiws9FDl1li1apVGjBihXr16KTY2Vtu3b1erVq2UmJh4\n3vHLly9XRESEIiIidOzYsWquFkBFEKaotCuuuEKStHfv3vP25+bmavny5ZKkSZMmqUWLFo6+zz77\nTJIUFRXlMi8qKkoFBQVunzqtLoGBgec9Ar3QEWt5AgICNGDAAO3cufO8/ePGjVNWVpaysrIcK0gB\nqJkIU1Ra7969JUnvvPOOfvzxR6e+06dPa8SIESoqKlJISIhmzpzp1P/DDz9Iktq0aePyuGVtNe26\naWhoqLKzs13ac3Jy1KVLl0o/rs1mM1MWgBqAMEWljR07Vl5eXioqKtLgwYMdf3btnXfeUc+ePbVz\n504FBQUpLS3N5TOYBQUFkiS73e7yuGXXYMvG1BSDBw/Wjh07dODAAUdbXl6ePvroIw0ePLjCj3fq\n1CllZGSoe/fuVVkmAAsQpqi0q666ynHE+fnnn6t79+4KCgrSzTffrC+//FJ/+ctf9N577+maa65x\nmevn5ydJKioqcukru1ZaNqamuO+++9S+fXsNGTJE69evV3p6uoYMGaK2bds6LcSQn58vLy8vPfnk\nk462BQsW6L777tOrr76qzMxMrVy5Uj179tSRI0c0d+5cK14OgCpEmMKUxMRErV27VpGRkbr00ksV\nEBCgbt266ZlnntGePXsUHh5+3nmtW7eWdP5TuWVt5zsFbKVLLrlEW7duVUhIiEaPHq0777xTwcHB\n2rp1q/z9/R3jDMNQSUmJSktLHW2dO3dWTk6OHnroIfXr108JCQkKDg7W9u3b1atXLyteDoAqZDMM\noyLjKzQYuJCXXnpJ99xzj1auXKm77rrLqe+GG27QZ599pp9//vmiH7sxo/3Utx3f580b4JHnqEoR\nERHKysqyugygPnLrpgaOTGGJIUOGyN/fX4sWLVJxcbGjPSsrS9u2bdPQoUM9FqQAUNVYtAGWuOyy\nyzRv3jxNmjRJffr00ejRo3X8+HH961//UrNmzTRnzhyrSwQAtxGmsMwDDzygoKAgLViwQPHx8fLz\n81Pfvn2VlJRU6/9UG4D6hTCFpUaNGqVRo0ZZXQYAmMI1UwAATCJMAQAwiTAFAMAkwhQAAJMIUwAA\nTCJMUeOcO3dOffv21W+//WZ1KQDgFsIUNY63t7datWqlWbNmWV0KALiFMEWNtGDBAr3yyiv68ssv\nrS4FAMpFmKJGat68uZ566imNHz9eJSUlVpcDABdFmKLGuvvuu+Xl5aXly5dbXQoAXBRhihqrQYMG\nWrZsmWbMmKEjR45YXQ4AXBBhihotNDRU9957rx555BGrSwGACyJMUeNNnz5dn376qd555x2rSwGA\n8yJMUeP5+fnp+eef18SJE3XmzBmrywEAF4QpaoVbbrlFXbt21dy5c60uBQBcEKaoNZKTk7Vs2TJ9\n/fXXVpcCAE4IU9QarVq10syZMzVhwgQZhmF1OQDgQJiiVrn//vtVUFCglJQUq0sBAAfCFLVKw4YN\ntWzZMk2dOlXHjx+3uhwAkESYohbq2rWrRo0apcmTJ1tdCgBIIkxRS82ePVtbtmzRtm3brC4FAAhT\n1E4BAQFatGiRJkyYoKKiIqvLAVDPEaaotW677TZ17NhR8+fPt7oUAPUcYYpay2az6bnnnlNycrL2\n799vdTkA6jHCFLXaFVdcoSlTpmjixIl89hSAZQhT1Hrx8fE6cuSIXnvtNatLAVBPEaao9by9vbVs\n2TI9+uijOnnypNXlAKiHCFPUCZGRkRoyZIimTZtmdSkA6iHCFHVGUlKS1q9frx07dlhdCoB6hjBF\nndGkSRMtXLhQ48eP17lz56wuB0A9QpiiThk5cqQuv/xyLVq06Lz9X331VTVXBKA+IExRp9hsNi1e\nvFjz5s1Tfn6+U9///vc/9erVy6LKANRlhCnqnI4dOyo+Pl4PPvig02dPL730UhUUFKiwsNDC6gDU\nRYQp6qTJkydr3759evPNNx1tNptNzZs3108//WRhZQDqIsIUdZLdbtfSpUv10EMP6ddff3W0t2jR\ngjAFUOUIU9RZvXv3Vt++fTVjxgxHW4sWLXTkyBELqwJQFxGmqDPGjh2rNWvWOF0nnT9/vl599VV9\n8cUXkghTAJ5BmKLOuP/++7VgwQL17dtX33zzjSSpadOmmjdvnsaPH6+SkhJdfvnlnOYFUOUIU9QZ\n3bt3186dOzV48GD97W9/0/Tp03XmzBnFxcXJz89PS5Ys4cgUgEcQpqhTvLy89PDDD+vLL7/U3r17\nFRYWpk2bNmnp0qV64okn5OPjQ5gCqHKEKeqk1q1bKzU1Vc8//7wmTZqkGTNmaNSoUXrttddMneY9\nePCghg4dqsaNG+vSSy/V7bffru+//96tuYWFhZo8ebJatmwpX19fRUZG6oMPPqh0LQBqDsIUdVpM\nTIy++uorXXnllY4bkQ4cOFCpxyooKFB0dLS++eYbrVy5UqtWrdK+fft0ww036PTp0+XOv+eee7Ri\nxQo9+eSTysjIUMuWLXXzzTdr165dlaoHQM1h++Odj26o0GCgJtm7d69GjhypPXv2qM1j6x3tefMG\nuDV/0aJFSkhI0N69e9WxY0dJUm5urjp16qRnnnlGCQkJF5z75ZdfKjw8XC+99JLGjh0rSSouLlZo\naKg6d+6s9PT0iz53RESEsrKy3KoTQJWyuTOII1PUG507d9YXX3xR6VOr6enp6tGjhyNIJSk4OFg9\ne/bU+vXrLzLz97ne3t4aMWKEo83Ly0sjR47U5s2bVVRUVKmaANQMhCnqFZvNpqioqErNzc7OVlhY\nmEt7aGiocnJyyp0bHBwsPz8/l7lnz57V/v37K1UTgJqhQqd5Y2JijOPHj3uwnIo7duyYmjVrZnUZ\n9Upd2OZ7fvif4/urWzd2a84XX3yhyy+/XK1bt3Zq/+GHH3TkyBF169btgnO//fZblZaW6sorr3Rq\nP3XqlPbt26fOnTvL39/fqe/YsWMq+3krKipSeHi4W3WiatSF93ltUxO3+eeff77ZMIyY8sbV+mum\nXEuqfnVhm7ef+rbje3evmfr4+CghIUHz5s1zak9MTNS8efNUXFx8wbk33XSTTp06pR07dji1b9my\nRf369dMHH3xw0T8Pd8kll7h1kxOqTl14n9c2NXSbc80UqEqBgYH65ZdfXNpPnDihwMDASs+VpKCg\noKopEoAlCFPATaGhocrOznZpz8nJUZcuXcqdm5ubq4KCApe5Pj4+Tjc1Aah9an2Yjhs3zuoS6p36\nus0HDx6sHTt2OH1ONS8vTx999JEGDx580bmDBg3SuXPnlJaW5mgrLi7W2rVrddNNN8lut190ftOm\nTc0Vjwqrr+9zK9XmbV7rr5kClVGZa6anT5/WtddeK19fX82ZM0c2m03Tp0/Xr7/+qt27dztuIMrP\nz1eHDh00Y8YMpz//VvYxmPnz5ys4OFhLlixRRkaGPv74Y3Xt2vWiz11DryUB9QHXTIGqdMkll2jr\n1q0KCQnR6NGjdeeddyo4OFhbt251uhPXMAyVlJSotLTUaf7LL7+ssWPHKjExUQMGDNDBgwe1adOm\ncoMUQM3HkSnqpcocmVqJI1PAMnXvyPTZZ5/VoEGD1LJlS9lsNs2aNatC87dv366oqCj5+vqqRYsW\nSkhI0JkzZzxTbB1RWlqqpKQktW/fXo0aNdK1116r119/3a25cXFxstlsLl/x8fEerrp2YNH86mdm\nm5/vvWyz2VhbuRyHDh3Sgw8+qMjISPn5+clmsykvL8+tuWb2P9WtVoXpihUrdPToUd16660Vnrt7\n927169dPzZs3V0ZGhubMmaOXX35ZcXFxVV9oHTJ9+nTNmjVLkyZN0saNG9WjRw8NGzZMGzZscGt+\ns2bN9Mknnzh9PfLIIx6uuuZj0fzqZ3abS7//gvjn93NISIiHK6/d9u/fr9TUVAUGBl70s9TnY3b/\nU60Mw6jIl6VKSkoMwzCMc+fOGZKMmTNnuj331ltvNTp27GicPXvW0bZy5UpDkvH5559Xdal1wk8/\n/WT4+PgYM2bMcGqPjo42rr766nLnjxkzxmjdurWnyjPliikZji8rJCcnGw0aNDD27dvnaDtw4IDR\nsGFDY+HChS7ju3Xr5vh+165dhiTjpZdecrSdO3fOCAkJMQYNGuTZwmuxim7zP5Nk/POf//RkiXVS\n2X7bMAxjxYoVhiQjNze33Hlm9z9VyK18rFVHpg0aVK7cc+fOadOmTRo+fLi8vb0d7cOHD5ePj0+5\ni5TXV5s3b9bZs2cVGxvr1B4bG6s9e/YoNzfXospqPxbNr35mtjkqr7L77dq2/6lVYVpZ3333nQoL\nC10WKW/UqJE6dOhQ7iLl9VV2drbsdrvLggKhoaGS5NZ2O3r0qJo2bSovLy+FhITo6aefVklJiUfq\nrU1YNL/6mdnmZZYsWSK73S4/Pz9FR0frww8/rOoy8X+qYv9TnbysLqA6lC3Zdr4l34KCghz9cHbi\nxAk1adJENpvzzWxlS9+Vt93Cw8PVrVs3hYaGqrCwUG+88YamTZumffv26YUXXvBY3bXBhZYgDAoK\nOu+yg+7OLeuHKzPbXPr9iGjgwIFq1aqV8vPzNX/+fEVHR+vdd99Vnz59PFBx/WZ2/1PdLAvTsgW+\ny9O7d29lZmZ6vqB6oLq3+Z/v2u3fv7/8/f2VnJysKVOmqFOnTqafA6guq1atcnzfq1cvDRkyRGFh\nYUpMTNT27dstrAw1gWVhGhUVpa+//rrccX8+lVUZZb+NXmih8bLTBnVdRbd5YGCgTp48KcMwnH47\nNLM4+6hRo5ScnKysrKx6HaZmF83Pz88/71yJRfMvxMw2P5+AgAANGDBAL774YlWUhz/xxP7HkywL\nUz8/P5e/7egpHTp0kN1ud1mkvLCwUAcOHNCwYcOqpQ6rVXSbh4aGqqioSN99953TdYuyaxXlLe5+\nMX8+dVPfmF00/4033lBBQYHTL5ssmn9xZrb5xdT397KneHL/4wn14gYkHx8fxcTEKDU11elvTq5b\nt05FRUXlLlJeX8XExMjb21tr1qxxal+9erXCwsIUHBxc4cdcs2aNbDabrrvuuqoqs1ayctH8+srM\nNj+fU6dOKSMjQ927d6/KMvF/PLH/8Sh3P0Nj1IDPme7cudNIS0sz1q5da0gyhg0bZqSlpRlpaWnG\n6dOnHePuvvtuo2HDhk5z//vf/xp2u9247bbbjC1bthgvvPCCERgYaAwdOrS6X0atMmXKFMNutxsL\nFy403n//fWPChAmGzWYz3nrrLadx0dHRRocOHRz/zsvLM3r16mU8//zzxubNm4309HRj7Nixhs1m\nMyZMmFDdL8OF1Z8z/e2334wOHToYYWFhxptvvmmsX7/euOaaa4zg4GDj119/dYzLy8szGjZsaLRq\n1cpp/ogRI4wmTZoYK1asMLZs2WLccccdht1u5zPTF1HRbf7EE0842ubPn2/ce++9xpo1a4z333/f\nSElJMcLCwgxvb2/jgw8+sOLl1Cpl++kJEyYYkozFixcbaWlpRmZmpmNMw4YNjbvvvttpnrv7Hw9z\nKx9rVZiOGTPG0O/rA7t8/fFDwGXj/mzbtm1Gjx49DLvdbjRv3tx4+OGHnUIYroqLi43Zs2cb7dq1\nM3x8fIyrr77aSEtLcxnXu3dv44orrnD8++effzaGDBlitGvXzrDb7Yavr6/x17/+1fjPf/7j9CFu\nq1gdpoZhGPn5+cbtt99uBAQEGP7+/saQIUNcPsyem5trSDJatmzp1F5QUGA88sgjxuWXX27Y7Xaj\ne/fuxvvvv199xddSFdnmf1wUJj093YiKijIuu+wyw8vLywgKCjIGDRpkfPrpp9X7AmqpC+23e/fu\n7TRmzJgxTvPc3f94mFv5yEL3qJdY6B6Am+reQvcAANREhCkAACYRpgAAmESYAgBgEmEKAIBJhCkA\nACYRpgAAmESYAgBgEmEKAIBJhCkAACYRpgAAmESYAgBgEmEKAIBJhCkAACYRpgAAmESYAgBgEmEK\nAIBJhCkAACYRpgAAmESYAgBgEmEKAIBJhCkAACYRpgAAmESYAgBgEmEKAIBJhCkAACYRpgAAmESY\nAgBgEmEKAIBJhCkAACYRpoCbSktLlZSUpPbt26tRo0a69tpr9frrr7s1Ny4uTjabzeUrPj7ew1UD\nqA5eVhcA1BbTp0/XggULNHfuXHXr1k2vvfaahg0bpoyMDPXv37/c+c2aNVN6erpTW8uWLT1VLoBq\nRJgCbjh69KgWLFigqVOn6rHHHpMk3XDDDdq/f7+mTp3qVpj6+PioR48eni4VgAU4zQu4YfPmzTp7\n9qxiY2Od2mNjY7Vnzx7l5uZaVBmAmoAwBdyQnZ0tu92ujh07OrWHhoZKknJycsp9jKNHj6pp06by\n8vJSSEiInn76aZWUlHikXgDVi9O8gBtOnDihJk2ayGazObUHBQU5+i8mPDxc3bp1U2hoqAoLC/XG\nG29o2rRp2rdvn1544QWP1Q2gehCmqPf+HJB/1Lt3b2VmZpp+jj/ftdu/f3/5+/srOTlZU6ZMUadO\nnVzmLF++XMuXL5ckHTt2zHQNADyHMEW99/XXX1+wz8/PT5IUGBiokydPyjAMp/AtOyItO0KtiFGj\nRik5OVlZWVnnDdNx48Zp3LhxkqSIiIgKPz6A6kOYot678soryx0TGhqqoqIifffdd07XTcuulXbp\n0qXSz3+xI2MAtQM3IAFuiImJkbe3t9asWePUvnr1aoWFhSk4OLjCj7lmzRrZbDZdd911VVUmAItw\nZAq4oXnz5kpISFBSUpICAgLUtWtXrV27Vlu3bnVZiOHGG29Ufn6+9u/fL0nKz8/X6NGjNXLkSHXs\n2FFFRUV64403lJKSovHjx6tDhw5WvCQAVYgwBdw0d+5c+fv7a9GiRTpy5Ig6d+6s1NRUDRw40Glc\nSUmJiouLHf8OCAhQUFCQnn76af30009q0KCBrrzySv373//WxIkTq/tlAPAAm2EYFRlfocFATdV+\n6tuO7/PmDbCwEvdEREQoKyvL6jKA+sitmxq4ZgoAgEmEKQAAJhGmAACYRJgCAGASYQoAgEmEKQAA\nJhGmAACYRJgCAGASYQoAgEmEKQAAJhGmAACYRJgCAGASYQoAgEmEKQAAJhGmAACYRJgCAGASYQoA\ngEmEKQAAJhGmAACYRJgCAGASYQoAgEmEKQAAJhGmAACYRJgCAGASYQoAgEmEKQAAJhGmAACYRJgC\nAGASYQoAgEmEKQAAJhGmAACYRJgCAGASYQoAgEmEKQAAJhGmAACYRJgCAGASYQoAgEmEKQAAJhGm\nAACYRJgCAGASYQoAgEmEKQAAJhGmAACYRJgCAGASYQq46dlnn9WgQYPUsmVL2Ww2zZo1q0Lzt2/f\nrqioKPn6+qpFixZKSEjQmTNnPFMsgGpFmAJuWrFihY4ePapbb721wnN3796tfv36qXnz5srIyNCc\nOXP08ssvKy4uruoLBVDtvKwuAKgtsrOz1aBBAxUXF2vp0qUVmjtz5ky1adNGaWlp8vb2liT5+Pho\nzJgxmjJlirp27eqJkgFUE45MATc1aFC5H5dz585p06ZNGj58uCNIJWn48OHy8fHR+vXrq6pEABYh\nTAEP++6771RYWKiwsDCn9kaNGqlDhw7KycmxqDIAVaVCp3n79OmjuLg4xcXF6fjx4xo6dKjLmPvv\nv18jRozQwYMHNXr0aJf+Rx99VIMGDdLevXs1fvx4l/7ExET17dtXu3btUnx8vEv/U089paioKH38\n8cf6e/ovFSkfOK8+ffpIqth7T5JSUlKUmZnp6E9OTlZ4eLi2bNmiOXPmONpPnTolSTp9+rQk6a23\n3tLChQslST/88IMOHz6sPn36aNWqVWrbtq3Wrl2rJUuWOD333r17dfz4cTVt2lQpKSlKSUlxqW/D\nhg3y8/PT4sWLlZqa6tJfVus/kmdLAAAC5UlEQVSCBQuUkZHh1Ofr66uNGzdKkmbPnq333nvPqf+y\nyy7T66+/LkmaNm2aPvnkE6f+Nm3aaPXq1ZKk+Ph47dq1y6k/JCREy5cvlySNGzdO3377rVN/eHi4\nkpOTJUmxsbE6dOiQU39kZKSSkpIkSXfccYd+/vlnp/4bb7xR06dPlyTdcsstLjd2DRw4UI899pik\n////+4+GDx+uiRMnqqCgQP3793fpr0n7vccff9yl/0LvvTLLli1T586dnd57f3Sx954krVu3rt6+\n9/74M34xXDNFvVZaVKBt27ZJkuO/f9S4cWOFh4dXd1mSpMOHD+vHH3+U9PupYgA1l80wjIqMr9Bg\nT2s/9W2rS0At5utlU2x4kO4Ia3LBMX5+fmrXrp1TW3Fxsby9vTVz5ky3Ph7z9ddfq0uXLnr11Vc1\natQop74uXbooNDRUaWlpF32MiIgIZWVllftcAKqczZ1BtfrING/eAKtLAMrVoUMH2e12ZWdnO7UX\nFhbqwIEDGjZsmEWVAagq3IAEeJiPj49iYmKUmpqq4uJiR/u6detUVFSkwYMHW1gdgKpQq49MgeqU\nlZWlvLw8lZaWSpJycnK0bt06SVL//v3l5+cnSbrnnnu0cuVKp+CcNWuWevTooeHDh+uBBx5QXl6e\nJk+erKFDh6pbt27V/2IAVCnCFHDTc889p5UrVzr+nZaW5rjWmZubq/bt20uSSkpKVFJS4jQ3PDxc\n77zzjqZMmaIBAwaocePGuuuuu/TUU09VW/0APKdW34AE1BfcgARYxq0bkLhmCgCASYQpAAAmEaYA\nAJhEmAIAYBJhCgCASYQpAAAmEaYAAJhEmAIAYBJhCgCASYQpAAAmEaYAAJhEmAIAYBJhCgCASYQp\nAAAmEaYAAJhU0b9nCsACNpttk2EYMVbXAeD8CFMAAEziNC8AACYRpgAAmESYAgBgEmEKAIBJhCkA\nACYRpgAAmESYAgBgEmEKAIBJhCkAACb9Py+Gd+rpbPBzAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f8808605750>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "bipolar_plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Perceptron – schemat\n",
    "\n",
    "<img src=\"perceptron.png\" />"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Perceptron – zasada działania\n",
    "\n",
    "1. Ustal wartości początkowe $\\theta$ (wektor 0 lub liczby losowe blisko 0).\n",
    "1. Dla każdego przykładu $(x^{(i)}, y^{(i)})$, dla $i=1,\\ldots,m$\n",
    "    * Oblicz wartość wyjścia $o^{(i)}$:\n",
    "    $$o^{(i)} = g(\\theta^{T}x^{(i)}) = g(\\sum_{j=0}^{n} \\theta_jx_j^{(i)})$$\n",
    "    * Wykonaj aktualizację wag (tzw. _perceptron rule_):\n",
    "    $$ \\theta := \\theta + \\Delta \\theta $$\n",
    "    $$ \\Delta \\theta = \\alpha(y^{(i)}-o^{(i)})x^{(i)} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "$$\\theta_j := \\theta_j + \\Delta \\theta_j $$\n",
    "\n",
    "Jeżeli przykład został sklasyfikowany **poprawnie**:\n",
    "\n",
    "* $y^{(i)}=1$ oraz $o^{(i)}=1$ : $$\\Delta\\theta_j = \\alpha(1 - 1)x_j^{(i)} = 0$$\n",
    "* $y^{(i)}=-1$ oraz $o^{(i)}=-1$ : $$\\Delta\\theta_j = \\alpha(-1 - -1)x_j^{(i)} = 0$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Czyli: jeżeli trafiłeś, to nic nie zmieniaj."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "$$\\theta_j := \\theta_j + \\Delta \\theta_j $$\n",
    "\n",
    "Jeżeli przykład został sklasyfikowany **niepoprawnie**:\n",
    "\n",
    "* $y^{(i)}=1$ oraz $o^{(i)}=-1$ : $$\\Delta\\theta_j = \\alpha(1 - -1)x_j^{(i)} = 2 \\alpha x_j^{(i)}$$\n",
    "* $y^{(i)}=-1$ oraz $o^{(i)}=1$ : $$\\Delta\\theta_j = \\alpha(-1 - 1)x_j^{(i)} = -2 \\alpha x_j^{(i)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Czyli: przesuń wagi w odpowiednią stronę."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Perceptron – zalety i wady\n",
    "\n",
    "Zalety:\n",
    "* intuicyjny i prosty\n",
    "* łatwy w implementacji\n",
    "* jeżeli dane można liniowo oddzielić, algorytm jest zbieżny w skończonym czasie"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Wady:\n",
    "* jeżeli danych nie można oddzielić liniowo, algorytm nie jest zbieżny"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "slideshow": {
     "slide_type": "notes"
    }
   },
   "outputs": [],
   "source": [
    "def plot_perceptron():\n",
    "    plt.figure(figsize=(12,3))\n",
    "\n",
    "    plt.subplot(131)\n",
    "    plt.ylim(-0.2,1.2)\n",
    "    plt.xlim(-0.2,1.2)\n",
    "\n",
    "    plt.title('AND')\n",
    "    plt.plot([1,0,0], [0,1,0], 'ro', markersize=10)\n",
    "    plt.plot([1], [1], 'go', markersize=10)\n",
    "\n",
    "    ax = plt.gca()\n",
    "    ax.spines['right'].set_color('none')\n",
    "    ax.spines['top'].set_color('none')\n",
    "    ax.xaxis.set_ticks_position('none')\n",
    "    ax.spines['bottom'].set_position(('data',0))\n",
    "    ax.yaxis.set_ticks_position('none')\n",
    "    ax.spines['left'].set_position(('data',0))\n",
    "\n",
    "    plt.xticks(np.arange(0, 2, 1.0))\n",
    "    plt.yticks(np.arange(0, 2, 1.0))\n",
    "\n",
    "\n",
    "    plt.subplot(132)\n",
    "    plt.ylim(-0.2,1.2)\n",
    "    plt.xlim(-0.2,1.2)\n",
    "\n",
    "    plt.plot([1,0,1], [0,1,1], 'go', markersize=10)\n",
    "    plt.plot([0], [0], 'ro', markersize=10)\n",
    "\n",
    "    ax = plt.gca()\n",
    "    ax.spines['right'].set_color('none')\n",
    "    ax.spines['top'].set_color('none')\n",
    "    ax.xaxis.set_ticks_position('none')\n",
    "    ax.spines['bottom'].set_position(('data',0))\n",
    "    ax.yaxis.set_ticks_position('none')\n",
    "    ax.spines['left'].set_position(('data',0))\n",
    "\n",
    "    plt.title('OR')\n",
    "    plt.xticks(np.arange(0, 2, 1.0))\n",
    "    plt.yticks(np.arange(0, 2, 1.0))\n",
    "\n",
    "\n",
    "    plt.subplot(133)\n",
    "    plt.ylim(-0.2,1.2)\n",
    "    plt.xlim(-0.2,1.2)\n",
    "\n",
    "    plt.title('XOR')\n",
    "    plt.plot([1,0], [0,1], 'go', markersize=10)\n",
    "    plt.plot([0,1], [0,1], 'ro', markersize=10)\n",
    "\n",
    "    ax = plt.gca()\n",
    "    ax.spines['right'].set_color('none')\n",
    "    ax.spines['top'].set_color('none')\n",
    "    ax.xaxis.set_ticks_position('none')\n",
    "    ax.spines['bottom'].set_position(('data',0))\n",
    "    ax.yaxis.set_ticks_position('none')\n",
    "    ax.spines['left'].set_position(('data',0))\n",
    "\n",
    "    plt.xticks(np.arange(0, 2, 1.0))\n",
    "    plt.yticks(np.arange(0, 2, 1.0))\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAArMAAADJCAYAAAAwwbqVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAFkxJREFUeJzt3X9w1Hedx/HXO7ggIbc0tFx7tmfh\nAH8AB9bGG7BSbKt3MNdmOhj8MQdqO9Z2MnJeyOjoHfXsDHPlqhxU53rl8AcYblS6dYSxRY/rSVUs\no6TVOEEqBFKg2iqlV0hi2y187o/PF9hsNsl+k3x395M8HzPf2fL9fva7n/32886+9vv97vdrzjkB\nAAAAIaoqdwcAAACAoSLMAgAAIFiEWQAAAASLMAsAAIBgEWYBAAAQLMIsAAAAgkWYBQAAQLAIs2Vg\nZg+bmTOzgwO0cdH0pJlZgeV10fItefO35DzXmVnWzE6Z2a+iZTeb2bgE3hYwppjZe80sY2bPmtkr\nZnbSzH5oZh83s9cVaN+ZV5tnzewPZrbLzP6mHO8BGC3MrDmqq3/vZ/lfmFmXmR03s8l5y95iZpvM\n7LCZ/dHMTkefvZ83s0v6Wd+2vHo+Z2YvmdkTZnaXmZGvSsi4aUJpmdllkn4r6XWSTNK7nHN7C7TL\n/R/zIefct/KW10n6uaStzrmP5szfIukjkjZHr1MlKS3prZIWSZoo6WeSPuCc6xyp9wWMFVFQ3STp\ndkldkr4n6aikSyUtlfTnkvZLutk593zO8zolvUHSv0SzJkiaLelm+Tpd6ZzbVpp3AYwuUXj8saSF\nkm5yzv0wZ5lJ2iPpeklLnHM/yFnWKOlL0T93S2qTr83Fkt4m6feSbnHO/Szv9bZJ+jv5vwXPydfw\n1ZKWSaqRtMk5d9eIv1EU5pxjKuEkqUmSk/TF6PGr/bRzkn4nqVvSIUmvy1teF7XZkjd/SzS/rsA6\nL5XUEi0/KKmm3NuDiSm0SdK/RTX0hKQr8pZNkPQfOctTOcs6JXUVWN/7o/bPlPu9MTGFPEl6k6Qe\n+S+XNTnz/yGqsc157ZflfNZeW2B9d0h6TdJJSVflLdsWPfdtefNnRZ/b5yRdXe5tMlYmdoOX3m2S\nzkj6nKR2Se83s0n9tH1B0v2SZsoX1bA4516Q9GFJ/yPpzZI+Mdx1AmOJmb1Z0ifla7PeOfdc7nLn\n3CuSGiX9SNIC+aMkg3lIfg/vG81s6sj2GBg7nHO/kbRG0jRJX5AkM3uT/NGQY5Kaz7c1s5T856vk\nj1S2FljfZkn3ye8IWltkHw5J+on8kde3D/GtICbCbAmZ2Tsk/aWk7zjneuT3ktbI75npz79KOiXp\nc2ZWPdw+OP/V8fxhzoFeF0BfH5H/u/mfzrk/FGqQV2O3xVx/dhh9AyBtlPRTSXea2V9L2ip/et3H\nnHOnc9q9V9JVkn7snPvRAOv7oqRXJX3QzF4fsy/Uc4kQZkvr9uixJXr8L/lDEbcXbi45516StE7S\nFfKHSkbCT+UPncwv9EMVAP1aGD0+Nki7H8nX2DuK+MHlcvkvte3Ouf8bZv+AMc05d07+S+TLknbK\nHyHZ5Jzbnde0qFp2zp2S9Av5U4gG3dNqZrMkvUvSK/LnzqMECDIlEn2j+6CkZyX9UJKccyfMbI+k\nG81sVnR4opAvS/p7SZ82swej4hoy59wrZvaCpMslTZE/wR3A4K6IHk8M1Mg598ecGrtUF2tsvJl9\n/vx/y/8A7Bb5c+waR7y3wBjknPuNmW2VdJf8kc1PFWhWVC1Hjkv6q5zn5LrLzM7/AOyNkt4nqVpS\nU/5pSEgOYbZ03ifpEvkT0M/lzG+RdKP83tnPFnqic+5lM7tH/goFn1XhwgRQ+VKS/jlvXo/8L6x/\nUob+AKOOmV0tf6UBye+wuV7SIwm93J0F5n3SOfelAvOREE4zKJ3zpxLkX3rnYfkPsw8Pcjjy6/JX\nIPiEmV01nI6Y2QT5vUVn5b+1AijO+T0tA9ZgdCTmUvlz7V7IWdTtnDPnnEmaLOkD8qcaPTzcugZw\n4TJcX5X0J/Kn5v1R0qb8a8uqyFrOa/O7Asuuiep5ovzlLw9KWh+dr4sSIcyWgJlNk3RD9M9f5l5o\nWdJp+UMSb5C0pL91OOfOyv9K8/WSPj/MLr1Tfq/8L51zrw1zXcBY8kT0eNMg7a6Xr7GfR7Xbh3Pu\ntHNuu6SPSfpTSQUv9g4glrvk63OLc+5+Sf8k6UpJG/LaFVXLZlYrf73ZVyQ91V8759zL0dGVm+V3\nFH3VzCYO6R0gNsJsadwmf5mOx+W/MeZP34na9ftDMElyzj0sf8ODj0p6y1A6En1r/cfon98eyjqA\nMewb8teWvMPMLi3UIKqx86cMfX2wFTrnvi1pn6R6M3vnSHUUGGuiHUf3yd8wqCmafb+kvZJuy7vT\n3m7537AsMrN3DbDaZvkff33TOffyYH1wznXI/87lKkmrYr4FDBFhNmHRXUk+In8ocYVz7mP5k/wl\nsn4n6ZboDmED+aykcRrC3lkzmyJ/mZL3SHpa0gNx1wGMZc65g/J3C7pM0g4zuzx3uZmNl/8ge7d8\nQP1Gkau+J+8RQAzRl8ivyV8Z5OPnrwwS/UbldvnTDTabWTqan9XFwLvdzK4psM7bJX1G/lShu2N0\n5wvypw9+ysxqhvaOEAc/AEveTfK3uPuBc67gryadc2fNrEXSpyWtVN/DIblt/9fM/lvSYOfjfNzM\nbpbfI5yW35O7WL1vZ9sV980A0Kck1crfgOSQmeXfzvaNklol3Rp9YA7KOfd9M/uZpPeY2SLn3I+T\n6TowajXKn863xTnX68de0dUN7pa/ZuwXFP1oyzn3kJl9Uv6ufj+PPlvb5K808m5J1+ji7WyLuerB\n+df7vZltkg/LqyTdO8z3hkGYv743kmJm35S/JNeHnHPfGqDdWyT9WtKvnHPzovNp251zcwu0fbv8\n9etM0lbn3Edzlm1R77sOnZW/49gJ+Q/YjKRH866oACCm6JDlnfLXsbxM/i5ebZK+Kelr+UHWzDol\nXeacK7inxsz+VtL3JO1xzt1QqA2AvsxsuqRfSXpJ0pxC12uOjpL+RP76su9xzj2Ws2y2fPC8Uf78\n2qykw/LXqd3onHuxwPq2yV8x4Rrn3C8KLL9C/ktuj6Rpzrkzw32f6B9hFgAAAMHinFkAAAAEizAL\nAACAYBFmAQAAECzCLAAAAIIVN8y6Uk5Lliwp6esxMQ1hqmQl3RbUK1MAUyUr6bagXpkCmIpW0Xtm\nT548We4uACgS9QqEg3rFaFLRYRYAAAAYCGEWAAAAwSLMAgAAIFiEWQAAAASLMAsAAIBgEWYBAAAQ\nLMIsAAAAgkWYBQAAQLAIswAAAAgWYRYAAADBIswCAAAgWIRZAAAABIswCwAAgGARZgEAABAswiwA\nAACCRZgFAABAsBINsydOnNCqVau0cOFCVVdXy8zU2dmZ5EsCGAZqFggH9Qp4iYbZw4cPa/v27aqt\nrdWiRYuKe1JHh9TYKKXTUmurf2xs9POBAjpOdajxkUal702r6p4qpe9Nq/GRRnWcYszEFbdmc7d9\n629b2fYYFPU6cqhXJC43k1VVVWwmM+dcnPaxGp87d05VVT4vf+UrX9Edd9yho0ePatq0aYWfsGuX\n1NAgZbNSNqs6SfslKZXyUyYjLV0apwsY5XYd2qWGhxqUPZtV9lz2wvxUVUqpcSlllme0dFaiY8aS\nXPkwxapXKV7N9tn2myTdWdJtj8BQrwOiXlFZ8jLZBaXLZEXXa6J7Zs8XWVE6OvxG6+npvdEk/++e\nHr+8wr4NoHw6TnWo4aEG9WR7en0wSlL2XFY92R41PNTAXocYiq1Ztj3iYsyMPOoViQksk1XOD8DW\nr++7wfJls9KGDaXpDyre+ifWK3t24DGTPZvVhn2MmZHGtkdcjJnyYdsjtsAyWeWE2W3bittwLS2l\n6Q8q3ra2bX32MuTLnsuqpY0xM9LY9oiLMVM+bHvEFlgmq5ww29U1su0w6nW9WtxYKLYdise2R1yM\nmfJh2yO2wDJZ5YTZmpqRbYdRr2Z8cWOh2HYoHtsecTFmyodtj9gCy2SVE2ZXrPC/jhtIKiWtXFma\n/qDirZi3QqmqgcdMqiqllfMYMyONbY+4GDPlw7ZHbIFlssoJs83NxW24pqbS9AcVr3lhs1LjBvkD\nPS6lpgWMmZHGtkdcjJnyYdsjtsAyWeJhNpPJKJPJqLW1VZK0a9cuZTIZPf74470bzpjhr1lWXd13\nA6ZSfn4m49sBkmZMmaHM8oyqU9V99jqkqlKqTlUrszyjGVMYM3EUU7Nse8TFmEkG9YpEBJbJEr1p\ngiSZFb7m7eLFi7Vnz56+Czo6/KUeWlpUd/q09qfTfjd2U1PFbDRUlo5THdqwb4Na2lrU9WqXasbX\naOW8lWpa0FSKP86j6iLsUryazd32p798WulV6VJuewSIeu0X9YrKk5PJ1NXlz5EtXSYrul4TD7PD\nUVdXp/3795fyJYG4Rt2H41BRrwgA9RqhXhGAyrgDGAAAAJAkwiwAAACCRZgFAABAsAizAAAACBZh\nFgAAAMEizAIAACBYhFkAAAAEizALAACAYBFmAQAAECzCLAAAAIJFmAUAAECwCLMAAAAIFmEWAAAA\nwSLMAgAAIFiEWQAAAASLMAsAAIBgEWYBAAAQLMIsAAAAgkWYBQAAQLAIswAAAAgWYRYAAADBIswC\nAAAgWIRZAAAABIswCwAAgGARZgEAABAswiwAAACCRZgFAABAsAizAAAACBZhFgAAAMEizAIAACBY\nhFkAAAAEizALAACAYBFmAQAAECzCLAAAAIJFmAUAAECwCLMAAAAIFmEWAAAAwSLMAgAAIFiEWQAA\nAASLMAsAAIBgEWYBAAAQLMIsAAAAgkWYBQAAQLAIswAAAAgWYRYAAADBIswCAAAgWIRZAAAABIsw\nCwAAgGARZgEAABAswiwAAACCRZgFAABAsAizAAAACBZhFgAAAMEizAIAACBYhFkAAAAEizALAACA\nYBFmAQAAECzCLAAAAIJFmAUAAECwCLMAAAAIFmEWAAAAwSLMAgAAIFiEWQAAAASLMAsAAIBgEWYB\nAAAQLMIsAAAAgkWYBQAAQLAIswAAAAgWYRYAAADBIswCAAAgWIRZAAAABIswCwAAgGARZgEAABAs\nwiwAAACCRZgFAABAsAizAAAACBZhFgAAAMEizAIAACBYhFkAAAAEizALAACAYBFmAQAAECzCLAAA\nAIJFmAUAAECwCLMAAAAIFmEWAAAAwSLMAgAAIFiEWQAAAASLMAsAAIBgEWYBAAAQLMIsAAAAgkWY\nBQAAQLAIswAAAAgWYRYAAADBIswCAAAgWIRZAAAABIswCwAAgGARZgEAABAswiwAAACCRZgFAABA\nsAizAAAACBZhFgAAAMEizAIAACBYhFkAAAAEizALAACAYCUeZo8fP66GhgZNnjxZ6XRay5Yt07Fj\nx/p/QkeH1NgopdNSa6t/bGz084FCcsdMVRVjZhioVySt41SHGh9pVPretKruqVL63rQaH2lUxynG\nTFzUK5IWSr2acy5O+1iNe3p6NH/+fE2YMEFr166VmWnNmjXq6elRW1ubJk2a1PsJu3ZJDQ1SNitl\ns6qTtF+SUik/ZTLS0qVxuoDRLm/MXFC6MWNJrnyYqFdUlF2HdqnhoQZlz2aVPXexXlNVKaXGpZRZ\nntHSWdRrMahXJC2kek00zN5///1avXq1nn76ac2cOVOSdPToUc2aNUv33XefVq9efbFxR4c0b57U\n03Nh1oViO6+6Wmprk2bMiNMNjFYFxkwfyY+ZUfPhSL0iSR2nOjTvwXnqyfZfr9WparXd1aYZU6jX\nwVCvSFJo9ZroaQY7d+7UggULLhSaJE2fPl3XXXedduzY0bvx+vW996wVks1KGzYk0FMEiTEzoqhX\nJGn9E+uVPTvwmMmezWrDPsZMMahXJCm0ek00zLa3t2vu3Ll95s+ZM0cHDhzoPXPbtuKKraVlBHuI\noDFmRhT1iiRta9vW61BlIdlzWbW0MWaKQb0iSaHVa6zTDJYsWeJOnjxZdPsnn3xSl19+ua688spe\n85999lk999xzuvbaay/ObG3t8/xfS3proRXnPg9jV4Ex06+Exkxra+sPnHNLEln5MFGvqCStvy2y\nXk269s+o18FQr0hSaPWa6Dmz48eP1+rVq7Vu3bpe89esWaN169bptddeuzgznZbOnOnVrs85Pefb\nvfRSnG5gtCowZvptl9yYGTXn4FGvSFL63rTOvDp4vaYnpPXSZ6jXwVCvSFJo9ZroaQa1tbV68cUX\n+8w/deqUamtre89cscL/onIgqZS0cuUI9hBBY8yMKOoVSVoxb4VSVQOPmVRVSivnMWaKQb0iSaHV\na6Jhds6cOWpvb+8z/8CBA5o9e3bvmc3NxRVbU9MI9hBBY8yMKOoVSWpe2KzUuEE+HMel1LSAMVMM\n6hVJCq1eEw2z9fX12rdvn44cOXJhXmdnp/bu3av6+vrejWfM8Ne5q67uW3SplJ+fyXDZEFzEmBlR\n1CuSNGPKDGWWZ1Sdqu6zxydVlVJ1qlqZ5ZkkL/MzqlCvSFJo9ZroObPd3d2aP3++Jk6ceOGiznff\nfbfOnDmjtrY21dTU9H1SR4e/PEhLi+pOn9b+dNof+mhqotBQWM6YUVeXVFNTyjEzas7Bo15RCh2n\nOrRh3wa1tLWo69Uu1Yyv0cp5K9W0oKkUH4zUK/WKGEKp10TDrCQdO3ZMTU1N2r17t5xzuummm7Rx\n40ZNmzZt0OfW1dVp//4+p6gDlWTUfDhK1CtGPeo1Qr0iAJUTZoeDYkMARtWH43BQrwgA9RqhXhGA\nyriawVAcP35cDQ0Nmjx5sp566iktW7ZMx44dK3e3UKFOnDihVatWaeHChaqurpaZqbOzs9zdGjOo\nV8RBvZYX9Yq4QqnZigqzPT09uvHGG3Xw4EFt3bpV06dP16FDh3TDDTeou7u73N1DBTp8+LC2b9+u\n2tpaLVq0qNzdGVOoV8RFvZYP9YqhCKVmKyrMbt68WUeOHNF3v/td3Xrrrbrkkku0c+dOPfPMM9q0\naVO5u4cKdP311+v555/Xo48+quXLl5e7O2MK9Yq4qNfyoV4xFKHUbEWF2Z07d2rBggWaOXPmhXnT\np0/Xddddpx07dpSxZ6hUVVUVNYTHFOoVcVGv5UO9YihCqdmK6mV7e7vmzp3bZ/6cOXN04MCBMvQI\nQH+oVyAc1CtGs4oKswVvwydpypQpBW/bB6B8qFcgHNQrRrOKCrMAAABAHBUVZmtrawt+Q+zvGyWA\n8qFegXBQrxjNKirMzpkzR+3t7X3mHzhwQLNnzy5DjwD0h3oFwkG9YjSrqDBbX1+vffv26ciRIxfm\ndXZ2au/evaqvry9jzwDko16BcFCvGM0q6na23d3dmj9/viZOnKi1a9equblZkyZN0pkzZ9TW1qaa\nmpokXx6BymQykqTHHntMDz74oB544AFNnTpVU6dO1eLFi5N++TF7e0zqFUNBvfaLekVFKmPNFl2v\nFRVmJenYsWNqamrS7t271d3drVtuuUUbN27UtGnTkn5pBMqs8HhfvHix9uzZk/jLJ/0Cw0C9ouJQ\nr/2iXlGRyliz4YbZXHV1ddq/f38pXxKIa0x/OOaiXhEA6jVCvSIARddrRZ0zCwAAAMRBmAUAAECw\n4p5mUFJm9n3n3JJy9wPA4KhXIBzUK0aTig6zAAAAwEA4zQAAAADBIswCAAAgWIRZAAAABIswCwAA\ngGARZgEAABAswiwAAACCRZgFAABAsAizAAAACBZhFgAAAMH6f4EPquIjEl56AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f880643ba90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_perceptron()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Funkcje aktywacji\n",
    "\n",
    "Zamiast funkcji bipolarnej możemy zastosować funkcję sigmoidalną jako funkcję aktywacji."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "slideshow": {
     "slide_type": "notes"
    }
   },
   "outputs": [],
   "source": [
    "def plot_activation_functions():\n",
    "    plt.figure(figsize=(16,7))\n",
    "    plt.subplot(121)\n",
    "    x = [-2,-.23,2] \n",
    "    y = [-1, -1, 1]\n",
    "    plt.ylim(-1.2,1.2)\n",
    "    plt.xlim(-2.2,2.2)\n",
    "    plt.plot([-2,2],[1,1], color='black', ls=\"dashed\")\n",
    "    plt.plot([-2,2],[-1,-1], color='black', ls=\"dashed\")\n",
    "    plt.step(x, y, lw=3)\n",
    "    ax = plt.gca()\n",
    "    ax.spines['right'].set_color('none')\n",
    "    ax.spines['top'].set_color('none')\n",
    "    ax.xaxis.set_ticks_position('bottom')\n",
    "    ax.spines['bottom'].set_position(('data',0))\n",
    "    ax.yaxis.set_ticks_position('left')\n",
    "    ax.spines['left'].set_position(('data',0))\n",
    "\n",
    "    plt.annotate(r'$\\theta_0$',\n",
    "                 xy=(-.23,0), xycoords='data',\n",
    "                 xytext=(-50, +50), textcoords='offset points', fontsize=26,\n",
    "                 arrowprops=dict(arrowstyle=\"->\"))\n",
    "\n",
    "    plt.subplot(122)\n",
    "    x2 = np.linspace(-2,2,100)\n",
    "    y2 = np.tanh(x2+ 0.23)\n",
    "    plt.ylim(-1.2,1.2)\n",
    "    plt.xlim(-2.2,2.2)\n",
    "    plt.plot([-2,2],[1,1], color='black', ls=\"dashed\")\n",
    "    plt.plot([-2,2],[-1,-1], color='black', ls=\"dashed\")\n",
    "    plt.plot(x2, y2, lw=3)\n",
    "    ax = plt.gca()\n",
    "    ax.spines['right'].set_color('none')\n",
    "    ax.spines['top'].set_color('none')\n",
    "    ax.xaxis.set_ticks_position('bottom')\n",
    "    ax.spines['bottom'].set_position(('data',0))\n",
    "    ax.yaxis.set_ticks_position('left')\n",
    "    ax.spines['left'].set_position(('data',0))\n",
    "\n",
    "    plt.annotate(r'$\\theta_0$',\n",
    "                 xy=(-.23,0), xycoords='data',\n",
    "                 xytext=(-50, +50), textcoords='offset points', fontsize=26,\n",
    "                 arrowprops=dict(arrowstyle=\"->\"))\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA5IAAAGRCAYAAAAXeoyVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3XlYlPXex/HPALKjgOCSKwpu4I7m\ncsxSK4/rqaz0pE/aomZppi1Wbsc0ray0U25tltVJs0yzNFMz23zMXcFdUdEQFFFkh7mfPzyO8aAC\nMnAPM+/XdXHp73cv8wVHvny4N4thGAIAAAAAoKjczC4AAAAAAFC+ECQBAAAAAMVCkAQAAAAAFAtB\nEgAAAABQLARJAAAAAECxECQBAAAAAMVCkAQAAAAAFAtBEgAAAABQLARJAAAAAECxeBRzfaNUqgCc\nTPfu3bV69WqzywDKA4vZBTgBejNQBPRmoMiK1Js5IgmUgjNnzphdAgAA+At6M2BfBEkAAAAAQLEQ\nJAEAAAAAxUKQBAAAAAAUC0ESAAAAAFAsBEkAAAAAQLEQJAEAAAAAxUKQBAAAAAAUC0ESAAAAAFAs\nBEkAAAAAQLEQJAEAAAAAxUKQBAAAAAAUC0ESAAAAAFAsBEkAAAAAQLEQJAEAAAAAxUKQBAAAAAAU\nC0ESAAAAAFAsBEkAAAAAQLEQJAEAAAAAxUKQBAAAAAAUC0ESAAAAAFAsBEkAAAAAQLEQJAEAAAAA\nxUKQBAAAAAAUC0ESAAAAAFAsBEk4rfj4eI0cOVLt27eXr6+vLBaL4uLiirSt1WrV9OnTVbduXXl7\ne6t58+b68ssvS7dgAACcHL0ZcB4ESTitQ4cOacmSJQoKClKnTp2Kte2ECRM0efJkPfHEE1q1apXa\ntWune++9V999910pVQsAgPOjNwPOw2IYRnHWL9bKgJmsVqvc3C79ruS9997To48+qqNHj6pu3brX\n3S4xMVG1atXSuHHj9K9//cs237VrVyUlJWnXrl2FvnZ0dLS2bNlSovoBF2ExuwAnQG9GuUFvBsqF\nIvVmjkjCaV1uVMX1/fffKzs7WwMHDsw3P3DgQO3evVtHjx61R3kAALgcejPgPEw5InnrrbcWmLvv\nvvs0YsQIpaenq0ePHgWWDx48WIMHD9aZM2fUr1+/Assfe+wx3X///Tpx4oQGDRpUYPnYsWPVu3dv\n7d+/X8OGDSuwfPz48erWrZt27Nih0aNHF1j+8ssvq0OHDvrtt9/0wgsvFFg+a9YstWjRQmvXrtXU\nqVMLLJ8/f74aNmyob775Rq+//nqB5YsWLVKtWrW0ePFizZ07t8DypUuXKiQkRAsXLtTChQsLLP/u\nu+/k6+urOXPmaMmSJQWWb9iwQZI0c+ZMrVy5Mt8yHx8frVq1SpL00ksvad26dfmWV65c2XYNwvPP\nP6/ff/893/KaNWvqk08+kSSNHj1aO3bsyLe8QYMGWrBggSRp6NChOnDgQL7lLVq00KxZsyRdagjx\n8fH5lrdv317Tp0+XJN1zzz06e/ZsvuVdu3bVhAkTJEl///vflZGRkW95r169FBgYqEcffVQ333yz\nvL29JUnnq0crpWZHGe6esrc/Pxqt6g/Osvt+AUcSN6OnPXbDEcmSozfTm52mN1/2/997R44cUXx8\nvG655RZJV957a9as0Z133qmoqChVrlzZtv3V3ntbt25V69atJfHec/X33tNPPy3Jub7vXf43tZMi\n9WYPe74iUJ6UVogEAAD2lZubKw+Pgj+2BgYG2pZfzaeffqqtW7dKknJyckqvQMAFcY0kXMLVrsOo\nO+7bUns9jkjCFXBE0mHQm1EuFecayaFDh2rFihVKSEjIN3/o0CFFRETo448/vurRn7/iGkmgyDgi\nCRTVX38gfu655zR79mxlZGTIYrny/2jz5s26+eabtXLlSvXsef0foKPXTtIW+/yQDQCAywsKClJK\nSooMw8jXm5OTkyVJwcHBZpUGlLnsXKuS07J1Ni1L59JydDYtS8lp2WpeK1CtageVWR0ESeD/iYyM\nVFZWlg4fPqzw8HDbfGxsrCSpSZMmZpUGAIBLojfD2RmGofMZOTp9IUunL2Qq4UKmklKz8n2cuXjp\n40Lm1U/lHtUlnCAJmKl79+6qUKGCPv30U02aNMk2/8knnygqKkphYWEmVgcAgOuhN6O8y8zJ08mU\nDJ08l2H781RKhv48n6k/z1/6MyvXWqLXOJuWbadqi4YgCae2dOlSSbJdaL9q1SqFhoYqNDQ033oP\nP/yw3n//fUlSlSpVNGbMGE2fPl0BAQFq1aqVFi9erPXr12vFihVl+wkAAOBkrtebO3fuLEny8PDQ\ngw8+SG9GuZKSnq0jZ9J0/Gy64s6m6djZdB1PTteJ5HQlpmbZ7XXcLFKwn5cq+3kq+C8fN9cr21O8\nCZJwavfee2++8YgRIyTpUqNq94xtPi8vL99606ZNk7+/v2bPnq2EhAQ1bNhQS5YsUa9evUq/aAAA\nnNj1evPlRxjk5eXRm+GQrFZD8ecydDAxVYcSL+pQ4kUdOZOmI0kXdS69ZHcG9vN0V9VK3qoa4K1q\nlbwVGuClKgFeCg3wUqi/l0ICvBTi76VAnwpyczP/XnXctRUu6693bbXT3SdtuDMcUGTmd8Lyj94M\nFAG9GcWVkp6t2FMXFPvnBe1PSNX+06k6cDpVmTnFPwXVzSJVr+SjmkE+qhHko5qBProp0EfVA31U\nvZK3qlfyVoB3hVL4LG4Id20FAAAAgMIkp2VrZ3yKdsef1+6T5xV76oJOpmQUax/eFdxUt7Kf6lb2\nU50QX9Wt7Kfawb6qFeSr6oHequDuVkrVm4MgCQAAAMBlZOdaFXPqvLYfT9G24+e040SK4s8VPTRW\n9vNUeBV/RVT1V3iov+pX8Ve9UH9Vr+jtEKeclhWCJAAAAACndTErV1uPndMfR5O1OS5ZO0+kFOkO\nqZ7ubmpYLUCNqweoUbWKalQtQA2rBaiyv1cZVO34CJIAAAAAnEZmTp62Hjun3w6f0e+Hz2pn/Hnl\nWa9/Obmnh5uaVK+o5jUrqWnNQEXVqKj6of5OdzqqPREkAQAAAJRbhmHoUOJF/XQgSRsPntH/Hjlb\n6BHHOpV91ap2kFrWDlTLWkFqVD2A0FhMBEkAAAAA5UpWbp7+90iy1u9L1Lp9p3Ui+drXOFosUqNq\nFXVzWLDa1A1Wm7pBqlLRuwyrdU4ESQAAAAAOLy0rVxv2J2l1TIJ+3Jeoi1m511y3Xoif/hYRog71\nK+vmsMoK8vMsw0pdA0ESAAAAgENKz87Vur2J+mbnKW04kKTsa5yy6u/lob+Fh+iWBqG6pUGIagb5\nlnGlrocgCQAAAMBh5OZZtfFgkr7efkpr955WenbeVderU9lXXRtVVdfGVdSmbrA8PbjGsSwRJAEA\nAACYLvbUBX21LV5f7zilMxezrrpOo2oB6h5VTd2jqqlh1QBZLK7z3EZHQ5AEAAAAYIq0rFx9s/OU\n/rP5uHbGn7/qOuFV/NWn+U3q1ay66oX6l3GFuBaCJAAAAIAytT8hVR/9Hqfl208q7SqnrlYJ8NJd\nLWuob4saalydI4+OiCAJAAAAoNTl5lm1dm+iFv52VJuOJBdY7unupjujqqlf65rqWL+yPHiuo0Mj\nSAIAAAAoNenZuVryxwm9/+vRqz7vMbyKvwa0ra27W9bgMR3lCEESAAAAgN2dvZilj36L08ebjikl\nPSffMnc3i+6MrKoH29dV27BgTl0thwiSAAAAAOwmMTVT7248ok82HVdGTv7rHwN9K+iBm2vrgZvr\n6KZAH5MqhD0QJAEAAACUWGJqpub8eFj/2XxcWbnWfMtqB/vqkU5h6te6pnw9iSDOgH9FAAAAADfs\nfHqO5m08rA9/ParMnPwBsnH1inr8tvrqHlmNm+c4GYIkAAAAgGLLzMnT+78c1byfDis1MzffsqY1\nKmlU1wh1a1yF6x+dFEESAAAAQJFZrYaW7zyp11bv16nzmfmWNa5eUc/c2UC3NSRAOjuCJAAAAIAi\n2RKXrCkrY7Ur/ny++bAQP425vYF6Nq0uNzcCpCsgSAIAAAC4rqTULE1ftVdfbTuZb76yn6eeur2B\n+repxTWQLoYgCQAAAOCqcvOsWrTpmN5Yc0CpWVeug/T0cNMjfwvTY7fWV4B3BRMrhFkIkgAAAAAK\n2HPyvMZ9tUt7Tl7IN989sppe7NlYtYJ9TaoMjoAgCQAAAMAmIztPs9Yd0Hs/H1We1bDN1wvx0+Q+\nkbqlQaiJ1cFRECQBAAAASJL+98hZPfvlLh07m26b8/Rw05NdI/RIpzB5ebibWB0cCUESAAAAcHGZ\nOXma+f1+vf/rURlXDkKqXb1gTb+7mcJC/MwrDg6JIAkAAAC4sF3xKRqzZKcOJV60zQV4e2h8z8a6\nL7oWz4PEVREkAQAAABdktRqat/Gw3lhzQLl/uRbylgaheuWepqpeycfE6uDoCJIAAACAizl9IVNP\nLd6h3w6ftc35errrxZ6N9c+2tTkKiUIRJAEAAAAX8uO+RI1ZskPn0nNscy1rB2rW/S1UpzLXQqJo\nCJIAAACAC8izGnrjh/1658fDtjmLRXr81nA92S1CFdzdTKwO5Q1BEgAAAHBySalZevLz7flOZa1W\n0Vtv3N9cHeqHmFgZyiuCJAAAAODEth47pxGfbtXpC1m2uU4RIZrdv6WC/TxNrAzlGUESAAAAcFJf\nbDmhF5ftUXaeVdKlU1lHdYnQqK4Rcnfjhjq4cQRJAAAAwMnk5lk1fdU+vf/LUdtcoG8Fzbq/hW5t\nWMXEyuAsCJIAAACAEzmfkaMnPtumnw+esc01rBqg9x6MVq1gXxMrgzMhSAIAAABOIv5cuoZ8+IcO\nJl60zd3RpKreuL+F/L340R/2w7sJAAAAcAK74lP00MItOnPxyk11RnUJ1+huDeTG9ZCwM4IkAAAA\nUM79EHtao/6zXRk5eZIkT3c3vXZvM/VtUcPkyuCsCJIAAABAOfb55uN6YdluWY1L40o+FbRgUGvd\nXK+yuYXBqREkAQAAgHLIMAzN2XBYr32/3zZXO9hXHw5po/qh/iZWBldAkAQAAADKGavV0NRv9+qD\nX6883iOqRkUtHNJWIf5eJlYGV0GQBAAAAMqR3Dyrnvtyt77cFm+b61C/suYPaq0A7womVgZXQpAE\nAAAAyomcPKtGL96hb3f9aZvr0bSa3ry/hbw83E2sDK6GIAkAAACUA1m5eXris+36Ifa0ba5/m1qa\ndldTufN4D5QxgiQAAADg4DJz8jT8k63asD/JNje4Q11N6t1EFgshEmWPIAkAAAA4sMycPA1dtFUb\nD1wJkcM619O47o0IkTANQRIAAABwUFm5eRrx6bZ8IXJU1wg91S2CEAlTESQBAAAAB5Sda9Xjn27X\n+n2JtrnR3SI0ulsDE6sCLnEzuwAAAAAA+eXmWTXqP9u1du+VG+s8cVu4nuwaYWJVwBUESQAAAMCB\nWK2Gnl26S6tjEmxzwzrX09g7GnA6KxwGQRIAAABwEIZhaMrKWH21/aRt7uG/hXFjHTgcgiQAAADg\nIN5ce1ALf4uzjQe0raXxPRsTIuFwCJIAAACAA3j/l6N6a91B27hns+qa+o+mhEg4JIIkAAAAYLIV\nO0/ppZWxtnHnBqF6874WcncjRMIxESQBAAAAE/12+IzGLtlhG7euE6R5A1vL04Mf1eG4eHcCAAAA\nJtmXcEHDPt6qnDxDkhRexV/vPxgtH093kysDro8gCQAAAJjgVEqGBn/wh1KzciVJVQK89NFDbRXo\n62lyZUDhCJIAAABAGUvNzNFDC/9QwoVMSZK/l4cWDmmrGoE+JlcGFA1BEgAAAChDuXlWjfrPdu1L\nSJUkVXC3aP6g1mpyU0WTKwOKjiAJAAAAlKGp3+7Vj/uTbOPpdzdTx/AQEysCio8gCQAAAJSRRb/H\naeFvcbbxiFvrq1/rmqbVA9wogiQAAABQBjYeSNLkb648K7JH02p6+o6GJlYE3DiCJAAAAFDK4s6k\n6YnPtinPeukxH81rVtLr97aQm5vF5MqAG0OQBAAAAEpRamaOHvl4iy5kXnrMR7WK3nr3f3hWJMo3\ngiQAAABQSqxWQ08t3qlDiRclSZ4ebpo/qLWqVPQ2uTKgZAiScFonTpxQv379VKlSJVWsWFF33323\njh8/XqRtLRbLVT927NhRylUDAOC8XLE3z1p3UGv3nraNZ9zdVM1rBZpYEWAfHmYXAJSG9PR0denS\nRV5eXvroo49ksVg0fvx43Xbbbdq1a5f8/PwK3cfgwYM1bNiwfHMNGjQorZIBAHBqrtib18Qk6K11\nB23jRzuF6e5W3KEVzoEgCaf07rvv6siRI9q/f7/Cw8MlSc2aNVNERITmz5+vMWPGFLqPGjVqqF27\ndqVdKgAALsHVevPRM2kau2SnbdwpIkTPdW9kYkWAfXFqK5zSihUr1K5dO1ujkqSwsDB17NhRy5cv\nN7EyAABckyv15vTsXD32yValZl26uU6NQB+91b+lPNz50RvOg3cznFJMTIyioqIKzEdGRio2NvYq\nWxQ0d+5ceXl5ydfXV126dNHPP/9s7zIBAHAZrtKbDcPQC1/t1r6EVEmXbq4zb2BrBfl5mlwZYF8E\nSTil5ORkBQUFFZgPDg7WuXPnCt1+4MCBmjNnjtauXasFCxbo7Nmz6tKlizZs2HDNbRYsWKDo6GhF\nR0crKSmpJOUDAOB0XKU3L9p0TF/vOGUbv9Q3Uk1rViqT1wbKEtdIAlexaNEi2987deqkvn37Kioq\nSuPHj9cvv/xy1W2GDh2qoUOHSpKio6PLpE4AAFxFeejNu+JT9NLKK0dX+7eppfvb1C711wXMwBFJ\nOKWgoKCr/nbzWr8NLUxAQIB69uypP/74wx7lAQDgcpy9N5/PyNHjn21TTp4hSYqqUVGT+0SaXBVQ\negiScEqRkZGKiYkpMB8bG6smTZrc8H4tFktJygIAwGU5c282DEPjvtylE8kZkqQALw+9889W8q7g\nbnJlQOkhSMIp9enTR5s2bdKRI0dsc3Fxcfr111/Vp0+fYu/vwoULWrlypdq2bWvPMgEAcBnO3Js/\n/v2YVu1JsI1f6ddMdSoX/lxMoDyzGIZRnPWLtTJglrS0NDVv3lw+Pj6aOnWqLBaLJkyYoNTUVO3a\ntUv+/v6qO+5b2/oP+W7VxIkTJUkzZ87U/v37ddttt+mmm27SsWPHbHPr1q1Tp06dCn396Ohobdmy\npdQ+P8CJmH8oofyjN6NcKEpvlqRjx46pfv36mjhxYrnozbvjz+ueub8pO88qSfqf9nU0pW/Bu9MC\n5UiRejM324FT8vPz0/r16/XUU09p0KBBMgxDXbt21axZs2yN6q+sVqvt7w0bNtSyZcu0bNkynT9/\nXhUrVlTHjh31/vvvO8RvPQEAKI+K2psNw1BeXl656M1pWbka+Z9tthAZeVNFvdCjsWn1AGWJI5Jw\nWX89Ihk3o6dd980RSaDIOCJZcvRmoAhKozc//cVOLd0aL0ny83TXt6M6qW4Ip7Si3CtSb+YaSQAA\nAKCYVuw8ZQuRkjT1rihCJFwKQRJ2l5KSopdeekktWrRQQECAgoOD1aVLF3333XdmlwYAgEuiN9vX\nieR0vfjVbtv4rpY1dFfLmiZWBJQ9rpGEXW3YsEH//Oc/9eeff+ab//HHH7VhwwbNnTtXw4YNM6k6\nAABcD73ZvnLzrBq9eIdSs3IlSbWDfTWlL8+LhOvhiCTsZsOGDerRo4f+/PNPDRw4UH/88YfOnTun\nzZs3q3379jIMQ2PGjFF8fHzhOwMAACVGb7a/d348rK3HzkmSPNwsmt2/hQK8K5hcFVD2CJKwi9On\nT+v+++9XRkaGXn31VS1atEjR0dEKDAxUmzZttHz5cvn7+ys9PV2fffaZ2eUCAOD06M32t+NEit5a\nf9A2fur2BmpZO8jEigDzECRhF88884wSExPVq1cvPfPMMwWWh4aGqmPHjpKkn376qazLAwDA5dCb\n7Ss9O1dPLd6hPOulGyW3rRus4Z3rm1wVYB6CJEps3759+vTTT2WxWPTqq69ec73Q0FBJlx40DAAA\nSg+92f5e/m6vjp5JkyT5e3no9fuay92NJxjBdREkUWLz58+X1WpVt27d1LjxtR/Cm5OTk+9PAABQ\nOujN9vXj/kR9sum4bTypdxPVCvY1sSLAfARJlIjVatXnn38uSXrggQeuu25ycrIkycfHp9TrAgDA\nVdGb7etcWraeXbrLNu4eWU39WvOoD4AgiRLZsWOHEhISJEmDBw+WxWK55scPP/wgSapVq5aZJQMA\n4NTozfY1cUWMklKzJEkh/l56+e6mslg4pRUgSKJEbuTi/IiIiFKoBAAASPRme/pu95/6Zucp2/iV\ne5oq2M/TxIoAx0GQRIls27ZNknTnnXcqIyPjmh/Lli2zbdOqVSuzygUAwOnRm+3jzMUsjf96j218\nb+ua6tq4qokVAY6FIIkSOXDggCSpdu3a8vb2vubHpk2bbNvccsst+faxePFiRUdHy8fHRyEhIerf\nv7/i4uLK8tMAAMBp0JtLzjAMvbhst5LTsiVJ1St5a0LvJiZXBTgWgiRKJD4+XpIUHBx83fW+/fZb\nSVLjxo1Vu3Zt2/y8efPUv39/eXt7a9asWRo9erTWrl2r9u3b2/YNAACKjt5ccst3nNL3Madt41fu\naaaK3hVMrAhwPB5mF4DyLSMjQ5Lk7e19zXX27dunPXsunRoyaNAg23xycrKeffZZtWrVShs2bJCH\nx6W3Y/fu3dW2bVuNHz9eCxcuLL3iAQBwQvTmkklMzdSkFTG28T9vrq1bGoSaWBHgmDgiiRLx9Lx0\nwXlaWto115kzZ46kS7cWHzJkiG3+66+/VmpqqkaNGmVrVJIUHR2tW265RV988YUyMzNLqXIAAJwT\nvfnGGYahCV/v0fmMS8/VrBnkoxd6XPs5nIArI0iiROrUqSNJ2r9//1WXHz16VAsWLJAkPfHEE6pW\nrZpt2ebNmyVJHTp0KLBdhw4dlJ6ertjYWHuXDACAU6M337jvdicUOKXV34sT+ICrIUiiRDp37ixJ\nWrNmjU6dOpVvWVpamu6//35lZWWpQYMGmjRpUr7lJ0+elCTVrFnwob6X51zlWgwAAOyF3nxjktOy\nNXH5lbu0DmhbSx3DQ0ysCHBsBEmUyJAhQ+Th4aGsrCz16dNHf/zxh86dO6c1a9aoY8eO+uOPPxQc\nHKwvvvhCfn5++bZNT0+XJHl5eRXY7+XrOi6vAwAAiobefGP+9U2Mzv7lLq3Pc0orcF0ESZRI48aN\nbb/N3Lp1q9q2bavg4GDdeeed2rlzp+rVq6d169apWbNmBbb19fWVJGVlZRVYdvn6i8vrAACAoqE3\nF9/a2NNavuPK0duX72rKXVqBQhAkUWLjx4/X4sWL1b59e1WsWFEBAQFq3bq1Xn31Ve3evVstWrS4\n6nY1atSQdPVTZC7PXe3UGgAAcH305qJLzczR+K+vnNJ6d6sauq1RFRMrAsoHrh6GXdx333267777\nirVN27ZtNX/+fP3++++KiIjIt+z333+Xr6+vmjTh4b8AANwIenPRvLp6vxIuXDraGuLvqYm9nOvz\nA0oLRyRhmr59+8rf31+zZ89Wbm6ubX7Lli366aef1K9fv+s+AwsAANiXq/XmLXHJWrTpmG08qXek\nAn09TawIKD84IgnTVK5cWTNmzNATTzyhW2+9VYMGDdKZM2f05ptvKjQ0VFOnTjW7RAAAXIor9eas\n3Dw99+Uu27hroyrq1ay6iRUB5QtBEqZ6/PHHFRwcrJkzZ2r06NHy9fVVt27dNH36dNWqVcvs8gAA\ncDmu0pvn/HhYh5PSJEl+nu566R9RslgsJlcFlB8ESZhuwIABGjBggNllAACA/3L23nzwdKrmbDhk\nGz/390a6KdDHxIqA8odrJAEAAOAyrFZDLyzbrZw8Q5LUqnagBt5cx+SqgPKHIAkAAACXsWTLCf0R\nd06S5OFm0fS7m8nNjVNageIiSAIAAMAlJKVm6eXv9trGQ2+pp4bVAkysCCi/CJIAAABwCdO+jdWF\nzEuPNakd7KuRXSIK2QLAtRAk4bBOnDghq9VqdhkAAOC/ynNvvpiVq693nLKNX/pHlHw83U2sCCjf\nCJJwWKNHj9Y777xjdhkAAOC/Hn/8cc2fP9/sMootMydPJ89l2MZ9mt+kzg1CTawIKP8IknBY06ZN\n05QpU3Ty5EmzSwEAAJJefvllTZw4UQkJCWaXUixzfjyk7LxLR1IrentofK/GJlcElH8ESTisRo0a\nafjw4Ro9erTZpQAAAElRUVF6+OGH9dRTT5ldSpEdSbqoeT8dsY2f+3sjVQnwNrEiwDkQJOHQXnjh\nBW3fvl3fffed2aUAAABJEydO1KZNm7RmzRqzSymUYRiasHyP7Whki1qBGtCmtslVAc6BIAmH5uPj\no7lz5+rxxx9Xenq62eUAAODyfH199c4772jEiBHKyMgofAMTrdh5Sr8eOmsbT/1HFM+MBOyEIAmH\nd/vtt6t9+/aaMmWK2aUAAABJPXr0UMuWLTVt2jSzS7mmC5k5mvrtlWdGVvb3VFSNSiZWBDgXgiTK\nhTfeeEMffPCBdu/ebXYpAABA0uzZszVv3jzt3bu38JVN8MaaA0pKzZIkVa3opWoVuS4SsCeCJMqF\natWqacqUKRo2bFi5fX4VAADO5KabbtKkSZM0fPhwGYZhdjn57Dl5Xh//HmcbT+jVRG4WTmkF7Ikg\niXJj6NChMgxD7733ntmlAAAASSNGjFB6eroWLlxodik2Vquhicv3yPrfbNspIkQ9m1Y3tyjACREk\nUW64ublp/vz5Gj9+vE6fPm12OQAAuDx3d3fNnz9f48aN05kzZ8wuR5K0dFu8th1PkSRVcLdocp9I\nWTgaCdgdQRLlSrNmzfTggw9q7NixZpcCAAAktWrVSgMGDNAzzzxjdilKSc/WjFX7bOOht9RT/VB/\nEysCnBdBEuXO5MmT9csvv2jt2rVmlwIAACS99NJLWrt2rX766SdT65i5Zr+S07IlSTUCffT4beGm\n1gM4M4Ikyh0/Pz/9+9//1ogRI5SZmWl2OQAAuLyAgADNnj1bw4cPV1ZWlik17I4/r0//97htPKFX\nE/l6ephSC+AKCJIol3r37q3XKzF4AAAgAElEQVSoqChNnz7d7FIAAICku+66S+Hh4XrttdfK/LWt\nVkMTlu/R5ZvHdm4Qqjsjq5Z5HYArIUii3Hrrrbf0zjvvaP/+/WaXAgCAy7NYLHr77bc1a9YsHTp0\nqExfe+m2eO04cekGO57ubtxgBygDBEmUWzVr1tT48eMd8vlVAAC4ojp16ui5557TiBEjyqw3n8/I\n0St/ucHOo7eEKSzEr0xeG3BlBEmUa0888YTOnz+vRYsWmV0KAACQNHr0aCUkJOjzzz8vk9d784cD\nOvvfG+zcVMmbG+wAZYQgiXLNw8ND8+fP17PPPquzZ8+aXQ4AAC6vQoUKmj9/vsaOHauUlJRSfa29\nf17Qx7/H2cbjucEOUGYIkij32rRpo3vvvVfPPfec2aUAAABJ7du3V58+ffT888+X2msYhqFJy2Nk\n/e8ZtB3DK+vvUdVK7fUA5EeQhFOYOnWqVq9erZ9//tnsUgAAgKTp06dr+fLl2rRpU6nsf8XOU9oc\nlyxJ8nCz6F/cYAcoUwRJOIVKlSrpzTff1PDhw5WdnW12OQAAuLygoCDNnDlTw4YNU05Ojl33nZaV\nq5e/22sbD+lYV+FVAuz6GgCujyAJp9GvXz/VqVNHr7/+utmlAAAASQMGDFCVKlU0e/Zsu+737R8P\n6fSFLElSaICXRnWNsOv+ARSOIAmnYbFY9M477+j111/XkSNHzC4HAACXZ7FYNHfuXM2YMUPHjh2z\nyz6PnknTez9f6fPjujdSgHcFu+wbQNERJOFUwsLC9PTTT+vxxx/n2ZIAADiA8PBwPfnkkxo5cqRd\nevOUb2KUk3dpP61qB+quljVKvE8AxUeQhNMZO3asTpw4oS+++MLsUgAAgKRnn31WBw4c0Ndff12i\n/azfd1o/7k+SJFks0r/6RMnNjRvsAGYgSMLpXH5+1VNPPaXz588XWP7GG28oOTnZhMoAAHBNXl5e\nmjdvnkaNGqXU1NQCy1977bWr9uy/ysrN05RvYm3j/m1qqWnNSnavFUDRECThlDp27KgePXroxRdf\nLLBs2bJl2r17twlVAQDgum699VZ17dpVEydOLLDsq6++UkxMzHW3/+CXOMWdTZckVfT20NN3NCyV\nOgEUDUESTuuVV17R0qVLtXnz5nzz1apVU0JCgklVAQDgumbOnKnPPvtM27ZtyzdftWrV6/bm0xcy\n9e/1B23jMbc3UGV/r1KrE0DhCJJwWsHBwXrttdc0fPhw5ebm2uarVaum06dPm1gZAACuKSQkRNOn\nT9fw4cOVl5dnmy+sN7+yap/Ssy+t36Cqvwa2q1PqtQK4PoIknNrAgQMVFBSkt99+2zZX2G89AQBA\n6RkyZIi8vb01b94829z1zhbaeuycvtp+0jae1DtSHu78CAuYjf+FcCopKSnKysqyjS0Wi+bMmaOp\nU6cqPj5eEqe2AgBQlq7Wm+fNm6fJkyfr1KlTkq79S16r1dDkFVeunfx7VDV1DA8p/aIBFIogCafy\n/vvvKyoqSmvWrLHNNWzYUE888YRGjRoliVNbAQAoS/PmzVOzZs20bt0621yTJk00dOhQPfXUU5Ku\n3Zu/2HpCu09eupurl4ebXujRuGyKBlAogiScytixYzV79mwNHz5c/fv3t/2mc9y4cdqzZ4+++eYb\nTm0FAKAMjRs3TjNnztTDDz+sBx54wNaDx48fry1btmj16tVXPVvofEaOXl293zYe1rm+agX7lmnt\nAK6NIAmn06NHD+3Zs0f169dX8+bN9e9//1sVKlTQvHnzNHLkSFWsWJEgCQBAGerdu7diYmJUq1Yt\nNW3aVO+88448PT01Z84cjRgxQpUqVSrQm99ad1Bn07IlSTdV8tZjneubUTqAayBIwin5+vpq2rRp\n2rhxo7788ku1bdtWAQEB6tSpk+bNm6ekpCQZhtXsMgEAcBl+fn6aMWOGNmzYoMWLF6tdu3YKCQlR\n27Zt9eGHH+r06dMyDEOSdCgxVR/9Fmfb9vkejeXj6W5S5QCuhiAJp3XixAlNmDBB27ZtU2xsrDp1\n6qSsrCwtWrRIPj4+smakXnPbzMxMPfPMM6pevbp8fHzUvn17bdy4sQyrBwDA+Zw4cUKTJk3Sjh07\ntGfPHnXs2FGGYeiDDz6Qu7u7Lly4IMMwNGXlXuVaL4XKtmHB6tWsOr0ZcDAESTil9PR0denSRfv2\n7dPHH3+szz//XGFhYVq9erVycnJ08eJF5V5Mvub2Dz/8sN59911NmTJFK1euVPXq1XXnnXdqx44d\nZfhZAADgPK7Vm1etWqXc3FxlZGTo1KlTWrc3URsPJEmS3CzSpN5NZLFY6M2Ag/EwuwCgNLz77rs6\ncuSI9u/fr/DwcElSs2bNFBERoccee0yLFy+WkXHxqtvu3LlTn332mT744AMNGTJEktS5c2dFRkZq\n4sSJWrFiRZl9HgAAOIvr9ebhw4dr8eLFOn3mrF7adOVayQFtayvypkr0ZsABcUQSTmnFihVq166d\nrVFJUlhYmDp27Khdu3YpMTFR3nWaXnPbChUq6P7777fNeXh4qH///vr+++/zPQsLAAAUzfV68+7d\nu5WUlKTYvGo6djZdklTR20Nj72ho25beDDgWgiScUkxMjKKiogrMR0ZGKjY2ttBtw8LC5Oub/xbj\nkZGRys7O1qFDh+xaKwAArqCw3nz6QqbeXn+lx465vYGC/Txt29KbAcdiuXx3rKLo3r27cebMmRK/\naFJSkkJDQ0u8HxSNK369t23bpqpVq6pGjRr55k+ePKmEhAS1bt3a9oBjSWpao5Lt7wcOHJDValWj\nRo3ybXvhwgUdPHhQDRs2lL+/f4HXTEpK0uX/H1lZWWrRooU9PyVchyu+x81kz6/31q1bvzcMo7td\nduai6M3lkyt+vQvrzVXqNVZKeo4kycvDTRFVA2T57zr05vLHFd/jZjKjNxcrSEoq1srXEh0drS1b\ntthjVygCV/x6e3p6asyYMZoxY0a++fHjx2vGjBnKzc1V3XHf2ubjZvS0/f2OO+7QhQsXtGnTpnzb\nrl27Vrfffrs2btyoTp06Xff1/fz8lJaWZofPBEXhiu9xM9n5620pfBUUgt5cDrni1/t6vfmNhV+p\nysDXbHOfPnKzOoaH2Mb05vLHFd/jZjKjN3NqK5xSUFCQzp07V2A+OTlZQUFBN7ytJAUHB9unSAAA\nXMi1+uvZ5GRVvvMx2/jOyKr5QuT1tqU3A+YhSMIpRUZGKiYmpsB8bGysmjRpUui2R48eVXp6eoFt\nPT09890kAAAAFM21evPmJIvcQ+tJkjw93DS+Z8E+TW8GHI8pQXLo0KFmvKzLcsWvd58+fbRp0yYd\nOXLENhcXF6dff/1Vffr0ue62vXv3Vk5Ojr744gvbXG5urhYvXqw77rhDXl5ehb5+SEhIoevAflzx\nPW4mvt7OiX/XsuWKX++r9eaYA4eVWLWdbTy0Uz3VCvYtsC29ufxxxfe4mcz4eptyjSRQ2tLS0tS8\neXP5+Pho6tSpslgsmjBhglJTU7Vr1y75+/vnu0byId+tmjhxom18+Xbir732msLCwjR37lytXLlS\nv/32m1q1alXo63NdAFBkXCNZcvRmlAtX681Pf/KrsuvdIkmqVtFb65/urKQ/T6p+/fqaOHEivRkw\nB9dIwnX5+flp/fr1atCggQYNGqQHHnhAYWFhWr9+/VXv6ma1WvONP/zwQw0ZMkTjx49Xz549deLE\nCa1evbpIjQoAABT0/3vzoBFPKyeso2358z0aydfTQ4ZhKC8vj94MODiOSMJlXeuurfbAbz2BIuOI\nZMnRm1EuDflws37cnyRJiq4TpC+Gt5fFUnrfEujNQJE5/hHJAwcO6Mknn1SzZs3k7++v6tWrq0+f\nPtq5c6eZZTm1N954Q71791b16tVlsVg0efJks0tyGidOnFC/fv1UqVIlbd++XXfffbeOHz9udllO\nKT4+XiNHjlT79u3l6+sri8WiuLg4s8tyWkuXLtU999yjOnXqyMfHRw0bNtTzzz+v1NRUs0tDKaA3\nlz1X7M3r9522hUiLRZrcJ7JUQiS9uezQm8uWI/RmU4PkmjVr9OOPP+rBBx/UN998ozlz5igpKUnt\n2rXT1q1bzSzNab377rtKTEzUP/7xD7NLcSrp6enq0qWL9u3bp48++khhYWE6ePCgbrvtNp5ZVQoO\nHTqkJUuWKCgoqNDnhqHkZs6cKXd3d7388stavXq1HnvsMc2dO1e33357gVPPUP7Rm8ueq/Xm7Fyr\nXlq51zbu36aWompUsvvr0JvLFr25bDlEbzYMozgfdpWUlGRYrdZ8cykpKUZgYKAxaNAge78cDMPI\ny8szDMMwcnJyDEnGpEmTzC3IRHWeW2n7KKlZs2YZbm5uxsGDBw3DMIzWrVsbR44cMdzd3Y3XX3+9\nxPtHfpffx4ZhGO+++64hyTh69Kh5BTm5xMTEAnMfffSRIclYt25dSXdf3D7EB73Z6bhab5634ZCt\n/0ZNWm2cSc0sldehN5ctenPZcoTebOoRyZCQkAKnMVSqVEkNGjTQyZMnTarKubm5cX+l0rBixQq1\na9cu33OswsLC1LFjRy1fvtzEypwT7+OyFRoaWmCuTZs2ksT3aidEby57rvQ9LfFCpt5ad9A2Ht2t\ngSr7F/7ojhtBby5brvQ+dgSO0Jsd7l88OTlZe/bsUePGjc0uBSiymJgYRUVFFZiPjIxUbGysCRUB\npeunn36SJL5Xuwh6M+xlxup9SsvOkySFV/HX/7SvU2qvRW+Gqynr3uxwQXLkyJEyDEOjR482uxSg\nyJKTkxUUFFRgPjg4WOfOnTOhIqD0nDx5UhMnTlS3bt0UHR1tdjkoA/Rm2MPWY+f01bYrR0om945U\nBffS+1GU3gxXYkZvtuv/3rVr18pisRT6ceutt151++nTp+uzzz7T22+/ne80BFxdSb/eAFBcFy9e\nVN++feXh4aEPP/zQ7HJQBPTmskVvvjqr1dDkFTG2cffIavpbRIiJFQHOw6ze7GHPnXXo0EF79+4t\ndD1fX98Cc/PmzdMLL7ygqVOn6qGHHrJnWU6rJF9v2FdQUNBVf7t5rd+GAuVRRkaGevfurSNHjuin\nn35SzZo1zS4JRUBvLlv05qv7YusJ7T55XpLk5eGmF3uW/ql39Ga4AjN7s12DpK+vrxo1alTs7RYt\nWqQRI0Zo7NixevHFF+1ZklO70a837C8yMlIxMTEF5mNjY9WkSRMTKgLsKycnR/369dOWLVv0ww8/\nqGnTpmaXhCKiN5ctenNB5zNy9Orq/bbxsM71VSu49IM0vRnOzuzebPo1ksuWLdOQIUP0yCOPaObM\nmWaXA9yQPn36aNOmTTpy5IhtLi4uTr/++qv69OljYmVAyVmtVj3wwANav369vv76a7Vr187sklDK\n6M2wp1lrD+hsWrYkqUagjx7rXL9MXpfeDGfmCL3Zrkcki2vjxo0aMGCAmjdvrsGDB2vTpk22ZV5e\nXmrZsqWJ1TmnLVu2KC4uzvag0tjYWC1dulSS1KNHD5c71cZeHn30Ub399tvq27evpk6dqpSUFPXt\n21e1atXSsGHDzC7PKV1+315+QPqqVasUGhqq0NBQde7c2czSnM7jjz+uL774Qi+++KL8/Pzyfa+u\nWbMmp7g6GXpz2XPm3rw/IVUf/37MNn6hR2P5eLqXyWvTm8sevbnsOERvLuoDJ41SeOjxpEmTDElX\n/ahTp469Xw6GYTz44IPX/Jq72kNjLz8Muc5zK+2yv2PHjhl33323ERAQYLi5uRl9+/Z1ua9pWbrW\n+7hz585ml+Z06tSpc82vtx0enF7cPsQHvdnpOGtvtlqtxv3zf7P12gELfjesVmuZ1kBvLlv05rLj\nCL3ZYhhGsXJncVYGHFndcd/a/h43o6dd9x0dHa0tW7bYdZ+Ak7KYXYAToDfDIX2z85RG/me7JMnD\nzaJVT3ZSRNUA0+qhNwNFVqTebPo1kgAAAHAuaVm5mvbtlbvXPtihrqkhEoD9ESQBAABgV+/8eEgJ\nFzIlSSH+XnqyW4TJFQGwN4IkAAAA7ObomTS99/NR2/j5vzdSRe8KJlYEoDQQJAEAAGAXhmFo0ooY\nZeddugNtq9qBuqtlDZOrAlAaCJIAAACwi+9jTmvjgSRJkptFmtI3Sm5u3FMLcEYESQAAAJRYRnae\nXloZaxs/cHMdRdWoZGJFAEoTQRIAAAAl9s6Ph3QyJUOSFOznqafvaGhyRQBKE0ESAAAAJXL0TJoW\nbDxiG4/r3kiVfLnBDuDMCJIAAAC4YYZhaPJfbrDTsnag+rWuaXJVAEobQRIAAAA37PuYBP303xvs\nWCzSS9xgB3AJBEkAAADckLSsXP3rmys32BnIDXYAl0GQBAAAwA15a91B/Xk+U5IU4s8NdgBXQpAE\nAABAse1PSNX7vxy1jV/o0Zgb7AAuhCAJAACAYjEMQxO+3qNcqyFJahsWrLta1jC5KgBliSAJAACA\nYvly20ltjkuWJHm4WTT1H1GyWLjBDuBKCJIAAAAosnNp2Xr5u7228cOdwtSgaoCJFQEwA0ESAAAA\nRTZj1T4lp2VLkm6q5K1RXSJMrgiAGQiSAAAAKJI/4pK1eMsJ23hyn0j5eXmYWBEAsxAkAQAAUKjs\nXKteXLbbNr69SVXdEVnNxIoAmIkgCQAAgEK998sRHTh9UZLk6+muf/WJNLkiAGYiSAIAAOC6TiSn\n6611B23jMbc30E2BPiZWBMBsBEkAAABck2EYevHrPcrMsUqSGlevqMEd6ppbFADTESQBAABwTSt2\nntLGA0mSJItFevmuKHm48yMk4Or4LgAAAICrOpeWrSnfxNrGD7avq5a1g0ysCICjIEgCAADgqqZ9\nt1dn//vMyOqVvPX0nQ1NrgiAoyBIAgAAoIDfDp3R0q3xtvFLfaPkzzMjAfwXQRIAAAD5ZObk6YW/\nPDOyR9Nq6takqokVAXA0BEkAAADk8+baA4o7my5JCvD20OTePDMSQH4ESQAAANjsik/RuxuP2MbP\n/72xqlT0NrEiAI6IIAkAAABJUk6eVc8u3SWrcWncvl5lDWhby9yiADgkgiQAAAAkSfM2HNa+hFRJ\nkncFN824p6ksFovJVQFwRARJAAAA6ODpVP17/SHb+Ok7GqpOZT8TKwLgyAiSAAAALi7PaujZL3cp\nO88qSWpeK1BDOoaZXBUAR0aQBAAAcHHv/3JE24+nSJIquFv06j3N5O7GKa0Aro0gCQAA4MIOJV7U\nzDUHbOORXSLUsFqAiRUBKA8IkgAAAC4qz2romaU7lZ176ZTWqBoV9dit9U2uCkB5QJAEAABwUe/9\nnP+U1pn3NlcFd348BFA4vlMAAAC4oEOJqXr9hyuntI7qEqFG1SqaWBGA8oQgCQAA4GJy8qwas+TK\nKa1Na1TScE5pBVAMBEkAAAAXM+fHw9oVf16S5OnuximtAIqN7xgAAAAuZFd8it5af9A2HntHA+7S\nCqDYCJIAAAAuIjMnT08t3qE8qyFJalM3SI90qmdyVQDKI4IkAACAi3h19X4dTkqTJPl6uuv1e1vI\n3c1iclUAyiOCJAAAgAv49dAZffDrUdt4Qq8mql3Z18SKAJRnBEkAAAAndy4tW2OW7LCNb2sYqv5t\naplYEYDyjiAJAADgxAzD0Livdun0hSxJUrCfp17p10wWC6e0ArhxBEkAAAAntmTLCX0fc9o2fq1f\nM1UJ8DaxIgDOgCAJAADgpI4kXdTkFbG28aB2ddS1cVUTKwLgLAiSAAAATig716rRi3coIydPkhRe\nxV8v9GhsclUAnAVBEgAAwAm9unqfdsWflyR5urtpdv8W8vF0N7kqAM6CIAkAAOBk1u87rfd+ufKo\nj2e7N1TkTZVMrAiAsyFIAgAAOJE/z2do7JKdtnHXRlX08N/CTKwIgDMiSAIAADiJPKuhJz/foXPp\nOZKkahW99dq9zXnUBwC7I0jCaVmtVk2fPl1169aVt7e3mjdvri+//LJI2w4ePFgWi6XAx+jRo0u5\nagAAbtzsdQe1+WiyJMnNIs3u30LBfp4mV3UFvRlwHh5mFwCUlgkTJmjmzJmaNm2aWrdurc8//1z3\n3nuvVq5cqR49ehS6fWhoqFasWJFvrnr16qVVLgAAJfLTgST9e/1B2/jJrg10c73KJlZUEL0ZcB4E\nSTilxMREzZw5U+PGjdPTTz8tSbrtttt06NAhjRs3rkjNytPTU+3atSvtUgEAKLFTKRka/fl2Gcal\ncYf6lfVEl3Bzi/p/6M2Ac+HUVjil77//XtnZ2Ro4cGC++YEDB2r37t06evToNbYEAKB8yc616vHP\nttmui6xa0UtvDWgpdzfHui6S3gw4F4IknFJMTIy8vLwUHp7/t7GRkZGSpNjY2EL3kZiYqJCQEHl4\neKhBgwZ65ZVXlJeXVyr1AgBwo17+bq+2H0+RJLm7WfT2P1spxN/L5KoKojcDzoVTW+GUkpOTFRgY\nWOAudcHBwbblUvA1t2/RooVat26tyMhIZWZmatmyZXr++ed18OBBvffee1fdZsGCBVqwYIEkKSkp\nyT6fCAAA1/HNzlNa+FucbTyueyO1qXvt/mamovXma6M3A46FIIlyYe3atbr99tsLXa9z587asGFD\niV/v/98BrkePHvL399esWbP03HPPKSIiosA2Q4cO1dChQyVJ0dHRJa4BAIDr2fvnBT27dJdtfGdk\nVT3SqeyeF0lvBlwbQRLlQocOHbR3795C1/P19ZUkBQUFKSUlRYZh5PvN5+Xfdl7+7WdxDBgwQLNm\nzdKWLVuu2qwAACgrKenZGrpoizJyLp3WGRbiV+bPi6Q3A66NIIlywdfXV40aNSry+pGRkcrKytLh\nw4fzXYtx+fqLJk2aSD8Xfi3G1fBQZwCAmfKshkb+Z7tOJGdIkvw83bVgUGtV9K5QpnWUSm++QfRm\noOxxsx04pe7du6tChQr69NNP881/8sknioqKUlhY8U/9+fTTT2WxWNSmTRt7lQkAQLG99v1+/Xzw\njG38+n0tFFE1wMSKiobeDDgXjkjCKVWpUkVjxozR9OnTFRAQoFatWmnx4sVav359gQcZS1J4eLgO\nHTokSTp27JgGDRqk/v37Kzw8XFlZWVq2bJkWLlyoYcOGqX79+mX96QAAIElavuOk5v102DYe2SVc\n3aOqmVhR0RWnN3ft2lXHjh2jNwMOjCAJpzVt2jT5+/tr9uzZSkhIUMOGDbVkyRL16tWrwLq5ubm2\nvwcEBCg4OFivvPKKTp8+LTc3NzVq1EhvvfWWRowYUZafAgAANtuPn9Mzf7m5zm0NQzW6WwMTKyq+\novbmvLw8ejPg4CyGYRRn/WKtDDiyuuO+tf09bkZPu+47OjpaW7Zsses+ASfFhU0lR292AadSMtTn\n7V915mKWJCm8ir++GtGhzK+LLM/ozUCRFak3c40kAACAA0vPztUjH22xhchA3wp6/8FoQiQAUxEk\nAQAAHFSe1dDoz3co9s8LkiQPN4vmPtBadSr7mVwZAFdHkAQAAHBQ077dqzWxp23jl/4Rpfb1K5tY\nEQBcQpAEAABwQB/8clQf/HrUNn74b2Ea0La2iRUBwBUESQAAAAezek+CXvo21jbuHllNL/RobGJF\nAJAfQRIAAMCBbDt+Tk9+vl2Xb6zfsnagZvVvIXc3bnIMwHEQJAEAABzEocRUPbTwD2XlWiVJdSr7\n6r3/iZZ3BXeTKwOA/AiSAAAADuBUSoYGvb9ZKek5kqQg3wpaOKStKvt7mVwZABREkAQAADDZubRs\nDXr/f/Xn+UxJkq+nuxYOaauwEB7zAcAxESQBAABMlJaVqyEL/9DhpDRJUgV3i+YPaq3mtQJNrgwA\nro0gCQAAYJLMnDw98tEW7TiRIkmyWKQ372+hThGhJlcGANdHkAQAADBBVm6ehi3aqt+PnLXNTekT\nqV7NbjKxKgAoGoIkAABAGcvJs2rkZ9v104Ek29xz3RtpUPu65hUFAMVAkAQAAChDuXlWjVmyU2ti\nT9vmRnWN0GO31jexKgAoHg+zCwAAAHAVuXlWjV68Qyt3/WmbG3ZLPT3VLcLEqgCg+AiSAAAAZSAn\nz6onP9+u73Yn2OYebF9H4/7eSBaLxcTKAKD4CJIAAAClLDvXqpH/2abvY66czjq4Q11N6t2EEAmg\nXCJIAgAAlKLMnDw98dk2rd2baJt7qGOYJvRqTIgEUG4RJAEAAErJxaxcPfLRH9p0JNk292inML3Q\ngxAJoHwjSAIAAJSC5LRsDf5ws3bFn7fNPXZrfT17Z0NCJIByjyAJAABgZ3+ez9D/vL9ZBxMv2uae\n696IR3wAcBoESQAAADvan5CqwR9u1p/nMyVJFos09R9ReuDmOiZXBgD2Q5AEAACwk98Pn9XQRVuU\nmpkrSfJws+iN+1uoT/ObTK4MAOyLIAkAAGAHK3ae0tNLdio7zypJ8vN019yBrXVLg1CTKwMA+yNI\nAgAAlIBhGHp7/SG9/sMB21xogJcWDmmjyJsqmVgZAJQegiQAAMANyszJ03Nf7tLyHadsc/VD/fTR\nQ21VM8jXxMoAoHQRJAEAAG5AYmqmhn68VTtOpNjm2tUL1ryBrRXo62liZQBQ+giSAAAAxbT9+DmN\n+HSb7c6skjSgbW1N6RupCu5uJlYGAGWDIAkAAFAM/9l8XJOWx9huquNmkSb0aqLBHerKYrGYXB0A\nlA2CJAAAQBFk5uRp0vIYLd5ywjZX0dtDbw1oqVsbVjGxMgAoewRJAACAQhw9k6YnPtummFMXbHON\nq1fUvIGtVKeyn4mVAYA5CJIAAADXsXzHSb3w1W6lZf9fe/caFNV5x3H8x20XVpGggBdAIRq8EUOR\nSQzQYBNJDYq5IXaaaEic5vqGaqxNRkcaHdLpxMRkrNEkM8Ea21HITIxGHYutsQaTCBNiBOsNEbwE\n5RJFUYTd7QsrdV1SPQHZZff7eSOe/1n2P+NxH348z3mOtePYoz+LVP6jdyrI5OfCzgDAdQiSAAAA\nnWi53K4/fFrpsJTV5J/Phy0AAAr0SURBVOerBVNHa+aEYdwPCcCrESQBAACuU177g367rlxH6y90\nHIsZYNHyXycqPjLEhZ0BgHsgSAIAAPxXu9WmP//ziN75xyFZbfaO45l3DVH+o/EKDgxwYXcA4D4I\nkgAAAJIO1TVrXtFeldf+0HGsj8lPi6aN1fTxUSxlBYBrECQBAIBXa7PatOrzI3pn++GOZ0NKUtKw\nUL01I0HR/S0u7A4A3BNBEgAAeK19J87qd0V7VXnqf4/1CPDzUe6kOD2fNlx+vsxCAkBnCJIAAMDr\nNF9q09JtB/WX3dW65lZI3RUVoj9l3aWRg4Jd1hsA9AYESQAA4DXsdrs+++6UXttYqdPNrR3Hzf6+\nmvtgnJ5JiZW/n68LOwSA3oEgCQAAvML+U+f02sZK7a5qcDieMmKAFj8cr9vD+7qoMwDofQiSAADA\nozVeuKyl2w7ob1/XOCxjDQ82a+HUMcocN5gdWQHAIIIkAADwSBcvW1VQUq0VOw6r+VJ7x3E/Xx/N\nnDBMcx6MUz+eCwkAPwlBEgAAeJR2q01FZcf1VvFB1Z1rdaj9/I4wLZw6RnED2UwHALqCIAkAADyC\n1WbXpr0n9fb2Q6o6c8GhFhvWRwumjNb9oyJYxgoA3YAgCQAAejWr7cpOrG8XH9SR6wJkeLBZuZPu\nUHZStALYjRUAug1BEgAA9EqX22365JsTWrnziNMMZHCgv55PG66nU2JkMfHjDgB0Nz5ZAQBAr9J8\nqU3r9tTqg38d1ffnLjnUgs3+ejo1VrNTYxUSxEY6AHCrECQBAECvUNPQooKSaq0vrdX51naHWnCg\nv3KSYzQ7NVa3WUwu6hAAvAdBEgAAuC2bza5dh+u15stjKt5fJ7vdsR4ebNbs1Fg9cc9QBfMoDwDo\nMQRJAADgdhrOt6qo7Lj++nWNjjW0ONVHRPTVMymxeiwxUoEBfi7oEAC8G0ESAAC4hXarTZ8fPKPC\n0uPa/u86tVntTuekxYXrmdRY3XdHGI/xAAAXIkgCAACXsdvtqjh5Tp98c0KffntSp5tbnc7pF+iv\nrPHRemLCUA0P7+uCLgEA1yNIAgCAHnf49Hl9tveUNnx7wunRHVclDr1Nv7p7qDLHDVGQieWrAOBO\nCJIAAOCWs9vtOlDXrK37vtfm707pYN35Ts8LDzbrscRITR8frRERzD4CgLsiSAIAgFuizWrTnupG\n/b2yTsX761TbeLHT8ywmP/1y7CA9nDBEqSPC5O/n28OdAgCMIkgCAIBuc+rsRX1+4Ix2HDijLw7X\nq/m65z1eZfb31S9GRihj3GCljx7I0lUA6GUIkgAA4Cc7e7FNX1U16IvD9friSIMOn+58yaok9TX7\nK21kuDLiB2viyHD1MfNjCAD0VnyCAwCAm9Z44bL2VDfqq6pGfXW0QZWnzsnu/JSODpG3Ben+URFK\nHzNQ99zeX2Z/Zh4BwBMQJAEAQKesNrsOnW5Wec0PKjvWpLJjTaqq73yH1atM/r66O6a/Jo4MV1pc\nuEZE9OV5jwDggQiSAABANptdRxsuaN+Js9p34qy+PX7lz5bL1v/7Ol8faeyQEKWMCFPqiDAlxYQq\nMIBZRwDwdARJAAC82IbyE1r7ZY0qTp7VhRuERkkK8PNRfGSI7o7trwmxAzQ+JlT9AgN6oFMAgDsh\nSAIA4MUazl/W19WNP1of1C9Q46JClDgsVEnDQhUfGcKMIwCAIAkAgDe7Myqk4+uwvibFR4YofkiI\n4iP7KSE6VINCAl3YHQDAXREkAQDwYvFDQvTBrCTFR4ZoYD8zG+MAAG4KQRIAAC8WZPLTpDEDXd0G\nAKCX8XV1AwAAAACA3oUgCY/05ptvKjMzU4MHD5aPj4/y8vIMvX7Xrl1KTk5WUFCQBg0apDlz5uji\nxYu3plkAALwAYzPgWQiS8Ejvv/++Tp8+rUceecTwa/fu3av09HRFRERo06ZNWrJkiT788EPl5OR0\nf6MAAHgJxmbAs3CPJDxSRUWFfH191d7erpUrVxp67aJFixQVFaXCwkIFBFx5NprJZNJTTz2l+fPn\nKzEx8Va0DACAR2NsBjwLM5LwSL6+P+3Sbmtr09atW5Wdnd0xUElSdna2TCaTNmzY0F0tAgDgVRib\nAc/iY7fbjZxv6OTOxPz+s65+C6DbVf9xilpaWjRx4kTt2bNHo0ePVkREhCQpJydHOTk5iouLU1NT\nk8aOHevw2hdeeEEzZsxQbW2tZs6cKUkqKyvT+PHjJUlz585VZmamDhw4oOeee87pvRcsWKBJkyap\nvLxcubm5TvX8/HwlJyerpKREr776qlN92bJlSkhIUHFxsZYsWeJUX7VqlUaOHKmNGzdq6dKlTvU1\na9YoOjpa69at07vvvutULyoqUlhYmAoKClRQUOBU37x5sywWi1asWKH169c71Xfs2CFJeuONN7Rp\n0yaHWlBQkLZs2SJJWrx4sbZv3+5QHzBggD7++GNJ0iuvvKLdu3c71KOiovTRRx9JknJzc1VeXu5Q\nj4uL03vvvSdJevbZZ3Xw4EGHekJCgpYtWyZJevLJJ3X8+HGH+r333qvXX39dkvT444+roaHBof7A\nAw9o4cKFkqSHHnrI6V6dqVOn6uWXX5YkTZw4UdfLzs7Wiy++qJaWFmVkZDjVr1579fX1ysrKcqp3\ndu1d61Zde1f/TbsJz5roui6PzZJnXaNX8fnonp+PdrtdO3fu1KJFi5SXl3fDa4+xmWvvWozN7jM2\ns7QVXi/Ax9bxdXt7uyTJ39/5v0ZoaKjOnDnzo99n7dq1Kisrk3Tlt6cAAKDrGJsB98SMJLxaH5Of\ncifF6Tf33S5JKikpUUpKirZs2aLJkyc7nJuamiqz2ez0m7nOJCUlqbS09Jb0DHgYZiS7rltmJAEj\niouLlZ6efsPz0tLSnGZK2tvbFRAQ0DEjeSOMzUCPc88Zyeo/Tunpt0Qv19LSopqamhueZ7FYNHTo\nUIdjRger0NBQSVJTU5NTrbGx0WnpDAAA3ig5OVn79++/4XkWi6XL78XYDLgnlrbC7VksFo0aNapH\n3mv48OEym82qqKhwOH7p0iVVVVVp+vTpPdIHAADujLEZALu2AtcwmUyaPHmy1q9f33FPhnTlpvbW\n1lZNmzbNhd0BAOB9GJsB98SMJDxSaWmpqqurZbNd2UinsrJSRUVFkqSMjIyOpTazZ8/W6tWrHQam\nvLw8TZgwQdnZ2XrppZdUXV2tefPmKSsrq2O3NwAAYAxjM+BZCJLwSMuXL9fq1as7/l5YWKjCwkJJ\n0tGjRxUTEyNJslqtslqtDq9NSEjQtm3bNH/+fE2ZMkUhISGaNWuW8vPze6x/AAA8DWMz4Fl6fNdW\nwBuwMxxw09i1tesYm4GbwNgM3LSbGpu5RxIAAAAAYAhBEgAAAABgCEESAAAAAGAIQRIAAAAAYAhB\nEgAAAABgCEESAAAAAGAIQRIAAAAAYAhBEgAAAABgCEESAAAAAGAIQRIAAAAAYAhBEgAAAABgCEES\nAAAAAGAIQRIAAAAAYAhBEgAAAABgCEESAAAAAGAIQRIAAAAAYAhBEgAAAABgCEESAAAAAGAIQRIA\nAAAAYAhBEgAAAABgCEESAAAAAGAIQRIAAAAAYAhBEgAAAABgiI/dbnd1D4DH8fHx2Wq32ye7ug8A\nAHAFYzPQvQiSAAAAAABDWNoKAAAAADCEIAkAAAAAMIQgCQAAAAAwhCAJAAAAADCEIAkAAAAAMIQg\nCQAAAAAwhCAJAAAAADCEIAkAAAAAMIQgCQAAAAAw5D9uqH5cCoY+SQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f88064ca690>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_activation_functions()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Perceptron a regresja liniowa"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "<img src=\"reglin.png\" />"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Uczenie regresji liniowej:\n",
    "* Model: $$h_{\\theta}(x) = \\sum_{i=0}^n \\theta_ix_i$$\n",
    "* Funkcja kosztu (błąd średniokwadratowy): $$J(\\theta) = \\frac{1}{m} \\sum_{i=1}^{m} (h_{\\theta}(x^{(i)}) - y^{(i)})^2$$\n",
    "\n",
    "* Po obliczeniu $\\nabla J(\\theta)$, zwykły SGD."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Perceptron a dwuklasowa regresja logistyczna"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"reglog.png\" />"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Uczenie dwuklasowej regresji logistycznej:\n",
    "* Model: $$h_{\\theta}(x) = \\sigma(\\sum_{i=0}^n \\theta_ix_i) = P(1|x,\\theta)$$\n",
    "* Funkcja kosztu (entropia krzyżowa): $$\\begin{eqnarray} J(\\theta) &=& -\\frac{1}{m} \\sum_{i=1}^{m} [y^{(i)}\\log P(1|x^{(i)},\\theta) \\\\ && + (1-y^{(i)})\\log(1-P(1|x^{(i)},\\theta))]\\end{eqnarray}$$\n",
    "\n",
    "* Po obliczeniu $\\nabla J(\\theta)$, zwykły SGD."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Perceptron a wieloklasowa regresja logistyczna"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "<img src=\"multireglog.png\" />"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Wieloklasowa regresji logistyczna\n",
    "* Model (dla $c$ klasyfikatorów binarnych): \n",
    "$$\\begin{eqnarray}\n",
    "h_{(\\theta^{(1)},\\dots,\\theta^{(c)})}(x) &=& \\mathrm{softmax}(\\sum_{i=0}^n \\theta_{i}^{(1)}x_i, \\ldots, \\sum_{i=0}^n \\theta_i^{(c)}x_i) \\\\ \n",
    "&=& \\left[ P(k|x,\\theta^{(1)},\\dots,\\theta^{(c)}) \\right]_{k=1,\\dots,c} \n",
    "\\end{eqnarray}$$\n",
    "* Funkcja kosztu (**przymując model regresji binarnej**): $$\\begin{eqnarray} J(\\theta^{(k)}) &=& -\\frac{1}{m} \\sum_{i=1}^{m} [y^{(i)}\\log P(k|x^{(i)},\\theta^{(k)}) \\\\ && + (1-y^{(i)})\\log P(\\neg k|x^{(i)},\\theta^{(k)})]\\end{eqnarray}$$\n",
    "\n",
    "* Po obliczeniu $\\nabla J(\\theta)$, **c-krotne** uruchomienie SGD, zastosowanie $\\mathrm{softmax}(X)$ do niezależnie uzyskanych klasyfikatorów binarnych."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "* Przyjmijmy: \n",
    "$$ \\Theta = (\\theta^{(1)},\\dots,\\theta^{(c)}) $$\n",
    "\n",
    "$$h_{\\Theta}(x) = \\left[ P(k|x,\\Theta) \\right]_{k=1,\\dots,c}$$\n",
    "\n",
    "$$\\delta(x,y) = \\left\\{\\begin{array}{cl} 1 & \\textrm{gdy } x=y \\\\ 0 & \\textrm{wpp.}\\end{array}\\right.$$\n",
    "\n",
    "* Wieloklasowa funkcja kosztu $J(\\Theta)$ (kategorialna entropia krzyżowa):\n",
    "$$ J(\\Theta) = -\\frac{1}{m}\\sum_{i=1}^{m}\\sum_{k=1}^{c} \\delta({y^{(i)},k}) \\log P(k|x^{(i)},\\Theta) $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "* Gradient $\\nabla J(\\Theta)$:\n",
    "$$ \\dfrac{\\partial J(\\Theta)}{\\partial \\Theta_{j,k}} = -\\frac{1}{m}\\sum_{i = 1}^{m} (\\delta({y^{(i)},k}) - P(k|x^{(i)}, \\Theta)) x^{(i)}_j \n",
    "$$\n",
    "\n",
    "* Liczymy wszystkie wagi jednym uruchomieniem SGD"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "## Podsumowanie\n",
    "\n",
    "* W przypadku jednowarstowej sieci neuronowej wystarczy znać gradient funkcji kosztu.\n",
    "* Wtedy liczymy tak samo jak w przypadku regresji liniowej, logistycznej, wieloklasowej logistycznej itp.\n",
    "  * Wymienione modele to szczególne przypadki jednowarstwowych sieci neuronowych.\n",
    "* Regresja liniowa i binarna regresja logistyczna to jeden neuron.\n",
    "* Wieloklasowa regresja logistyczna to tyle neuronów ile klas."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Funkcja aktywacji i funkcja kosztu są **dobierane do problemu**."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## 5.2. Wielowarstwowe sieci neuronowe\n",
    "\n",
    "czyli _Artificial Neural Networks_ (ANN) lub _Multi-Layer Perceptrons_ (MLP)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "<img src=\"nn1.png\" />"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Architektura sieci\n",
    "\n",
    "* Sieć neuronowa jako graf neuronów. \n",
    "* Organizacja sieci przez warstwy.\n",
    "* Najczęściej stosowane są sieci jednokierunkowe i gęste."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "* $n$-warstwowa sieć neuronowa ma $n+1$ warstw (nie liczymy wejścia).\n",
    "* Rozmiary sieci określane poprzez liczbę neuronów lub parametrów."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Sieć neuronowa jednokierunkowa (_feedforward_)\n",
    "\n",
    "* Mając daną $n$-warstwową sieć neuronową oraz jej parametry $\\Theta^{(1)}, \\ldots, \\Theta^{(L)} $ oraz $\\beta^{(1)}, \\ldots, \\beta^{(L)} $ liczymy:<br/><br/> \n",
    "$$a^{(l)} = g^{(l)}\\left( a^{(l-1)} \\Theta^{(l)} + \\beta^{(l)} \\right). $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "<img src=\"nn2.png\" />"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "* Funkcje $g^{(l)}$ to tzw. **funkcje aktywacji**.<br/>\n",
    "Dla $i = 0$ przyjmujemy $a^{(0)} = \\mathrm{x}$ (wektor wierszowy cech) oraz $g^{(0)}(x) = x$ (identyczność)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "* Parametry $\\Theta$ to wagi na połączeniach miedzy neuronami dwóch warstw.<br/>\n",
    "Rozmiar macierzy $\\Theta^{(l)}$, czyli macierzy wag na połączeniach warstw $a^{(l-1)}$ i $a^{(l)}$, to $\\dim(a^{(l-1)}) \\times \\dim(a^{(l)})$."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "* Parametry $\\beta$ zastępują tutaj dodawanie kolumny z jedynkami do macierzy cech.<br/>Macierz $\\beta^{(l)}$ ma rozmiar równy liczbie neuronów w odpowiedniej warstwie, czyli $1 \\times \\dim(a^{(l)})$."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "* **Klasyfikacja**: dla ostatniej warstwy $L$ (o rozmiarze równym liczbie klas) przyjmuje się $g^{(L)}(x) = \\mathop{\\mathrm{softmax}}(x)$.\n",
    "* **Regresja**: pojedynczy neuron wyjściowy jak na obrazku. Funkcją aktywacji może wtedy być np. funkcja identycznościowa."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "* Pozostałe funkcje aktywacji najcześciej mają postać sigmoidy, np. sigmoidalna, tangens hiperboliczny.\n",
    "* Mogą mieć też inny kształt, np. ReLU, leaky ReLU, maxout."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## 5.3. Metoda propagacji wstecznej – wprowadzenie"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Jak uczyć sievi neuronowe?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "* W poznanych do tej pory algorytmach (regresja liniowa, regresja logistyczna) do uczenia używaliśmy funkcji kosztu, jej gradientu oraz algorytmu gradientu prostego (GD/SGD)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "* Dla sieci neuronowych potrzebowalibyśmy również znaleźć gradnient funkcji kosztu."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "* Co sprowadza się do bardziej ogólnego problemu:<br/>jak obliczyć gradient $\\nabla f(x)$ dla danej funkcji $f$ i wektora wejściowego $x$?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Pochodna funkcji\n",
    "\n",
    "* **Pochodna** mierzy, jak szybko zmienia się wartość funkcji względem zmiany jej argumentów:\n",
    "\n",
    "$$ \\frac{d f(x)}{d x} = \\lim_{h \\to 0} \\frac{ f(x + h) - f(x) }{ h } $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Pochodna cząstkowa i gradient\n",
    "\n",
    "* **Pochodna cząstkowa** mierzy, jak szybko zmienia się wartość funkcji względem zmiany jej *pojedynczego argumentu*."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "* **Gradient** to wektor pochodnych cząstkowych:\n",
    "\n",
    "$$ \\nabla f = \\left( \\frac{\\partial f}{\\partial x_1}, \\ldots, \\frac{\\partial f}{\\partial x_n} \\right) $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Gradient – przykłady\n",
    "\n",
    "$$ f(x_1, x_2) = x_1 + x_2 \\qquad \\to \\qquad \\frac{\\partial f}{\\partial x_1} = 1, \\quad \\frac{\\partial f}{\\partial x_2} = 1, \\quad \\nabla f = (1, 1) $$ "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "$$ f(x_1, x_2) = x_1 \\cdot x_2 \\qquad \\to \\qquad \\frac{\\partial f}{\\partial x_1} = x_2, \\quad \\frac{\\partial f}{\\partial x_2} = x_1, \\quad \\nabla f = (x_2, x_1) $$ "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "$$ f(x_1, x_2) = \\max(x_1 + x_2) \\hskip{12em} \\\\\n",
    "\\to \\qquad \\frac{\\partial f}{\\partial x_1} = \\mathbb{1}_{x \\geq y}, \\quad \\frac{\\partial f}{\\partial x_2} = \\mathbb{1}_{y \\geq x}, \\quad \\nabla f = (\\mathbb{1}_{x \\geq y}, \\mathbb{1}_{y \\geq x}) $$ "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Własności pochodnych cząstkowych\n",
    "\n",
    "Jezeli $f(x, y, z) = (x + y) \\, z$ oraz $x + y = q$, to:\n",
    "$$f = q z,\n",
    "\\quad \\frac{\\partial f}{\\partial q} = z,\n",
    "\\quad \\frac{\\partial f}{\\partial z} = q,\n",
    "\\quad \\frac{\\partial q}{\\partial x} = 1,\n",
    "\\quad \\frac{\\partial q}{\\partial y} = 1 $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Reguła łańcuchowa\n",
    "\n",
    "$$ \\frac{\\partial f}{\\partial x} = \\frac{\\partial f}{\\partial q} \\, \\frac{\\partial q}{\\partial x},\n",
    "\\quad \\frac{\\partial f}{\\partial y} = \\frac{\\partial f}{\\partial q} \\, \\frac{\\partial q}{\\partial y} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Propagacja wsteczna – prosty przykład"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "# Dla ustalonego wejścia\n",
    "x = -2; y = 5; z = -4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3, -12)\n"
     ]
    }
   ],
   "source": [
    "# Krok w przód\n",
    "q = x + y\n",
    "f = q * z\n",
    "print(q, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-4, -4, 3]\n"
     ]
    }
   ],
   "source": [
    "# Propagacja wsteczna dla f = q * z\n",
    "dz = q\n",
    "dq = z\n",
    "# Propagacja wsteczna dla q = x + y\n",
    "dx = 1 * dq  # z reguły łańcuchowej\n",
    "dy = 1 * dq  # z reguły łańcuchowej\n",
    "print([dx, dy, dz])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "<img src=\"exp1.png\" />"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "* Właśnie tak wygląda obliczanie pochodnych metodą propagacji wstecznej!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "* Spróbujmy czegoś bardziej skomplikowanego:<br/>metodą propagacji wstecznej obliczmy pochodną funkcji sigmoidalnej."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Propagacja wsteczna – funkcja sigmoidalna"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Funkcja sigmoidalna:\n",
    "\n",
    "$$f(\\theta,x) = \\frac{1}{1+e^{-(\\theta_0 x_0 + \\theta_1 x_1 + \\theta_2)}}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "<img src=\"exp2.png\" />"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.3932238664829637, -0.5898357997244456]\n",
      "[-0.19661193324148185, -0.3932238664829637, 0.19661193324148185]\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "\n",
    "# Losowe wagi i dane\n",
    "w = [2,-3,-3]\n",
    "x = [-1, -2]\n",
    "\n",
    "# Krok w przód\n",
    "dot = w[0]*x[0] + w[1]*x[1] + w[2]\n",
    "f = 1.0 / (1 + math.exp(-dot))  # funkcja sigmoidalna\n",
    "\n",
    "# Krok w tył\n",
    "ddot = (1 - f) * f  # pochodna funkcji sigmoidalnej\n",
    "dx = [w[0] * ddot, w[1] * ddot]\n",
    "dw = [x[0] * ddot, x[1] * ddot, 1.0 * ddot]\n",
    "\n",
    "print(dx)\n",
    "print(dw)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Obliczanie gradientów – podsumowanie\n",
    "\n",
    "* Gradient $f$ dla $x$ mówi jak zmieni się całe wyrażenie przy zmianie wartości $x$.\n",
    "* Gradienty łączymy korzystając z **reguły łańcuchowej**.\n",
    "* W kroku wstecz gradienty informują, które części grafu powinny być zwiększone lub zmniejszone (i z jaką siłą), aby zwiększyć wartość na wyjściu.\n",
    "* W kontekście implementacji chcemy dzielić funkcję $f$ na części, dla których można łatwo obliczyć gradienty."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## 5.4. Uczenie wielowarstwowych sieci neuronowych metodą propagacji wstecznej"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "Mając algorytm SGD oraz gradienty wszystkich wag, moglibyśmy trenować każdą sieć."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "* Niech:\n",
    "$$\\Theta = (\\Theta^{(1)},\\Theta^{(2)},\\Theta^{(3)},\\beta^{(1)},\\beta^{(2)},\\beta^{(3)})$$\n",
    "\n",
    "* Funkcja sieci neuronowej z grafiki:\n",
    "\n",
    "$$\\small h_\\Theta(x) = \\tanh(\\tanh(\\tanh(x\\Theta^{(1)}+\\beta^{(1)})\\Theta^{(2)} + \\beta^{(2)})\\Theta^{(3)} + \\beta^{(3)})$$\n",
    "* Funkcja kosztu dla regresji:\n",
    "$$J(\\Theta) = \\dfrac{1}{2m} \\sum_{i=1}^{m} (h_\\Theta(x^{(i)})- y^{(i)})^2 $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "* Jak obliczymy gradienty?\n",
    "\n",
    "$$\\nabla_{\\Theta^{(l)}} J(\\Theta) = ? \\quad \\nabla_{\\beta^{(l)}} J(\\Theta) = ?$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### W kierunku propagacji wstecznej\n",
    "\n",
    "* Pewna (niewielka) zmiana wagi $\\Delta z^l_j$ dla $j$-ego neuronu w warstwie $l$ pociąga za sobą (niewielką) zmianę kosztu: \n",
    "\n",
    "$$\\frac{\\partial J(\\Theta)}{\\partial z^{l}_j}  \\Delta z^{l}_j$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "* Jeżeli $\\frac{\\partial J(\\Theta)}{\\partial z^{l}_j}$ jest duża, $\\Delta z^l_j$ ze znakiem przeciwnym zredukuje koszt.\n",
    "* Jeżeli $\\frac{\\partial J(\\Theta)}{\\partial z^l_j}$ jest bliska zeru, koszt nie będzie mocno poprawiony."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "* Definiujemy błąd $\\delta^l_j$ neuronu $j$ w warstwie $l$: \n",
    "\n",
    "$$\\delta^l_j \\equiv \\dfrac{\\partial J(\\Theta)}{\\partial z^l_j}$$ \n",
    "$$\\delta^l \\equiv \\nabla_{z^l} J(\\Theta) \\textrm{ (zapis wektorowy)} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Podstawowe równania propagacji wstecznej\n",
    "\n",
    "$$\n",
    "\\begin{array}{ccll}\n",
    "\\delta^L & = & \\nabla_{a^L}J(\\Theta) \\odot {(g^{L})}^{\\prime}(z^L) & (BP1) \\\\[2mm]\n",
    "\\delta^{l} & = & ((\\Theta^{l+1})^T \\delta^{l+1}) \\odot {{(g^{l})}^{\\prime}}(z^{l}) & (BP2)\\\\[2mm]\n",
    "\\nabla_{\\beta^l} J(\\Theta) & = & \\delta^l & (BP3)\\\\[2mm]\n",
    "\\nabla_{\\Theta^l} J(\\Theta) & = & a^{l-1} \\odot \\delta^l & (BP4)\\\\\n",
    "\\end{array}\n",
    "$$\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Algorytm propagacji wstecznej"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Dla jednego przykładu (x,y):\n",
    "\n",
    "1. **Wejście**: Ustaw aktywacje w warstwie cech $a^{(0)}=x$ \n",
    "2. **Feedforward:** dla $l=1,\\dots,L$ oblicz \n",
    "$$z^{(l)} = a^{(l-1)} \\Theta^{(l)} + \\beta^{(l)} \\textrm{ oraz } a^{(l)}=g^{(l)}(z^{(l)})$$\n",
    "3. **Błąd wyjścia $\\delta^{(L)}$:** oblicz wektor $$\\delta^{(L)}= \\nabla_{a^{(L)}}J(\\Theta) \\odot {g^{\\prime}}^{(L)}(z^{(L)})$$\n",
    "4. **Propagacja wsteczna błędu:** dla $l = L-1,L-2,\\dots,1$ oblicz $$\\delta^{(l)} = \\delta^{(l+1)}(\\Theta^{(l+1)})^T \\odot {g^{\\prime}}^{(l)}(z^{(l)})$$\n",
    "5. **Gradienty:** \n",
    "    * $\\dfrac{\\partial}{\\partial \\Theta_{ij}^{(l)}} J(\\Theta) = a_i^{(l-1)}\\delta_j^{(l)} \\textrm{ oraz } \\dfrac{\\partial}{\\partial \\beta_{j}^{(l)}} J(\\Theta) = \\delta_j^{(l)}$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "W naszym przykładzie:\n",
    "\n",
    "$$\\small J(\\Theta) = \\frac{1}{2}(a^{(L)} - y)^2 $$\n",
    "$$\\small  \\dfrac{\\partial}{\\partial a^{(L)}} J(\\Theta) = a^{(L)} - y$$\n",
    "\n",
    "$$\\small \\tanh^{\\prime}(x) = 1 - \\tanh^2(x)$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "<img src=\"nn3.png\" />"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Algorytm SGD z propagacją wsteczną\n",
    "\n",
    "Pojedyncza iteracja:\n",
    "* Dla parametrów $\\Theta = (\\Theta^{(1)},\\ldots,\\Theta^{(L)})$ utwórz pomocnicze macierze zerowe $\\Delta = (\\Delta^{(1)},\\ldots,\\Delta^{(L)})$ o takich samych wymiarach (dla uproszczenia opuszczono wagi $\\beta$).\n",
    "* Dla $m$ przykładów we wsadzie (_batch_), $i = 1,\\ldots,m$:\n",
    "    * Wykonaj algortym propagacji wstecznej dla przykładu $(x^{(i)}, y^{(i)})$ i przechowaj gradienty $\\nabla_{\\Theta}J^{(i)}(\\Theta)$ dla tego przykładu;\n",
    "    * $\\Delta := \\Delta + \\dfrac{1}{m}\\nabla_{\\Theta}J^{(i)}(\\Theta)$\n",
    "* Wykonaj aktualizację wag: $\\Theta := \\Theta - \\alpha \\Delta$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Propagacja wsteczna – podsumowanie\n",
    "\n",
    "* Algorytm pierwszy raz wprowadzony w latach 70. XX w.\n",
    "* W 1986 David Rumelhart, Geoffrey Hinton i Ronald Williams pokazali, że jest znacznie szybszy od wcześniejszych metod.\n",
    "* Obecnie najpopularniejszy algorytm uczenia sieci neuronowych."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## 5.5. Implementacja sieci neuronowych"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>łod.dł.</th>\n",
       "      <th>łod.sz.</th>\n",
       "      <th>pł.dł.</th>\n",
       "      <th>pł.sz.</th>\n",
       "      <th>Iris setosa?</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.2</td>\n",
       "      <td>3.4</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.1</td>\n",
       "      <td>3.7</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.4</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6.7</td>\n",
       "      <td>3.1</td>\n",
       "      <td>5.6</td>\n",
       "      <td>2.4</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6.5</td>\n",
       "      <td>3.2</td>\n",
       "      <td>5.1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.9</td>\n",
       "      <td>2.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1.7</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6.0</td>\n",
       "      <td>2.7</td>\n",
       "      <td>5.1</td>\n",
       "      <td>1.6</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   łod.dł.  łod.sz.  pł.dł.  pł.sz.  Iris setosa?\n",
       "0      5.2      3.4     1.4     0.2           1.0\n",
       "1      5.1      3.7     1.5     0.4           1.0\n",
       "2      6.7      3.1     5.6     2.4           0.0\n",
       "3      6.5      3.2     5.1     2.0           0.0\n",
       "4      4.9      2.5     4.5     1.7           0.0\n",
       "5      6.0      2.7     5.1     1.6           0.0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas\n",
    "src_cols = ['łod.dł.', 'łod.sz.', 'pł.dł.', 'pł.sz.', 'Gatunek']\n",
    "trg_cols = ['łod.dł.', 'łod.sz.', 'pł.dł.', 'pł.sz.', 'Iris setosa?']\n",
    "data = (\n",
    "    pandas.read_csv('iris.csv', usecols=src_cols)\n",
    "    .apply(lambda x: [x[0], x[1], x[2], x[3], 1 if x[4] == 'Iris-setosa' else 0], axis=1))\n",
    "data.columns = trg_cols\n",
    "data[:6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1.   5.2  3.4  1.4  0.2]\n",
      " [ 1.   5.1  3.7  1.5  0.4]\n",
      " [ 1.   6.7  3.1  5.6  2.4]\n",
      " [ 1.   6.5  3.2  5.1  2. ]\n",
      " [ 1.   4.9  2.5  4.5  1.7]\n",
      " [ 1.   6.   2.7  5.1  1.6]]\n",
      "[[ 1.]\n",
      " [ 1.]\n",
      " [ 0.]\n",
      " [ 0.]\n",
      " [ 0.]\n",
      " [ 0.]]\n"
     ]
    }
   ],
   "source": [
    "m, n_plus_1 = data.values.shape\n",
    "n = n_plus_1 - 1\n",
    "Xn = data.values[:, 0:n].reshape(m, n)\n",
    "X = np.matrix(np.concatenate((np.ones((m, 1)), Xn), axis=1)).reshape(m, n_plus_1)\n",
    "Y = np.matrix(data.values[:, n]).reshape(m, 1)\n",
    "\n",
    "print(X[:6])\n",
    "print(Y[:6])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "150/150 [==============================] - 0s - loss: 2.0678 - acc: 0.6667     \n",
      "Epoch 2/10\n",
      "150/150 [==============================] - 0s - loss: 1.9711 - acc: 0.6667     \n",
      "Epoch 3/10\n",
      "150/150 [==============================] - 0s - loss: 1.8811 - acc: 0.6667     \n",
      "Epoch 4/10\n",
      "150/150 [==============================] - 0s - loss: 1.7793 - acc: 0.6667     \n",
      "Epoch 5/10\n",
      "150/150 [==============================] - 0s - loss: 1.6948 - acc: 0.6667     \n",
      "Epoch 6/10\n",
      "150/150 [==============================] - 0s - loss: 1.5993 - acc: 0.6667     \n",
      "Epoch 7/10\n",
      "150/150 [==============================] - 0s - loss: 1.5162 - acc: 0.6667     \n",
      "Epoch 8/10\n",
      "150/150 [==============================] - 0s - loss: 1.4308 - acc: 0.6667     \n",
      "Epoch 9/10\n",
      "150/150 [==============================] - 0s - loss: 1.3487 - acc: 0.6667     \n",
      "Epoch 10/10\n",
      "150/150 [==============================] - 0s - loss: 1.2676 - acc: 0.6667     \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f87f40aa150>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(3, input_dim=5))\n",
    "model.add(Dense(3))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "\n",
    "model.fit(X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8209257125854492"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict(np.array([1.0, 3.0, 1.0, 2.0, 4.0]).reshape(-1, 5)).tolist()[0][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## 5.6. Funkcje aktywacji"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "* Każda funkcja aktywacji ma swoje zalety i wady.\n",
    "* Różne rodzaje funkcji aktywacji nadają się do różnych zastosowań."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "slideshow": {
     "slide_type": "notes"
    }
   },
   "outputs": [],
   "source": [
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, SimpleRNN, LSTM\n",
    "from keras.optimizers import Adagrad, Adam, RMSprop, SGD\n",
    "\n",
    "from IPython.display import YouTubeVideo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "slideshow": {
     "slide_type": "notes"
    }
   },
   "outputs": [],
   "source": [
    "def plot(fun):\n",
    "    x = np.arange(-3.0, 3.0, 0.01)\n",
    "    y = [fun(x_i) for x_i in x]\n",
    "    fig = plt.figure(figsize=(14, 7))\n",
    "    ax = fig.add_subplot(111)\n",
    "    fig.subplots_adjust(left=0.1, right=0.9, bottom=0.1, top=0.9)\n",
    "    ax.set_xlim(-3.0, 3.0)\n",
    "    ax.set_ylim(-1.5, 1.5)\n",
    "    ax.grid()\n",
    "    ax.plot(x, y)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Funkcja logistyczna"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "$$ g(x) = \\frac{1}{1 + e^{-x}} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "* Przyjmuje wartości z przedziału $(0, 1)$."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### Funkcja logistyczna – wykres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2cAAAG9CAYAAACRcQ4FAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XmUnOld2Pvvr/d9qVa39m5JI41m\npNlHs4B9bXmwPXZibIgXILZjE+MBEgiG41zswAlmyQVuiOEe35DDEILZTojNvcEYfA0YW4MDnn3x\njMazat+l3vf1uX9UdUvqac20Rl2qt7u/n3PqVNVb71vz6Ph1q7963/d5I6WEJEmSJKm0yko9AEmS\nJEmScSZJkiRJmWCcSZIkSVIGGGeSJEmSlAHGmSRJkiRlgHEmSZIkSRlQtDiLiE0R8bmI+FZEjERE\niogti9z2UGH9+Y/vK9Z4JUmSJKmUKor43duBDwCPAd8E3n6Z2/818Jl5y56/8mFJkiRJUvYUM87+\nPqW0FiAifoTLj7NzKaUHl35YkiRJkpQ9RTutMaU0U6zvliRJkqSVJssTgnxv4Vq18Yh40OvNJEmS\nJK1kWY2zLwM/CdwLfBAYA/5nRHyopKOSJEmSpCKJlFLx/yP5a85+F9iaUjr0OrYvBx4E1qWUNl9i\nnfuA+wBqampu7+zsfP0D1oo0MzNDWVlW/z1CpeS+oYW4X+hS3De0EPcLLeSFF144l1JqX+z6xZwQ\nZMmklKYj4ovAr0fE+pTSyQXWuR+4H2Dnzp3p+eed2FEX27dvH3v37i31MJRB7htaiPuFLsV9Qwtx\nv9BCIuLw5ay/HPO++If6JEmSJOkqWxZxFhEVwA8AR1JKp0o9HkmSJElaakU9rTEi3ld4eXvh+Z0R\ncRY4m1J6oLDOFPAHKaWPFd7/EPAe4CvAUWAt8K+B24AfKuZ4JUmSJKlUin3N2Rfnvf/twvMDwN7C\n6/LCY9ZBoAP4j0AOGAYeBd6RUvrroo1UkiRJkkqoqHGWUorLXSel9CBwT9EGJUmSJEkZtCyuOZMk\nSZKklc44kyRJkqQMMM4kSZIkKQOMM0mSJEnKAONMkiRJkjLAOJMkSZKkDDDOJEmSJCkDjDNJkiRJ\nygDjTJIkSZIywDiTJEmSpAwwziRJkiQpA4wzSZIkScoA40ySJEmSMsA4kyRJkqQMMM4kSZIkKQOM\nM0mSJEnKAONMkiRJkjLAOJMkSZKkDDDOJEmSJCkDjDNJkiRJygDjTJIkSZIywDiTJEmSpAwwziRJ\nkiQpA4wzSZIkScoA40ySJEmSMsA4kyRJkqQMMM4kSZIkKQOMM0mSJEnKAONMkiRJkjLAOJMkSZKk\nDDDOJEmSJCkDjDNJkiRJygDjTJIkSZIywDiTJEmSpAwwziRJkiQpA4wzSZIkScoA40ySJEmSMsA4\nkyRJkqQMMM4kSZIkKQOMM0mSJEnKAONMkiRJkjLAOJMkSZKkDDDOJEmSJCkDjDNJkiRJygDjTJIk\nSZIywDiTJEmSpAwwziRJkiQpA4wzSZIkScoA40ySJEmSMsA4kyRJkqQMMM4kSZIkKQOMM0mSJEnK\nAONMkiRJkjLAOJMkSZKkDChanEXEpoj4XER8KyJGIiJFxJZFblsWEZ+OiEMRMRYRT0XEe4s1VkmS\nJEkqtWIeOdsOfADoBb55mdv+MvAZ4P8G3gk8CHwxIv7JUg5QkiRJkrKioojf/fcppbUAEfEjwNsX\ns1FEdACfBH4tpfQbhcXfiIjtwK8BXynGYCVJkiSplIp25CylNPM6N70XqAL+eN7yPwZujIitVzQw\nSZIkScqgLE4IshsYB16at3x/4XnX1R2OJEmSJBVfFuMsB/SllNK85T0XfC5JkiRJK0oxrzm7qiLi\nPuA+gPb2dvbt21faASlzhoaG3C+0IPcNLcT9QpfivqGFuF9oKWQxznqBloiIeUfPZo+Y9SywDSml\n+4H7AXbu3Jn27t1b1EFq+dm3bx/uF1qI+4YW4n6hS3Hf0ELcL7QUsnha436gGrhm3vLZa82evbrD\nkSRJkqTiy2KcfRWYBD44b/mHgGdSSgev/pAkSZIkqbiKelpjRLyv8PL2wvM7I+IscDal9EBhnSng\nD1JKHwNIKZ2JiM8Cn46IQeBx4AeAe4B3F3O8kiRJklQqxb7m7Ivz3v924fkBYG/hdXnhcaGfA4aA\nnwLWAc8DH0gp/WVxhilJkiRJr256JjEyMcXw+DTDE1OMjE8zND6VXzYxzfD4FMPjU4xMTFNdcfkn\nKRY1zlJK8XrWSSlNA79SeEiSJEnS6zIzkxiemGJofIqhsSkGC8/D4+dfD43nH4Ozr8cm596PzEbX\nxBRjkzOL/u9uztVe9lizOFujJEmSJDE9kxgam2JgbJL+0UkGRicZGJtkYHSK/tFJBscmXyWwzi9b\njNrKchpqKmisrqChpoKG6go25+poqK6grqqc+uoK6qsqqK8up67wXF9VQV3hub46v05dVX79yvIy\n4mcv789rnEmSJEkqipQSwxPTr4iq2ff511OFzwrvx6byn4/mw+vVREBD9cVB1VRbycaWWhouWHbR\n63kB1lhdSX11ORXlpZ8r0TiTJEmS9JrGp6bpG5mkd2SCvpFJ+kYm6C287y889xaW59ebpH90gsnp\n9Krf21BdQVNNPqpmw+r69Y0011bSVJNfln9dWKemkua6/Pv6qgrKyl7zSqplwziTJEmSVpmJqRl6\nhifoHh6ne2iCnuEJzg2Nn4+v0UJ8DRdia3SSkYnpS35fVUUZrXWVtNZV0VJXyfaOBlrqKmmpq6Jl\nNq7mIquSptoKmmoqaaypyMQRq6wwziRJkqRlbmJqht6RfGD1DE/QPTRB9/AEPYX46h6eoHv2s+EJ\nBscWPl2wLMgHVV0lLbWVrG+u4fr1TbTWVc7FVmtdFa11+aNXrYX3tVXzJ1/X62GcSZIkSRk0NjnN\nuaFxzg4WHkPjnBnIP58bHC/EVz7ILhVb5WVBrr6Ktvoq2hqquLG1Jf+6vopcw+zy6rl1mmoqV9Rp\ngsuNcSZJkiRdJTMzid6RCc5eEF1nBi8IsLkIG2NggeCKgFxdFWsaqmlrqOKGjc350CrEl7G1vBln\nkiRJ0hVKKdE/OsnpgTFO9Y9x6oLn0/1jcwF2bmicqZlXTpBRW1lOR1M17Q3VXLu2gTdc00Z7YzXt\njdV0NNbMvc7VV1HpNVorlnEmSZIkvYqp6RnODo3nY2s2vArRdWpgjNMD4xzvHWHir//mFdvm6qtY\n21TD2qZqrl/fmI+shmraC8HVUYiu+mp/LZdxJkmSpFUspcS5oQlO9I1yom+U432jnOgby7/vH+VU\n/xjnhsaZf7CrsjxY21TDuqYadm1o4tr6cfbs3sHa5vyydU01dDRVU1PpRBlaPONMkiRJK9bY5PQF\n0TXK8dnwmn30jzExNXPRNnVV5WxoqWV9cw3XrWtkXVPNXHStbaphXXMNubqqi67l2rdvH3vftO1q\n//G0whhnkiRJWraGx6c42jvC0Z5RjvSMcKx3pBBe+QjrHp64aP0IWNtYw4aWGm7Y2My9u9exoaW2\n8KhhY0stzbWVRDiJhq4+40ySJEmZNTE1w4m+0bkAyz8XHr2j9MyLr7qqcja15mPrxk3NbCxE14bm\n/LJ1zTVOqKHMMs4kSZJUMiklzg6Oc6RnhKO9IxzpPh9gx3pHOdk/etH1XhVlwcbWWjpzddy7oZnN\nuVo2t9bRmatjc66O1jqPemn5Ms4kSZJUVCklTg+Mc6h7mEPnhjnUPcLh7mEOnhvmcPcIo5PTF63f\n0VhNZ66OO7fm2Nxay6bc+fha11RDufft0gplnEmSJOmKzcwkTg+OzQXXbIjNvh6bPD/pRmV5sDlX\nx5a2er7rmja2tNXT2VbH5tY6NrXWOsOhVi3jTJIkSYvWNzLBy2eHePnsMC+fHeLg2XyAHe65OMCq\nysvYnKtlS1s9b9i+hi1tdWxZU8+WtnrWN9dQ4XVf0isYZ5IkSbrI1PQMR3tHefnMEAfODfHymeH8\n89nhiybgqCovo7MtfwTsf9uxZi6+utrq2NBS6+mH0mUyziRJklap/pFJXj43xIHCUbADhSNih7uH\nmZw+PwtHW30V17Q3cO/utWxb08A1HfVsW9PAptZaj4BJS8g4kyRJWuF6hyd48cwQL5we5MXTg7xw\neogXzwxxbmh8bp2KsqCrrY5r2ht46/Vruaa9nm3tDVzTXk9LXVUJRy+tHsaZJEnSCtE3MsELp89H\nWD7ILo6whuoKtnc08Jad7WzvaOCa9ga2tdezOVfn/b+kEjPOJEmSlpnZCHvxzCAvFmJsfoTVV5Wz\nY20jb9nZzrVrG9mxtoEdaxvZ0FzjfcCkjDLOJEmSMmpiaoYD54Z47uQg3zk1wHMnB3nu1ACnBy6O\nsO0XRNj2tQ1ca4RJy5JxJkmSVGIpJc4MjvOdkwM8d2qQ5wrPL58dmpuYo6q8jO0dDbxh+xquW9fI\njrWNRpi0whhnkiRJV9HoxDQvnM4fAfvOyUGeP5V/3TsyObfOhuYarlvfxFuu6+C6dY1cv76JrWvq\nvSZMWuGMM0mSpCLpGZ5g/4l+9p8YKDz6OXhumFSYpb62spyd6xp5xw3ruG5dE9eta+S6dU0011WW\nduCSSsI4kyRJukIpJU70j7H/eP9chO0/McDJ/rG5dTa21LJrQxPfe9MGrl+fj7DOXB1l3qhZUoFx\nJkmSdBlmUuKlM0MXHBHLP/cVTkssC9jW3sCdW3Ps3tDE7g3N7FrfRGu99wqT9OqMM0mSpEuYnJ7h\n+VOD7D/RzzPHBwrPI0z89QNAfpKOnesaecfudeze0MSuDc1cv76Ruip/xZJ0+fzJIUmSBEzPJF4+\nO8S3j/Xz7WN9fPtYP8+eHGBiagbI37x51/om3rypgnvv3M3uDU1s72hwkg5JS8Y4kyRJq87MTOJw\nz8hchD19rJ9nTvQzMjEN5EPsho1NfPS7t3DjxmZu3Ng8d33Yvn372Hv7phL/CSStRMaZJEla0VJK\nHO8bLRwR6+fp4/kgGxybAqCmsozdG5r5wJ7N3LSpmZs2tbBtTb0TdUi66owzSZK0onQPjfPk0T6e\nOtrHU8f6efp4Pz3DEwBUlgfXr2/i3Tdv4OZNLdy4qZkdHQ1UeGqipAwwziRJ0rI1PjXNsycGeOJI\nH08ezT+O9IwAUF4W7Oho4K3Xd3DTphZu2tTMznWNVFeUl3jUkrQw40ySJC0LKSWO9ozyxNHeuRh7\n9sQAE9P5CTvWN9dwa2cLH7q7k1s7W7lhQzO1VYaYpOXDOJMkSZk0MDbJt4/288SR3rmjYt2F0xNr\nK8u5cVMzP/zGLdy6uYVbNreyrrmmxCOWpCtjnEmSpJKbmp7hhdNDPHm0by7GXjo7REr5z7d3NPCW\n6zq4tbOFWza3sHNto9eJSVpxjDNJknTVDYxN8sSRPh471MNjR3p58kgfw4Vp7HP1VdyyuYXvvXkD\nt3a2cNOmFpprK0s8YkkqPuNMkiQVVUqJIz0jPHqol8eO9PLYoV5eODNISlAWcP36Jt57+yZu62zl\n1s4WOnN1RDiNvaTVxziTJElLanxqmmeO9/PY4V4ePdTL40d6OTeUv1assbqCW7ta+ac3ref2rlZu\n3txCQ7W/jkgSGGeSJOkKnR0c5/EjvTx2OP94+lj/3AyKXW11vOnadm7vamVPV44dHQ3e3FmSLsE4\nkyRJizYzk3jxzBCPHu6Zi7HD3fn7ilWVl3HjpmY++oYt3N7Vym2drbQ3Vpd4xJK0fBhnkiTpkiam\nZnj6eD+PHOrhkYM9PHq4l/7RSQDWNFRxW2crH7yrk9u7WrlhY7M3eJakK2CcSZKkOcPjUzx+pJdH\nDvbw8KEenjzax9hk/hTFbWvqecfuddyxNceerla62py4Q5KWknEmSdIq1j00ziOHevNHxg71sP/E\nANMzibKAXRua+KE7O7lzS449W3KeoihJRWacSZK0ihzrHeHhg/kQe/hgDy+fHQagqqKMWza38ONv\nvoY7tua4rbOFxhrvLSZJV5NxJknSCjUzk3jp7NBcjD1ysIcT/WMANNZUsKerlffevok7t+S4cZPX\ni0lSqRlnkiStEJPTMzxTmLzj4YO9PHq4h76R/OQdHY3V3LE1x49uyXHHlhw71zVS7pT2kpQpxpkk\nScvUxNQM3z7Wx4MHunnoYH5q+5GJaQC2rqnn7bvWcseWHHduzdGZc/IOSco640ySpGVifGqaJ4/0\n8dDBHh480M3jR3rnZlK8bl0j7799E3dubeOOra10NNaUeLSSpMtlnEmSlFFjk9M8cWT2yFg3jx/p\nY2Jqhgi4bl1+JsW7trZx59YcufqqUg9XknSFjDNJkjJidGKax4/08tCBbh48kL/H2MT0zNy09h++\nu4u7t7Vx55YczXXOpChJK41xJklSiQyPT/HY4V4eOtjNQwd6eOpYH5PT+XuM3bixmY++YQt3bc3f\nY6y51hiTpJXOOJMk6SoZGp/i0UM9PHigh4cOdvP0sX6mZhLlZcGNG5v52Bu3cde2HHu6Wr3HmCSt\nQsaZJElFMjA2yaOHenjoQH4Cj2dODDA9k6gsD27a1MJ9b9rG3dvauL2rlfpq/0qWpNWuqH8TRMRm\n4DeBtwEBfA34RErpyCK2TZf46NaU0pNLN0pJkpZG/+gkjxRmUnzoYA/7T/Qzk6CqvIxbNrfwr/Ze\nw11b27itq4W6KmNMknSxov3NEBF1wNeBceAjQAJ+BfhGRNyUUhpexNd8HvidecteWMpxSpL0evWN\nTPDQwfNHxr5zaoCUoKqijFs3t/CT9+zgrm05butspaayvNTDlSRlXDH/2e7jwDZgZ0rpJYCI+Dbw\nIvCjwGcX8R3HU0oPFm+IkiQtXvfQOA8f7Jm7z9hzpwYBqK4o4/auVj7xPddy97YcN29uMcYkSZet\nmHH2buDB2TADSCkdjIh/AN7D4uJMkqSSOTc0zkOFyTsePNDNC6eHAKitLGfPllbeddN67trWxk2b\nmqmuMMYkSVemmHG2G/jSAsv3A+9f5Hf8eET8W2AaeBD4hZTSN5dofJIkXeTs4PhciH396RFOfPVr\nANRXlXP7lhzvuWUjd29r48aNzVRVlJV4tJKklSZSutS8G1f4xRETwGdTSp+at/xXgE+llF41DCPi\nj4C/BE4AXcC/BXYBb0sp7Vtg/fuA+wDa29tv/8IXvrAUfwytIENDQzQ0NJR6GMog943Vq29shud6\nZ3iuZ5rne6Y5OZz/O7GmHLY1JW5or2JnrpyupjIqyqLEo1VW+DNDC3G/0ELe8pa3PJZS2rPY9TMb\nZwt8XyPwDHA0pfTGV1t3586d6fnnn7/cIWuF27dvH3v37i31MJRB7hurx+mBMR480J2/z9iBbg6c\ny89N1VhdwR1bc9y1Ncfd29rYvaGJ//XNv3e/0IL8maGFuF9oIRFxWXFWzNMae4HWBZbnCp9dlpTS\nYET8FfCxKx2YJGl1ONU/VpjWPh9kBy+IsTu35vihOzu5a1uOXeubqCj3NEVJUmkVM872k7/ubL5d\nwLNX8L3FOdQnSVr2TvSN5kPs5fwkHoe6RwBoqsnH2Afv6uTubW1cv76Jck9TlCRlTDHj7C+A34iI\nbSmlAwARsQV4A/CpV9luQRHRBLwLeHgJxyhJWsaO943y4Mvnj4wd6cnHWHNtJXduzfGhu7uMMUnS\nslHMOPtd4CeAL0XEz5M/4vXLwFEuuLF0RHQBLwO/lFL6pcKyTwI7gW9wfkKQTwLrgA8WccySpAw7\n2jMyd4+xhw52c7RnFICWukru3JLjo9+9hbu3tXHdukbKjDFJ0jJTtDhLKQ1HxD3AbwJ/BATwd8An\nUkpDF6waQDlw4cn+zwPfX3g0AwPAPwAfSyl55EySVoGUEsd6R/nWgW4eOpAPsuN9+Rhrravkrq1t\n/Ms3bOXubW3sXGuMSZKWv2IeOSOldAR472usc4h8oF247MvAl4s3MklS1qSUONozWphNsZuHDvbM\nxViuvoq7tua4703buGtbjms7jDFJ0spT1DiTJOlSUkq8fHaYhw/28PDBfIyd7B8DoK2+iru25fjR\nN2/j7m1tbG9vMMYkSSuecSZJuiqmZxLPnRooxFj+0T08AcCahurCPcby9xnb3tFAhDEmSVpdjDNJ\nUlFMTM3w9PF+Hj7YwyOH8o/BsSkANrXW8uad7dy1NccdW3JsXVNvjEmSVj3jTJK0JEYnpnniaO/c\nUbHHj/QyNjkDwPaOBt5104Z8jG3NsbGltsSjlSQpe4wzSdLrMjA2yWOHenn4UD7Gvn2sj8npRATs\nWt/ED93ZyV1bc+zZkmNNQ3WphytJUuYZZ5KkRekeGueRQz08VDgy9p2TA8wkqCwPbtrUwsfeuI27\ntua4rauV5trKUg9XkqRlxziTJL3C7D3G8teK9fLIoR5eOpO/RWVNZRm3dbbyb75nB3duzXHr5lZq\nq8pLPGJJkpY/40ySxNT0DN85Ocgjh3p47HA+xs4MjgPQWFPBHVtyvPe2Tdy5NceNG5upqigr8Ygl\nSVp5jDNJWoWGxqd44kgvjxzq5bHDPTxxpI+RiWkANrbU8l3XtLFnS449Xa1cu7aRcu8xJklS0Rln\nkrQKnOwf5dFDvTx6qIdHD/fOXS9WFnD9+ibef/umfIxtaWV9szMpSpJUCsaZJK0w0zOJF04P8ujh\nQowd6uV43ygAtZXl3NrZwk/cs4M9Xa3c2tlCY42Td0iSlAXGmSQtc0PjUzx1tI/HD/fy6OFeHj/S\nO3ez5/bGau7Y0srH3riVPVtauX59E5XlXi8mSVIWGWeStIyklDhwbpjHD/fy+JE+njjSywunB5lJ\n+c93FG72vKerlTu25NicqyXC68UkSVoOjDNJyrALj4o9fqSXJ4720TcyCeRnUbxlcwv37l7HbV2t\n3LKpheY6T1GUJGm5Ms4kKSMWOir2/OlB0gVHxe7dtY5bO1u4rauV7e0NlDmLoiRJK4ZxJkklMjg2\nyVNH+3niyMJHxW7tbD1/VGxzC821HhWTJGklM84k6SqYmJrh+VODPHmsj6eO5h8vnR16xVGx27pa\nuK2zlWs8KiZJ0qpjnEnSEpuZSRzqHuapY308dbSfJ4/28ezJASamZgBoq6/ils0tvOumDdza2cLN\nHhWTJEkYZ5J0xc4MjPHk0T6+fay/EGR9DBSmsq+tLOfGTc189Lu3cPOmFm7e3MzGFmdQlCRJr2Sc\nSdJlGByb5Onj/Tx1tD9/euKxPk72jwFQXhbsXNvIP71pA7dsbubmzS1sb2+gwvuKSZKkRTDOJOkS\n+kcn2X+in2eO9/P08QGeOd7PwXPDc593tdVxx5YcN29u4ZbNzexa30xtVXkJRyxJkpYz40ySgP6R\nSZ450c/Tx/OPZ473c7h7ZO7zDc013LCxmX9260Zu2NTMLZtaaK2vKuGIJUnSSmOcSVp1eocn5kLs\nmeP9PPLSCGe/+jdzn29sqeXGjc18YM9mbtjYzA0bmmhrqC7hiCVJ0mpgnElasVJKnB0c59mTA+w/\nMcDTx/JBdrxvdG6dzblauprK+OE3b+fGjc3s3tBMziNikiSpBIwzSSvC1PQMB84N852TAzx7YoBn\nTw7wnZMDnBuamFunq62OWzpb+PB3dRVCrImWuir27dvH3r3bSzh6SZIk40zSMjQwNslzJwd59kQ/\n3zk5yLMnB3j+9ODcfcSqysu4dl0Db9nZwa4NTVy/Pv/wXmKSJCnLjDNJmZVS4ljvaP5oWOFI2LMn\nBzjac/60xFx9FbvWN/GR7+qaC7Fr2huodPp6SZK0zBhnkjKhe2ic508P8vypQV44PchzpwZ58fQQ\nQ+P5mzlHwNa2em7a1MIP3tHJrvVN7NrQREdjtTd0liRJK4JxJumqGh6f4oXT5wPshUKQXXhtWEtd\nJTvXNvLe2zZy7bpGrl/fxHXrGqmr8keWJElaufxNR1JRTEzNcPDcMM+dGpgLsOdPD150SmJtZTnX\nrs1fG7ZzXWP+sbaRdo+GSZKkVcg4k3RFhsenePnsEC+dufhxuGeE6ZkEQHlZsG1N/pTED9y+mWvX\nNXLdukY2t9ZRVmaESZIkgXEmaZF6hyd4qRBhL54e4qWzQ7x8Zuiie4ZVlAVdbXXsWNvAP7lxPds7\nGti5rpFt7fVUV5SXcPSSJEnZZ5xJmjMzkzjRP8rBc8OvOBLWPXz+mrCayjKuaW9gz5ZWfrB9M9s7\nGtixtoHOXD1VFc6SKEmS9HoYZ9Iqk1Kid2SSg+eGOHB2mIPn8o8DZ4c51D3MeOFeYQBNNRVs72jg\nrdevZXtHw9xjY0utpyNKkiQtMeNMWqFGJqbmwutgIcIOFN73j07OrVdRFnS21bFtTT1vunYNW9c0\nsGVNHds7GmhvcGIOSZKkq8U4k5axgbFJjnSPcKRnhMPdIxzpGebQuREOnhvm1MDYRetuaK5ha3s9\n33vzerauaWDbmnq2rqlnU2stFd6wWZIkqeSMMynDZmYSZwbHOdw9zOGeEY4WIuxwzwhHuofpHZm8\naP22+io62+r47u1thfhqYFt7PVva6qmtckIOSZKkLDPOpBIbm5zmeN8oR7pHXhFhR3pGLroGrCxg\nY2stXbl63nnjejpzdXTl6uhsq6MzV0djTWUJ/ySSJEm6EsaZVGSjE9Mc7xvhaO8ox3pHOd47yrHe\nEY735d+fHRy/aP3aynK62urYuqaevTvb6czV0dlWT1eujo2ttVR6CqIkSdKKZJxJV2hofOoVwXWs\nd6SwbPSiKegBKsuDDS21bGqt5Z6dHWxsrWVjSy1dbfkjYE7CIUmStDoZZ9KrGJ+a5nT/OCf6RznV\nP3b+uW+Mk/2jHO8bpW/edV9VFWVsKgTX2zc0s6m1du6xsaWOjsZqp6GXJEnSKxhnWrXGp6Y5MzDO\nib5RTg3kg+tU/ygn+vPhdap/jHNDE6/Yrrm2kvXNNaxvruHWzhY2tdaxsXAkbGNrLWvqjS9JkiRd\nPuNMK87MTKJnZILTA2OcGRznzMAYpwfGefK5cf7kyKOc6h/jZP8Y54bGX7FtU00F65trWd9Sw40b\nm1nfXMu65ho2FJata6qhvtr/20iSJGnp+Vumlo2ZmUT38ARnBsc4MzDOmcF8dM09F2Ls7OA4UzPp\nFds3VMLGiRHWNtewe0NTPsLXTpHLAAAVuUlEQVSaa1jfkj8Ktq65lgbDS5IkSSXib6IqqZQS/aOT\nnBsa59zQRP55cJzu4fzrs4MTnC3E17mhhaOrta6StU01tDdWs2NtIx2N1axtqqGjsZqOphrWNlXT\n3ljNt/7XN9m7900l+FNKkiRJr80405Kbmp6hZ2SCc4MTdA+PF4JrgnPDheeh/LLuofznk9OvDK6y\ngFx9NWsaquhoqmHH2kbWNl0cXR2N+eiqrvDmypIkSVr+jDO9qpQSwxPT9A5P0DM8Qc/IxPnXwxP0\njhSehyfPfzYyQXplb1FVXsaahirWNFbT0VjNrvVNrGmsZk1DPsLyz/nXLXVVlDuphiRJklYR42wV\nSSkxOD5F/8gk/aOTF4TVBD0jk3Nh1XtBePUOTzIxPbPg95WXBa11VeTqK2mtq2JHRwOt9VWsqa+6\nILrysdXWUE1TTYX375IkSZIuwThbZlJKjE3O0D86Sd/oBP0jk/SN5mNrYHSSvkJ4zS7rH5nIPxce\nC1yyBUAEtNRW0lpfRa6uis25Om7e1JJ/X4ivXH3V3Oet9VXGliRJkrSEjLOrbPY0wcGxSQbHphgc\nm2RgbGru9cXPUwyMzo+tSx/Jgvy1Wi11VTTXVtJUW0lLXRVdbfW01FXSXHv+0VJXRUvd+ehqrq30\nNEJJkiSphIyzRZo9YjU0PsXIxBRD41MMj08zNJ4PqYFLxNX810PjU5c8ejWrvCxorKnIP6oraamr\nZEdHAy11heCqrSoE1vzgqqSh2qNZkiRJ0nK0YuNsZiYxMjnNyPj5kBqemGK48H5kYvqi1/l1CuuN\nT82te+H714oqgIq5sKqcC6zNuToaaypoumDZ+c8rC5+df11bWW5gSZIkSavMioyzwwMzXPNzX1lw\nxsCF1FSWUV9VQX114VFVTmtdFZtb66irKqe+uoKG6grqqstpqK4orHt+eWNN5Vxc1VSWGVaSJEmS\nLtuKjLPGquAn37L9fGxVl1NfNRtYFTQUwqquKh9iFeVlpR6yJEmSpFWuqHEWEZuB3wTeBgTwNeAT\nKaUji9i2Bvhl4ENAC/Ak8LMppb9/rW1zNcHPvH3nlQxdkiRJkq6qoh0yiog64OvAdcBHgA8DO4Bv\nRET9Ir7i94CPA/8eeBdwEvjriLilOCOWJEmSpNIp5pGzjwPbgJ0ppZcAIuLbwIvAjwKfvdSGEXEz\n8M+Bf5lS+v3CsgeA/cAvAe8u4rglSZIk6aor5sVW7wYenA0zgJTSQeAfgPcsYttJ4H9csO0U8KfA\nvRFRvfTDlSRJkqTSKWac7QaeWWD5fmDXIrY9mFIaWWDbKmD7lQ9PkiRJkrKjmKc15oDeBZb3AK1X\nsO3s5xeJiPuA+wDa29vZt2/fogeq1WFoaMj9Qgty39BC3C90Ke4bWoj7hZbCiplKP6V0P3A/wM6d\nO9PevXtLOyBlzr59+3C/0ELcN7QQ9wtdivuGFuJ+oaVQzNMae1n4CNmljootdls4fwRNkiRJklaE\nYsbZfvLXjs23C3h2EdtuLUzHP3/bCeClV24iSZIkSctXMePsL4C7I2Lb7IKI2AK8ofDZq/kyUAm8\n/4JtK4AfAP4mpTS+1IOVJEmSpFIqZpz9LnAI+FJEvCci3g18CTgK/M7sShHRFRFTEfHvZ5ellJ4g\nP43+b0XEj0TE95CfRn8r8AtFHLMkSZIklUTR4iylNAzcA7wA/BHwJ8BB4J6U0tAFqwZQvsBYfhj4\nfeBXgL8CNgPvSCk9XqwxS5IkSVKpFHW2xpTSEeC9r7HOIfKBNn/5KPAzhYckSZIkrWjFPK1RkiRJ\nkrRIxpkkSZIkZYBxJkmSJEkZYJxJkiRJUgYYZ5IkSZKUAcaZJEmSJGWAcSZJkiRJGWCcSZIkSVIG\nGGeSJEmSlAHGmSRJkiRlgHEmSZIkSRlgnEmSJElSBhhnkiRJkpQBxpkkSZIkZYBxJkmSJEkZYJxJ\nkiRJUgYYZ5IkSZKUAcaZJEmSJGWAcSZJkiRJGWCcSZIkSVIGGGeSJEmSlAHGmSRJkiRlgHEmSZIk\nSRlgnEmSJElSBhhnkiRJkpQBxpkkSZIkZYBxJkmSJEkZYJxJkiRJUgYYZ5IkSZKUAcaZJEmSJGWA\ncSZJkiRJGWCcSZIkSVIGGGeSJEmSlAHGmSRJkiRlgHEmSZIkSRlgnEmSJElSBhhnkiRJkpQBxpkk\nSZIkZYBxJkmSJEkZYJxJkiRJUgYYZ5IkSZKUAcaZJEmSJGWAcSZJkiRJGWCcSZIkSVIGGGeSJEmS\nlAHGmSRJkiRlgHEmSZIkSRlgnEmSJElSBhhnkiRJkpQBxpkkSZIkZYBxJkmSJEkZYJxJkiRJUgYY\nZ5IkSZKUAcaZJEmSJGVA0eIsIsoi4tMRcSgixiLiqYh47yK3/XxEpAUev1Ws8UqSJElSKVUU8bt/\nGfgk8HPAY8APAl+MiHellL6yiO3PAu+et+zk0g5RkiRJkrKhKHEWER3kw+zXUkq/UVj8jYjYDvwa\nsJg4m0gpPViM8UmSJElS1hTrtMZ7gSrgj+ct/2PgxojYWqT/riRJkiQtS8WKs93AOPDSvOX7C8+7\nFvEdHRFxLiKmIuKFiPjZiChf0lFKkiRJUkYU65qzHNCXUkrzlvdc8PmreZL8dWr7gRrg+4FfBXYA\nP7LQBhFxH3AfQHt7O/v27XtdA9fKNTQ05H6hBblvaCHuF7oU9w0txP1CS2FRcRYRbwX+dhGrPpBS\n2ntFIwJSSvNnZfxKRAwBn4iIX08pvbjANvcD9wPs3Lkz7d17xcPQCrNv3z7cL7QQ9w0txP1Cl+K+\noYW4X2gpLPbI2T8C1y9ivZHCcy/QEhEx7+jZ7BGzHi7ffwc+AewBXhFnkiRJkrScLSrOUkojwHOX\n8b37gWrgGi6+7mz2WrNnL+O7XjGcK9hWkiRJkjKpWBOCfBWYBD44b/mHgGdSSgdfx3d+kHyYPXKF\nY5MkSZKkzCnKhCAppTMR8Vng0xExCDwO/ABwD/NuLB0Rfwd0pZS2F953AX8E/Cn5o27V5CcE+Sjw\nOymll4sxZkmSJEkqpWLN1gjwc8AQ8FPAOuB54AMppb+ct175vHEMkr8m7WeBtcAM+VMq/w3w20Uc\nryRJkiSVTNHiLKU0DfxK4fFq6+2d974H+L5ijUuSJEmSsqhY15xJkiRJki6DcSZJkiRJGWCcSZIk\nSVIGGGeSJEmSlAHGmSRJkiRlgHEmSZIkSRlgnEmSJElSBhhnkiRJkpQBxpkkSZIkZYBxJkmSJEkZ\nYJxJkiRJUgYYZ5IkSZKUAcaZJEmSJGWAcSZJkiRJGWCcSZIkSVIGGGeSJEmSlAHGmSRJkiRlgHEm\nSZIkSRlgnEmSJElSBhhnkiRJkpQBxpkkSZIkZYBxJkmSJEkZYJxJkiRJUgYYZ5IkSZKUAcaZJEmS\nJGWAcSZJkiRJGWCcSZIkSVIGGGeSJEmSlAHGmSRJkiRlgHEmSZIkSRlgnEmSJElSBhhnkiRJkpQB\nxpkkSZIkZYBxJkmSJEkZYJxJkiRJUgYYZ5IkSZKUAcaZJEmSJGWAcSZJkiRJGWCcSZIkSVIGGGeS\nJEmSlAHGmSRJkiRlgHEmSZIkSRlgnEmSJElSBhhnkiRJkpQBxpkkSZIkZYBxJkmSJEkZYJxJkiRJ\nUgYYZ5IkSZKUAcaZJEmSJGWAcSZJkiRJGWCcSZIkSVIGGGeSJEmSlAHGmSRJkiRlgHEmSZIkSRlQ\ntDiLiJ+JiC9HxMmISBHxmcvc/o0R8Y8RMRoRpyLisxFRW6ThSpIkSVJJFfPI2ceBDuDPL3fDiLgJ\n+FvgDPAu4OeBHwY+v4TjkyRJkqTMqCjid+9OKc1ERAXwY5e57S8Cx4D3p5QmASJiAviDiPj1lNLj\nSzxWSZIkSSqpoh05SynNvJ7tIqISeAfwhdkwK/gCMAG8ZwmGJ0mSJEmZksUJQa4BaoBnLlyYUhoD\nXgZ2lWJQkiRJklRMWYyzXOG5d4HPei74XJIkSZJWjEVdcxYRbyU/QcdreSCltPeKRvQ6RcR9wH0A\n7e3t7Nu3rxTDUIYNDQ25X2hB7htaiPuFLsV9Qwtxv9BSWOyEIP8IXL+I9UauYCyzZo+YtS7wWQ7Y\nv9BGKaX7gfsBdu7cmfbu3bsEQ9FKsm/fPtwvtBD3DS3E/UKX4r6hhbhfaCksKs5SSiPAc0Uey6yX\ngXFg94ULI6IG2AZ88SqNQ5IkSZKumsxdc5ZSmgC+CnygMA3/rPcB1cBflGRgkiRJklRERbvPWUTs\nAbZwPgB3RcT7Cq+/UjgaR0T8HvCRlNKFY/kM8CDwhYj4z4Xv+Y/An6WUHivWmCVJkiSpVIp5E+qf\nAD5ywfv3Fx4AW4FDhdflhceclNKTEfF24NeBvwL6gT8E/l0RxytJkiRJJVO0OEspfRT46OtdL6X0\n98B3LfGwJEmSJCmTMnfNmSRJkiStRsaZJEmSJGWAcSZJkiRJGWCcSZIkSVIGGGeSJEmSlAHGmSRJ\nkiRlgHEmSZIkSRlgnEmSJElSBhhnkiRJkpQBxpkkSZIkZYBxJkmSJEkZYJxJkiRJUgYYZ5IkSZKU\nAcaZJEmSJGWAcSZJkiRJGWCcSZIkSVIGGGeSJEmSlAHGmSRJkiRlgHEmSZIkSRlgnEmSJElSBhhn\nkiRJkpQBxpkkSZIkZYBxJkmSJEkZYJxJkiRJUgYYZ5IkSZKUAcaZJEmSJGWAcSZJkiRJGWCcSZIk\nSVIGGGeSJEmSlAHGmSRJkiRlgHEmSZIkSRlgnEmSJElSBhhnkiRJkpQBxpkkSZIkZYBxJkmSJEkZ\nYJxJkiRJUgYYZ5IkSZKUAcaZJEmSJGWAcSZJkiRJGWCcSZIkSVIGGGeSJEmSlAHGmSRJkiRlgHEm\nSZIkSRlgnEmSJElSBhhnkiRJkpQBxpkkSZIkZYBxJkmSJEkZYJxJkiRJUgYYZ5IkSZKUAcaZJEmS\nJGWAcSZJkiRJGWCcSZIkSVIGGGeSJEmSlAHGmSRJkiRlQNHiLCJ+JiK+HBEnIyJFxGcuY9vPFLaZ\n//jzYo1XkiRJkkqpoojf/XFgAPhz4Mde53e8EZi+4H3PlQ5KkiRJkrKomHG2O6U0ExEVvP44eyil\nNLWUg5IkSZKkLCraaY0ppZlifbckSZIkrTRZnxDkaERMR8ThiPj1iKgt9YAkSZIkqRiKeVrjlXgJ\n+BTwBJCAtwM/DdwGvK2E45IkSZKkolhUnEXEW4G/XcSqD6SU9l7RiICU0h/PW/S3EXEM+K2IeGtK\n6WsLjPE+4L7C2/GIeOZKx6EVZw1wrtSDUCa5b2gh7he6FPcNLcT9QgvZeTkrL/bI2T8C1y9ivZHL\n+Y9fpv8O/BZwB/CKOEsp3Q/cDxARj6aU9hRxLFqG3C90Ke4bWoj7hS7FfUMLcb/QQiLi0ctZf1Fx\nllIaAZ57XSNaeqnUA5AkSZKkpZb1CUEu9MHC88MlHYUkSZIkFUHRJgSJiD3AFs4H4K6IeF/h9VcK\nR+OIiN8DPpJSqrhg2yeAPwSeJ3+k7G3ATwJfTSl9fRH/+fuX5A+hlcb9QpfivqGFuF/oUtw3tBD3\nCy3ksvaLSKk4ZwlGxOeBj1zi460ppUMXrpdSigu2/VPy15atJx93B8hfc/Z/ppTGizJgSZIkSSqh\nosWZJEmSJGnxltM1Z5ctIhoj4gsR8VJEDEdEX0Q8HBEfKvXYVDoRcW1E/F8R8e2IGIqIkxHxFxFx\nc6nHptKLiJ+JiC8X9osUEZ8p9Zh09UTE5oj4s4joj4iBiPh/I6Kz1ONSaUXEpoj4XER8KyJGCj8b\ntpR6XCqtiHhfRPw/EXE4IkYj4vmI+NWIaCz12FRaEXFvRHw9Ik5FxHhEHCs0ya7X2nZFxxlQBUwB\nvwq8G/jnwHeAP4qIny7lwFRSbwfeAvwB8L3AvwLagQcj4vZSDkyZ8HGgA/jzUg9EV1dE1AFfB64j\nf1r+h4EdwDcior6UY1PJbQc+APQC3yzxWJQdnwSmgX8HvAP4L8CPk78/70r/HVuvLgc8BvwE+d87\nPw3sJv+7ZterbbgqT2uMiG8BDSmlG0s9Fl19EbEG6E4X7PwR0QwcAr6cUvoXpRqbSi8iylJKMxFR\nAUwCv5hS+kyJh6WrICJ+CvgssDOl9FJh2VbgReB/Tyl9tpTjU+nM/lwovP4R4He54Pp5rU4R0Z5S\nOjtv2b8g/4+/37PISey0SkTETvK3JvtkSuk/XWq91Vr13eSPqGkVSimdS/P+VSKl1A+8AGwszaiU\nFbO/gGlVejfw4GyYAaSUDgL/ALynZKNSyflzQQuZH2YFjxSe/X1C83UXnl+1QVZFnEVeRUS0RcR9\nwL3Ab5Z6XMqOiMgBN5A/7VXS6rQbeGaB5fuB17xOQJKANxee/X1CRER5RFRFxA7gd4BT5Gegv6Si\n3ecsY/418LnC60ngp1JKf1jC8Sh7PgcE8FulHoikksmRv6Zovh6g9SqPRdIyExEbgV8CvpZSerTU\n41EmPATMzmfwEnBPSunMq22wrI6cRcRbCzMkvdZj37xN/wf5+6a9E/ivwOci4kev9vhVHFewX8xu\n/2nyk8X8xIWnM2n5u9J9Q5KkxYiIBuBL5E9Z++ESD0fZ8WHgbvK/Zw6Qnyxmy6ttsNyOnP0jcP0i\n1hu58E3hnODZ84K/WpiR6zci4r+llCaXeIy6+l7XfgEQET8G/B/Az6eU/ttSD0wl97r3Da1KvSx8\nhOxSR9QkiYioBb4MbAPenFI6VuIhKSNSSrOntz4UEf8f+cnnPgX82KW2WVZxllIaIT/LyZV6lPw0\nyWsB/w+0zL3e/SIiPgz8NvCfUkr/YckHppJbwp8ZWh32k7/ubL5dwLNXeSySloGIqAT+DNgDvC2l\n9HSJh6SMSin1RcRL5G/NcUnL6rTGJfRmYAh41XM+tXJFxPcDvw/815TSJ0s9HkmZ8BfA3RGxbXZB\n4fSTNxQ+k6Q5hXuZ/QlwD/B9KaUHSzwkZVhErCV/H82XX229ZXXk7HIVriu7G/ga+SNkbeRvIvk+\n4FMppYkSDk8lEhFvIj9TzlPA5yPi7gs+Hk8pPVGakSkLImIPsIXz/3i1KyLeV3j9lcLROK1Mv0v+\nhqFfioifBxLwy8BR8rNsaRW74OfA7MX974yIs8DZlNIDJRqWSus/A+8H/gMwPO/3iWOe3rh6RcT/\nBB4Hvk3+WrNrgZ8mf03iJe9xBiv8JtQR8d3AzwO3kr9m4Bz5qU1/M6X0V6Ucm0onIj4D/MIlPj6c\nUtpy9UajrImIz5M/7Xkh3nR2hYuITvK3Wnkb+Rlc/w74hP+7KyIu9QvTAymlvVdzLMqGiDgEdF3i\n419MKX3m6o1GWRIRP0v+gNA1QBX5f+TbB/zqa/19sqLjTJIkSZKWi9V6zZkkSZIkZYpxJkmSJEkZ\nYJxJkiRJUgYYZ5IkSZKUAcaZJEmSJGWAcSZJkiRJGWCcSZIkSVIGGGeSJEmSlAHGmSRJkiRlwP8P\n9K/ZqnBq/u4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f8808515990>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(lambda x: 1 / (1 + math.exp(-x)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Tangens hiperboliczny"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "$$ g(x) = \\tanh x = \\frac{e^{x} - e^{-x}}{e^{x} + e^{-x}} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "* Przyjmuje wartości z przedziału $(-1, 1)$.\n",
    "* Powstaje z funkcji logistycznej przez przeskalowanie i przesunięcie."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### Tangens hiperboliczny – wykres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2cAAAG9CAYAAACRcQ4FAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3Xd8W/W9//H3V7LkPRM7znI2zh5k\nshM2ZZWy9w4tpXTclu7bUri/Cx20vW25lwTCCJAWKFBWKaMYAmRAErL33rbjeFvW+v7+kDJxwEms\nnGP59Xw89JB0dI78SXsa59Wjc2SstQIAAAAAOMvj9AAAAAAAAOIMAAAAAFyBOAMAAAAAFyDOAAAA\nAMAFiDMAAAAAcAHiDAAAAABcIGFxZozpYYz5kzFmljGm0RhjjTG9W7nthvj6B9++mqh5AQAAAMBJ\nKQl87/6SrpA0T9JMSWcf5vb/kvTLg5atPPqxAAAAAMB9EhlnH1hru0iSMeY2HX6cVVprZ7f9WAAA\nAADgPgn7WKO1Npqo9wYAAACAZOPmC4JcGD9XrdkYM5vzzQAAAAAkM7fG2auSviXpHEnXSgpIeskY\nc52jUwEAAABAghhrbeJ/SOycs6mS+lhrNxzB9l5JsyUVW2t7HmKdyZImS1JaWtrokpKSIx8YSSka\njcrjcev/HwEnsW+gJewXOBT2DbSE/QItWbVqVaW1trC16yfygiBtxlobMcY8L+lBY0xXa+32FtaZ\nImmKJJWWltqVK7mwIw5UVlamiRMnOj0GXIh9Ay1hv8ChsG+gJewXaIkxZuPhrN8e8z7xh/oAAAAA\n4BhrF3FmjEmRdKWkTdbaHU7PAwAAAABtLaEfazTGXBZ/ODp+f54xpkJShbX2/fg6YUlPWmtvjT+/\nWtLFkt6QtFlSF0nflHS8pKsTOS8AAAAAOCXR55w9f9Dzh+P370uaGH/sjd/2WC+pSNJvJBVIapD0\nqaRzrbX/StikAAAAAOCghMaZtdYc7jrW2tmSTk/YUAAAAADgQu3inDMAAAAASHbEGQAAAAC4AHEG\nAAAAAC5AnAEAAACACxBnAAAAAOACxBkAAAAAuABxBgAAAAAuQJwBAAAAgAsQZwAAAADgAsQZAAAA\nALgAcQYAAAAALkCcAQAAAIALEGcAAAAA4ALEGQAAAAC4AHEGAAAAAC5AnAEAAACACxBnAAAAAOAC\nxBkAAAAAuABxBgAAAAAuQJwBAAAAgAsQZwAAAADgAsQZAAAAALgAcQYAAAAALkCcAQAAAIALEGcA\nAAAA4ALEGQAAAAC4AHEGAAAAAC5AnAEAAACACxBnAAAAAOACxBkAAAAAuABxBgAAAAAuQJwBAAAA\ngAsQZwAAAADgAsQZAAAAALgAcQYAAAAALkCcAQAAAIALEGcAAAAA4ALEGQAAAAC4AHEGAAAAAC5A\nnAEAAACACxBnAAAAAOACxBkAAAAAuABxBgAAAAAuQJwBAAAAgAsQZwAAAADgAsQZAAAAALgAcQYA\nAAAALkCcAQAAAIALEGcAAAAA4ALEGQAAAAC4AHEGAAAAAC5AnAEAAACACxBnAAAAAOACxBkAAAAA\nuEDC4swY08MY8ydjzCxjTKMxxhpjerdyW48x5sfGmA3GmIAxZqEx5tJEzQoAAAAATkvkkbP+kq6Q\ntFvSzMPc9j5Jv5T0Z0nnSZot6XljzFfackAAAAAAcIuUBL73B9baLpJkjLlN0tmt2cgYUyTp+5Ie\nsNb+Nr74PWNMf0kPSHojEcMCAAAAgJMSduTMWhs9wk3PkeSX9PRBy5+WNMwY0+eoBgMAAAAAF3Lj\nBUGGSGqWtOag5Uvj94OP7TgAAAAAkHiJ/FjjkSqQVG2ttQctr9rvdQAAAAA4KtGoVcRaRaJW0T33\nUX1u2QGvW6tIVHsfhw9+Pf6efu/hHwdzY5wdEWPMZEmTJamwsFBlZWXODgTXqa+vZ79Ai9g30BL2\nCxwK+wZa0lH3i1icKHazUji63/P441BUilgp9CWvReKvRWwsjqJWsQiy+5ZFooot33OL2n3PD1o3\nesDzWIiFrQ5434OPBrWlTmnmsLdxY5ztlpRnjDEHHT3bc8SsqoVtZK2dImmKJJWWltqJEycmdEi0\nP2VlZWK/QEvYN9AS9gscCvsGWuKW/SIYjqoxGFZjMKLGYFhNwaiaQhEF9tzCUQVCETWHIgqEovFl\n+z0ORRUIx15v2n+d+ONgJKpgOKpQ/D4cbdu88RgpxetRisfEbi099hp5vR75vEZ+j5HP45E3vvzA\n9fbf1iglvp7Pa+T1xLb3mNjrHo+R12PkNfHHRvLuWW723e+/zOtRbPv4+xywvccoLcWr4fce3p/f\njXG2VFKqpH468LyzPeeaLTvmEwEAAAAJEI5EVRcIqzYQOuC+PhBWYzCshmBEjc3x+2BYDc0H3R/0\neihy+LEUCwmP0nxepfm8SvV5lJbiVZovtiw33Rd7nBJ7ze/1yOf1yJ8Su/m8HqWm7Fu29zWvR/4U\nI7/XGwup/db1H7T+nu29nsM/2pRM3Bhnb0oKSbpW0v6teZ2kJdba9Y5MBQAAABzEWqv65rAqGqNa\ntKVauxtDqm4MqroxpLq9wbUvuvYua4rdN4Uirfo5mX6vMlJTYvf+FGWmepWX4Vf3/PjzFl5P96co\n3bcvsvYPrlSfJ/6aV74jODcKiZHQODPGXBZ/ODp+f54xpkJShbX2/fg6YUlPWmtvlSRrbbkx5iFJ\nPzbG1EmaL+lKSadLuiiR8wIAAKBja2gOq7K+OX4LandDcG9w7W7c//G+CNv70b4PPvrc+6WmeJSd\n5lNOWoqy01KUk+5T19w0Zaf6lJ2WEnstPXafvWedNJ+yUlOUmRqLrLQUrzwd/IhSR5HoI2fPH/T8\n4fj9+5Imxh9747f9/VRSvaRvSyqWtFLSFdba1xIzJgAAAJJVUzCi8rqAKuubVVEXVEV9syrrmuPP\n94VYZX2zGoMtH8nyez3Ky/ApP8OvvAyf+hdmKT/Tp7wMv/IzfNq5aZ0mHD9cBfFluemx2EpNOfif\nucChJTTOrLVfmvgtrWOtjUi6P34DAAAAPsdaq9pAWDtrA9peE9COmqb4fez5nuU1TaEWty/I9Ktz\nll+ds1I1qiRPnbNS4ze/CrNjj/cEWYbfK2MO/U/bsrLNmji4S6L+qOgg3HjOGQAAACBrrSrqm7W5\nqklbdjdqy+4mba6K3W+radKOmsDnjnQZI3XOSlVxTpp6FmRoXJ8CdclJU5ectL0hVpSdqoJMv1I4\n1wouQ5wBAADAMYFQRBt3NWp9Zb027mrU5t2NB8RYczh6wPqds/zqkZ+hQcU5mlRapOKcNBXnpqlr\nbuy+KDtN/hSiC+0TcQYAAICECkei2lrdpHWVDVpf0aD1lftu22qatP832+am+9QjP10DirJ1+sAi\n9SzIUI/8dPXMz1CP/Ayl+zmHC8mLOAMAAECbCIajWl/ZoFU76/be1pTXa1NV4wHfv5WdlqK+nTM1\ntne++nTuqT6FmerTKVMlnTKUm+5z8E8AOIs4AwAAwGGJRq027GrQyh11WrWzfm+Ira9s2HtZeY+R\nenfO1ICiLJ09pFh9Omeqb+dM9e6cqU6Z/i+8uAbQURFnAAAAOKRQJKrVO+u1ZFuNlm2r1dL4fUP8\nQhzGSCUFGRpQlK2zBndRaXG2BhRlq29hptJ8fAQROBzEGQAAACTFQmzF9jp9tqVaS7fWaMm2Gq3a\nUa9gJHZRjgy/V4O75uiy0T00pFuuBnaNhRjngQFtgzgDAADooHbUBLRg024t2FytBZt2a9GWmr1X\nR8zP8GlIt1zdfFJvDemeqyHdctS7U6a8Hj6OCCQKcQYAANABBMNRLd5ao3kbq7RgU7U+21yt7TUB\nSZI/xaOh3XJ03YReGlWSp5E989Q9L53zwoBjjDgDAABIQoFQRJ9trtacdVWau2GX5m3crUAodlSs\nJP7lzKN65mlUSb4Gdc3hu8EAFyDOAAAAkkAgFNG8jbs1a+0uzV1fpc82VysYicoYaVBxjq4aW6IJ\nfQs0uleBCrNTnR4XQAuIMwAAgHbIWqsVO+r04epKfbC6QnPXV6k5HJXXYzS0e65uOqm3xvcp0Jhe\nBcrN4LvDgPaAOAMAAGgnyusC+nB1pWaurtSHaypVUdcsSepflKWrx5Xo1OM6a1yfTspK5Z94QHvE\n/3IBAABcylqrZdtr9e7ycr27fKcWbqmRJBVk+nVy/846eUBnnTKgs7rmpjs8KYC2QJwBAAC4SCAU\n0ay1u/TO8p3694pyba8JyBhpRI88/cdZx2nSwCIN7pojD5e0B5IOcQYAAOCwukBI/15RrjcWb9cH\nqyrVFIoo3efVKQM667tnxoKMi3gAyY84AwAAcEBdIKR3lu/U64t26IPVFQqGoyrKTtWlo7vrjEFd\ndELfTkrzeZ0eE8AxRJwBAAAcI7WBkN5ZtnPvEbJgJKrinDRdM65E5w/vqtEl+XxcEejAiDMAAIAE\nCkWien9lhV5asFVvL9+pYDiqrrlpum5CL50/vFijehJkAGKIMwAAgDZmrdXirTV6cf5Wvbpwm3Y1\nBFWQ6dc140p04YhuGtUzjyAD8DnEGQAAQBvZVt2klxZs1Yvzt2htRYP8Xo/OHFykr43qodNKC+Xz\nepweEYCLEWcAAABHIRSJ6t8ryjVj7ia9v6pC1kpje+fr1pP76vxhXZWb4XN6RADtBHEGAABwBDZX\nNepvn2zWc59uVnlds7rkpOpbk/rrstE9VdIpw+nxALRDxBkAAEArRaJW83aGNW3aXM1cXSEjaWJp\nka4eV6JJpYVK4WOLAI4CcQYAAPAlahpD+tunm/Tkxxu1tbpZXXPrdPfpA3TF2J7qnpfu9HgAkgRx\nBgAAcAgrd9TpiY836KUFWxQIRTW+T4Eu6R3Vdy6fxFEyAG2OOAMAANiPtVZlKys0deY6fbx2l1JT\nPPrqyO668cTeGtwtR2VlZYQZgIQgzgAAABS76uKrC7fpkffXaeXOOnXNTdM955bqqrElKsj0Oz0e\ngA6AOAMAAB1aQ3NYf/1ksx6buU7bagIq7ZKth64YoQtHdON7yQAcU8QZAADokHY3BDXto/V6atZG\n1TSFNK5Pgf7rkmGaWFooY4zT4wHogIgzAADQoVQ1BDV15jo99fEGNYYiOntwF91xWj8dX5Lv9GgA\nOjjiDAAAdAi76ps1deZ6PTVrg5pCEV0wvJvuPr2/BnTJdno0AJBEnAEAgCS3q75ZU2au0/RZG9UU\niujC4d30LaIMgAsRZwAAICnVBUKaOnO9Hp25ToFQRBeN6Ka7Th+g/kVZTo8GAC0izgAAQFJpDkf0\n9OxN+st7a1TVENT5w7vqu2ceR5QBcD3iDAAAJIVI1OqlBVv1+7dXaWt1k04Z0Fk/OKdUw3vkOT0a\nALQKcQYAANo1a63eW1muB/65Qqt21mt4j1z9+rLhOql/Z6dHA4DDQpwBAIB2a9XOOt332jLNXF2p\nPp0z9fC1x+u8ocV8TxmAdok4AwAA7U5VQ1C/f3uVnpmzUVmpKfrPCwbr+hN6yef1OD0aABwx4gwA\nALQbwXBUT83aoD++u1qNwYiun9BL3znzOOVn+p0eDQCOGnEGAADahbKV5br31WVaX9mgU48r1M/P\nH8R3lQFIKsQZAABwtW3VTfrVq8v05tId6ts5U4/fPFaTSoucHgsA2hxxBgAAXCkUiWrah+v1x3dX\nK2qtfnBOqW47pY9SU7xOjwYACUGcAQAA15m9bpd+/vISrS6v15mDuugXFw5Wz4IMp8cCgIQizgAA\ngGvsqm/W/a8v10sLtqpHfroevWGMzhzcxemxAOCYIM4AAIDjrLV6ZeE23fvqMtUFQvrmpH66a9IA\npfv5CCOAjoM4AwAAjtpW3aSfvbxE/15RrhE98/TrS4ertJirMALoeIgzAADgiGjU6pm5m/TgP1co\nErX6+QWDddOJveX1GKdHAwBHEGcAAOCYW1tRrx//fbHmbqjSyf0767+/NowLfgDo8IgzAABwzESj\nVk98vEEPvrlCqSke/fqy4bp8dA8Zw9EyACDOAADAMbFld6N+8PwizVq3S5NKC/XgpcNVlJPm9FgA\n4BrEGQAASChrrZ6ft0W/enWZrLV64GvDdOXYnhwtA4CDEGcAACBhKuqa9eMXF+ud5Ts1rk+Bfnf5\nCM4tA4BDIM4AAEBCvLV0h3704mLVN4f1s/MH6ZaT+sjDlRgB4JCIMwAA0KYCoYjue22ZnpmzSUO7\n5+j3V4zUgC58bxkAfBlPIt/cGNPTGPOCMabGGFNrjHnRGFPSym3tIW4jEzkzAAA4cit21OqiP3+o\nZ+Zs0uRT++rFb5xEmAFAKyXsyJkxJkPSvyU1S7pRkpV0v6T3jDHDrbUNrXibJyQ9ctCyVW05JwAA\nOHrWWk2fvVH3v75cOWk+PXXLOJ16XKHTYwFAu5LIjzXeLqmvpFJr7RpJMsYskrRa0h2SHmrFe2y1\n1s5O3IgAAOBoVTUEdc8LC/XO8nJNKi3Uby4foc5ZqU6PBQDtTiLj7CJJs/eEmSRZa9cbYz6SdLFa\nF2cAAMDF5qzbpbv/ukC7G0L6zwsG6+aTenOJfAA4Qok852yIpCUtLF8qaXAr3+MbxphmY0yjMebf\nxphT2m48AABwpKJRq4fL1ujqqbOV4U/RS988Ubec3IcwA4CjYKy1iXljY4KSHrLW/uig5fdL+pG1\n9guP2hljpkt6TdI2Sb0k/UCxqDvLWlvWwvqTJU2WpMLCwtHPPfdcW/wxkETq6+uVlZXl9BhwIfYN\ntIT94tDqg1ZTFzdrYUVE44q9unloqtJTOk6UsW+gJewXaMmkSZPmWWvHtHZ918ZZC++XrdiRuM3W\n2pO/aN3S0lK7cuXKwx0ZSa6srEwTJ050egy4EPsGWsJ+0bLPNlfrm8/MV3ldQD+/YLCun9Crwx0t\nY99AS9gv0BJjzGHFWSLPOdstKb+F5QXx1w6LtbbOGPO6pFuPdjAAAHB4rLV6atZG3f/6MhVlp+n5\nr5+okT3znB4LAJJKIuNsqWLnnR1ssKRlR/G+iTnUBwAAWtTQHNYP/75Iry3arjMGFul3V4xQXobf\n6bEAIOkkMs5ekfRbY0xfa+06STLG9JZ0kqQffcF2LTLG5Ei6QNLcNpwRAAB8gQ2VDbpj+jytLq/T\nPeeW6uun9pPH07E+xggAx0oi42yqpLsk/cMY8zPFjnjdJ2mz9vtiaWNML0lrJf3KWvur+LLvSyqV\n9J72XRDk+5KKJV2bwJkBAEDceyvL9e0ZC+TxGD15yzidMoAvlQaAREpYnFlrG4wxp0v6vaTpkoyk\ndyV9x1pbv9+qRpJXB17Wf6WkS+K3XEm1kj6SdKu1liNnAAAkkLVWf3lvjX739ioNKs7RI9ePVs+C\nDKfHAoCkl8gjZ7LWbpJ06Zess0GxQNt/2auSXk3cZAAAoCX1zWF9/7mFenPpDl08spse+Npwpfu9\nTo8FAB1CQuMMAAC0H+sq6jV5+jytr2zQz84fpFv5UmkAOKaIMwAAoA9XV+rOZ+YpxevR9FvH6cR+\nnZ0eCQA6HOIMAIAObvrsjfrlK0vVvzBLj944hvPLAMAhxBkAAB1UOBLVr15bpqdmbdQZA4v0x6tH\nKSuVfxoAgFP4GxgAgA6opimku56dr5mrKzX51L764bkD5eX7ywDAUcQZAAAdzPrKBt365CfaXNWo\nX186XFeM7en0SAAAEWcAAHQoH6+t1Deeni+PkZ6+dbzG9+3k9EgAgDjiDACADmLG3E36+ctL1Kdz\nph67caxKOnHhDwBwE+IMAIAkZ63Vb99aqb+8t1anHVeoP18zStlpPqfHAgAchDgDACCJNYcjuueF\nRfrHZ9t09bgS3XfxEKV4PU6PBQBoAXEGAECSqmkMafL0TzVnfZXuObdU3zitn4zhiowA4FbEGQAA\nSWjL7kbd9Pgn2rSrUX+8aqQuHtnd6ZEAAF+COAMAIMks3lKjW578RM2hiJ66dZwmcEVGAGgXiDMA\nAJLIeyvK9c1n5ys/w68Zt49X/6Jsp0cCALQScQYAQJKYMXeTfvbyEg3qmq1pN41VUXaa0yMBAA4D\ncQYAQDtnrdX/vLtGv39nlSaVFurP1xyvzFR+xQNAe8Pf3AAAtGORqNW9ry7VU7M26tLje+iBS4fJ\nx6XyAaBdIs4AAGinmsMRfe9vC/X64u2647S++tG5A7lUPgC0Y8QZAADtUF0gpDumz9PHa3fpZ+cP\n0m2n9HV6JADAUSLOAABoZyrqmnXzE3O1Ynudfn/lCF0yqofTIwEA2gBxBgBAO7JpV6NumDZHO2ub\nNfXGMZpUWuT0SACANkKcAQDQTizbVqsbH5+rUCSqZ24fr+NL8p0eCQDQhogzAADagU83VOnmJz5R\ndmqKZtx+Al8uDQBJiDgDAMDlPlhVoTumz1PX3DRNv228uuelOz0SACABiDMAAFzszSXbdfeMz9Sv\nKEtP3TJOhdmpTo8EAEgQvqUSAACXemHeFt35zHwN7Z6jv94+gTADgCTHkTMAAFzoyY836BevLNXJ\n/TvrketHKzOVX9kAkOz4mx4AABex1urhsrX6zb9W6qzBXfSnq0cpzed1eiwAwDFAnAEA4BLWWj3w\n5go98v46XTKqu3592XD5vJyBAAAdBXEGAIALRKJWP//HEj07Z5Oum1CiX100VB6PcXosAMAxRJwB\nAOCwUCSq7z+/UP/4bJu+MbGf7jmnVMYQZgDQ0RBnAAA4KBSJ6u4ZC/TPJTt0z7mlunNif6dHAgA4\nhDgDAMAhwXBUdz07X28t26mfXzBYt57cx+mRAAAOIs4AAHBAMBzVnc/M1zvLd+qXFw7WTScRZgDQ\n0RFnAAAcY83hiO58er7eXVGuX108RDec0NvpkQAALkCcAQBwDAVCEX3j6Xl6b2WF7v/qUF03oZfT\nIwEAXII4AwDgGAmEIrpj+jy9v6pC/++SYbpmfInTIwEAXIQ4AwDgGAiEIrr9qU/14ZpKPXjpMF05\nljADAByIOAMAIMGagrEw+2htpR68dLiuGNPT6ZEAAC5EnAEAkECNwbBue/JTzVq3S7+5bIQuG93D\n6ZEAAC5FnAEAkCCNwbBueeITzV1fpYeuGKFLRhFmAIBDI84AAEiAhuawbn78E326sUq/v3KkLh7Z\n3emRAAAuR5wBANDG6pvDumnaXC3YXK0/XjVKF47o5vRIAIB2gDgDAKAN1QVCuunxT/TZ5mr9z1Wj\ndP7wrk6PBABoJ4gzAADaSG0gpBunzdXiLTX689WjdN4wwgwA0HrEGQAAbaCmKaQbps3V0q01+vM1\nx+vcocVOjwQAaGeIMwAAjlJNY0jXT5uj5dtr9b/XjdZZg7s4PRIAoB0izgAAOArVjUFd99gcrdpR\nr/+7brTOGESYAQCODHEGAMAR2t0Q1LWPztGa8no9cv1oTRpY5PRIAIB2jDgDAOAIVMXDbG1Fvabc\nMFoTSwkzAMDRIc4AADhMu+qbde2jc7S+skGP3jBGpx5X6PRIAIAkQJwBAHAYKuubde3UOdpY1aDH\nbhyrkwd0dnokAECSIM4AAGilirpmXTN1tjbvbtS0G8fqxP6EGQCg7RBnAAC0QnltQFdPna1t1QE9\ncfM4TejbyemRAABJxpPINzfG9DTGvGCMqTHG1BpjXjTGlLRy2zRjzG+MMduNMU3GmFnGmFMTOS8A\nAC3ZWRvQVVNma3tNQE/cPJYwAwAkRMLizBiTIenfkgZKulHS9ZIGSHrPGJPZird4TNLtkv5T0gWS\ntkv6lzFmZGImBgDg83bUxMJsZ21AT94yTuMJMwBAgiTyY423S+orqdRau0aSjDGLJK2WdIekhw61\noTFmhKRrJN1irX08vux9SUsl/UrSRQmcGwAASdL2miZdPWW2KuuDeurWcRrdq8DpkQAASSyRH2u8\nSNLsPWEmSdba9ZI+knRxK7YNSfrbftuGJf1V0jnGmNS2HxcAgH12NUV15SOztYswAwAcI4mMsyGS\nlrSwfKmkwa3Ydr21trGFbf2S+h/9eAAAtGzL7kY9MDeg3Y1BTb9tvI4vyXd6JABAB5DIjzUWSNrd\nwvIqSV/2W+6Ltt3z+gGMMZMlTZakwsJClZWVtXpQdAz19fXsF2gR+wb2V9EY1QNzA2oMRXXPuFRV\nr/1MZWudngpuwt8ZaAn7BdpC0lxK31o7RdIUSSotLbUTJ050diC4TllZmdgv0BL2DeyxaVejfjp1\ntsImRT8c59VNF5/h9EhwIf7OQEvYL9AWEvmxxt1q+QjZoY6KtXZbad8RNAAA2sTGXQ26asosNQTD\neua28eqd63V6JABAB5PIOFuq2LljBxssaVkrtu0Tvxz/wdsGJa35/CYAAByZ9ZUNuvKR2WoKRfTs\nbRM0tHuu0yMBADqgRMbZK5ImGGP67llgjOkt6aT4a1/kVUk+SZfvt22KpCslvWWtbW7rYQEAHdPa\ninpdNWWWgpGonr19ggZ3y3F6JABAB5XIOJsqaYOkfxhjLjbGXCTpH5I2S3pkz0rGmF7GmLAx5j/3\nLLPWLlDsMvp/MMbcZow5Q7HL6PeR9IsEzgwA6EDWlNfr6imzFY5Yzbh9ggZ1JcwAAM5JWJxZaxsk\nnS5plaTpkp6RtF7S6dba+v1WNZK8Lcxys6THJd0v6XVJPSWda62dn6iZAQAdx5ryOl01Zbai1mrG\n5AkqLc52eiQAQAeX0Ks1Wms3Sbr0S9bZoFigHby8SdL34jcAANrMqp11umbqbBljNOP2CepfRJgB\nAJyXyI81AgDgOit21OrqKbPlMUZ/nUyYAQDcgzgDAHQYy7bV6pqpc5TijYVZv8Isp0cCAGCvpPkS\nagAAvsjSbTW67tE5SvN5NeP2CerdOdPpkQAAOABHzgAASW/J1hpd++gcpfu8+utkwgwA4E7EGQAg\nqS3cXK1rps5Wpj9Ff518gnp1IswAAO5EnAEAktb8Tbt13aNzlJvh018nT1BJpwynRwIA4JA45wwA\nkJQ+3VClmx7/RJ2y/Jpx+wR1y0t3eiQAAL4QR84AAElnzrpdumHaXBVlp+pvk08gzAAA7QJHzgAA\nSeXjNZW69clP1T0/Xc/eNl5FOWlOjwQAQKtw5AwAkDRmrq7QzU98opKCDM24fQJhBgBoVzhyBgBI\nCu+tLNcd0+epb+dMPXPbeHXQ4YzWAAAfa0lEQVTKSnV6JAAADgtxBgBo995ZtlN3PjNfA7pk6elb\nxys/0+/0SAAAHDY+1ggAaNfeXLJD33hmngZ1zdazt00gzAAA7RZHzgAA7dYbi7fr7hkLNKxHrp68\nZZxy0nxOjwQAwBHjyBkAoF16ZeE2fWvGAo3smaenCDMAQBLgyBkAoN15acEW/cdzCzWmd4Eev2ms\nMlP5dQYAaP84cgYAaFeenbNJ33tuoSb07aQnbibMAADJg99oAIB249GZ63T/68t1+sAiPXzt8Urz\neZ0eCQCANkOcAQBcz1qrP/17jR56e5W+MqxYf7hylPwpfPgDAJBciDMAgKtZa/Xgmyv1f++v1deO\n765fXzpcKV7CDACQfIgzAIBrRaNW9766VE/O2qhrx5fovouHyuMxTo8FAEBCEGcAAFeKRK1+9PdF\nen7eFt1+Sh/95CuDZAxhBgBIXsQZAMB1QpGovvu3z/Taou369hkD9J0zBxBmAICkR5wBAFwlEIro\nrmfn653l5frxeQN1x2n9nB4JAIBjgjgDALhGQ3NYd0yfpw/XVOq+i4fo+hN6Oz0SAADHDHEGAHCF\n3Q1B3fzEJ1q0pVq/uWy4Lh/T0+mRAAA4pogzAIDjdtQEdP1jc7SxqlH/e91onTOk2OmRAAA45ogz\nAICj1lc26LpH56imKaQnbh6rE/t1dnokAAAcQZwBAByzZGuNbpw2V1bSjNsnaFiPXKdHAgDAMcQZ\nAMARc9bt0m1PfqqcdJ+eunWc+hVmOT0SAACOIs4AAMfc28t26q5n56tnQYaeumWcuuWlOz0SAACO\nI84AAMfU3+dt0T1/X6Sh3XL0+M3jVJDpd3okAABcgTgDABwzj85cp/tfX66T+nfSI9ePUVYqv4YA\nANiD34oAgISz1urX/1qp/y1bq/OGFusPV41UaorX6bEAAHAV4gwAkFDBcFQ//PsivbRgq64ZX6L7\nLh4qr8c4PRYAAK5DnAEAEqYuENI3np6vD9dU6gfnlOrOif1kDGEGAEBLiDMAQELsrA3oxmlztaa8\nXr+9fIQuG93D6ZEAAHA14gwA0OZW76zTTY9/ourGoKbdNFanHlfo9EgAALgecQYAaFNz11fptic/\nUarPq7/dcYKGds91eiQAANoF4gwA0GbeWLxd3/nbZ+qRn64nbx6nngUZTo8EAEC7QZwBAI6atVaP\nfbhe//XGch1fkq9HbxijfL5cGgCAw0KcAQCOSigS1S9eWapn52zSeUOL9fsrRyrNx3eYAQBwuIgz\nAMARq2kK6a5n52vm6kp9Y2I//eDsUnn4DjMAAI4IcQYAOCKbdjXqlic/0YbKBv36suG6YkxPp0cC\nAKBdI84AAIft0w1Vmjx9niJRq+m3jtcJ/To5PRIAAO0ecQYAOCwvL9iqe15YpO756XrsxjHqW5jl\n9EgAACQF4gwA0CrWWv3+ndX6n3dXa3yfAv3fdaO5IiMAAG2IOAMAfKnGYFj3vLBIry3arstG99D/\nu2SY/Ckep8cCACCpEGcAgC+0ZXejJj81T8t31OqH5w7U10/rK2O4IiMAAG2NOAMAHNLsdbt05zPz\nFYpENe3GsZo0sMjpkQAASFrEGQDgc6y1enrOJt37ylKVdMrQ1BvGqB8X/gAAIKGIMwDAAYLhqH7x\nylLNmLtJpw8s0h+uGqmcNJ/TYwEAkPSIMwDAXuV1Ad359Hx9unG37pzYT/9xdqm8Hs4vAwDgWCDO\nAACSpLnrq/TNZ+erLhDSn64epQtHdHN6JAAAOpSEXQfZGOMxxvzYGLPBGBMwxiw0xlzaym2fMMbY\nFm5/SNS8ANBRWWv16Mx1unrqbGX6vXr5mycRZgAAOCCRR87uk/R9ST+VNE/SVZKeN8ZcYK19oxXb\nV0i66KBl29t2RADo2Oqbw/rhC4v0+uLtOntwF/32ihGcXwYAgEMSEmfGmCLFwuwBa+1v44vfM8b0\nl/SApNbEWdBaOzsR8wEApNU763TH0/O0obJBPz5voCafyveXAQDgpEQdOTtHkl/S0wctf1rSNGNM\nH2vt+gT9bADAl3hl4Tb96O+LlOH36pnbJuiEfp2cHgkAgA4vUeecDZHULGnNQcuXxu8Ht+I9iowx\nlcaYsDFmlTHmh8YYb5tOCQAdTCAU0U9eWqy7ZyzQ4K45ev3uUwgzAABcwlhr2/5NjZki6SJrbfFB\ny/tLWi3pBmvt9C/Y/juSIorFXJqkSyTdKmmatfa2Q2wzWdJkSSosLBz93HPPtcUfBUmkvr5eWVl8\niS4+r6PsG9vqo3r4s4C21Ft9pY9PXxvgUwqXyT+kjrJf4PCxb6Al7BdoyaRJk+ZZa8e0dv1WfazR\nGHOmpLdbser71tqJrf3hh2KtPfiqjG8YY+olfccY86C1dnUL20yRNEWSSktL7cSJRz0GkkxZWZnY\nL9CSZN83rLV6ft4W3ffuUmX4fXri5hGaWFrk9Fiul+z7BY4c+wZawn6BttDac84+ljSoFes1xu93\nS8ozxhh74KG5gvh9VSt/7v5mSPqOpDGKHX0DAHyJ+uawfvbSYr382Tad0LeT/nDVSHXJSXN6LAAA\n0IJWxZm1tlHSisN436WSUiX104Hnne0512zZYbzX58Y5im0BoMNYsrVG35qxQBt3Neh7Zx2nb07q\nLy8fYwQAwLUSdUGQNyWFJF170PLrJC05wis1XqtYmH1ylLMBQFKLRK3+t2ytLnn4IzUFI5px+wTd\nfcYAwgwAAJdLyKX0rbXlxpiHJP3YGFMnab6kKyWdroO+WNoY866kXtba/vHnvSRNl/RXxY66pSp2\nQZCbJD1irV2biJkBIBlsrmrUfzy3UHM3VOkrw4r1X18dpvxMv9NjAQCAVkjU95xJ0k8l1Uv6tqRi\nSSslXWGtfe2g9bwHzVGn2DlpP5TURVJUsY9U3i3p4QTOCwDtlrVWL87fql+8EvvGkt9dPkJfO747\nXyoNAEA7krA4s9ZGJN0fv33RehMPel4l6auJmgsAkk11Y1A/fWmJXl+8XWN75+uhK0aqZ0GG02MB\nAIDDlMgjZwCABCtbWa4f/n2RqhqCuufcUt1xaj/OLQMAoJ0izgCgHappCun+15bp+XlbNKAoS4/d\nOFZDu+c6PRYAADgKxBkAtDPvLt+pn7y0WJX1QX1zUj/dfcYApaZ4nR4LAAAcJeIMANqJ6sagfvXq\nMr24YKtKu2Tr0RvGalgPjpYBAJAsiDMAaAfeWrpDP315iXY3BHX3GQN016T+8qck6qsqAQCAE4gz\nAHCxHTUB3fvqUv1zyQ4N6pqjx2/i3DIAAJIVcQYALhSJWk2ftUG/fWuVQpGofnBOqW4/pS9HywAA\nSGLEGQC4zJKtNfrJS4u1aEuNThnQWfd/dah6dcp0eiwAAJBgxBkAuER9c1gPvbVKT3y8XgWZqfqf\nq0fpwuFdZQzfWwYAQEdAnAGAw6y1enXRdv2/15drR21A144v0T3nDlRuus/p0QAAwDFEnAGAg5Zu\nq9G9ryzT3A1VGto9R3+59niN7pXv9FgAAMABxBkAOGB3Q1C/e3ulnp2zSXkZfv3314bpijE95fXw\nEUYAADoq4gwAjqFwJKoZczfpt2+tUn1zWDec0FvfPfM45WbwEUYAADo64gwAjgFrrcpWVui//7lc\nq3bW68R+nfSLC4eotDjb6dEAAIBLEGcAkGCLtlTrv99YoVnrdql3pwz933XH65whxVyFEQAAHIA4\nA4AE2VzVqN/8a6VeWbhNBZl+3XvREF0zvkQ+L18kDQAAPo84A4A2VtUQ1MPvrdFTszbK45HumtRf\nd5zWV9lpnFcGAAAOjTgDgDZS0xTSozPXadqH69UUiujy0T313bOOU3FumtOjAQCAdoA4A4CjVBcI\n6fGPNmjqzHWqC4R1/rCu+s6ZAzSgCxf7AAAArUecAcARagyG9eTHG/XIB2tV3RjSWYO76LtnHqfB\n3XKcHg0AALRDxBkAHKbaQEhPz96oaR+uV2V9UBNLC/W9s47T8B55To8GAADaMeIMAFppV32zHv9o\ng56ctUF1gbBOPa5Q3z6jv0b3KnB6NAAAkASIMwD4EtuqmzR15jrNmLtJzeGozhtarDsn9tfQ7rlO\njwYAAJIIcQYAh7BiR60em7leL3+2VdZKXx3VXV8/rZ/6F2U5PRoAAEhCxBkA7CcatXp/dYUem7le\nH66pVJrPo6vHlWjyqX3VIz/D6fEAAEASI84AQFJTMKIXF2zRtA/Xa21Fg7rkpOqec0t1zbgS5WX4\nnR4PAAB0AMQZgA5tc1Wjnl8Z1Hc/eFe7G0Ma2j1Hf7hypL4yrKv8KR6nxwMAAB0IcQagw4lErcpW\nluvp2RtVtqpCstLZQ7rolpP6aFyfAhljnB4RAAB0QMQZgA6joq5Zz326Wc/O2aSt1U0qyk7Vt04f\noN6RLfrauWOcHg8AAHRwxBmApBaORFW2skLPz9usf68oVyhidVL/TvrZ+YN05uAu8nk9Kivb5vSY\nAAAAxBmA5LSmvE7Pf7pFf5+/VZX1zeqc5deNJ/TW1eNL1K+QS+EDAAD3Ic4AJI3dDUG9vni7Xpi3\nRZ9trlaKx2jSwCJdPrqHJg0sks/LBT4AAIB7EWcA2rWG5rDeXrZT//hsq2aurlQ4anVclyz97PxB\n+uqo7uqcler0iAAAAK1CnAFod5rDEZWtrNArC7fp3eU7FQhF1S03Tbee3EcXjuimId1yuOIiAABo\nd4gzAO1CIBTRh6sr9ebSHfrX0h2qC4TVKdOvy0f31MUju+n4knx5PAQZAABov4gzAK5V0xTSeyvK\n9dayHSpbWaHGYETZaSk6e3CxLhrZTSf166QUziMDAABJgjgD4Co7awN6a9lOvbV0h2at3aVw1Koo\nO1WXjOquc4YUa0LfTvKnEGQAACD5EGcAHBWORDV/U7XKVparbGWFlm2vlST16ZypW0/po3OGFGtk\njzw+sggAAJIecQbgmCuvDahsVYXeX1mhmasrVBsIy+sxGl2Srx+cU6qzBnfRgKIsLuoBAAA6FOIM\nQMLVN4f1yfoqfby2Uh+t2bX36FhRdqrOHVqsiaVFOql/Z+Wm+xyeFAAAwDnEGYA2FwhFNG/jbn28\ntlKz1u7Swi01ikSt/F6PRpXk6QfnlGpSaZEGdc3m6BgAAEAccQbgqNUGQlqwqVqfbqjSJxuqNH9j\ntYKRqLweo+E9cvX10/rqxH6dNbpXvtJ8XqfHBQAAcCXiDMBh21rdpE83VOnTDbv1yYYqrdxZJ2sl\nj5EGd8vRjSf20on9OmtsnwJlpfLXDAAAQGvwryYAX6g2ENLiLTVauKVaizbH7rfXBCRJmX6vju+V\nr3OHFmtMrwKNLMkjxgAAAI4Q/4oCsFcgFNHSbTVauLlGi7ZUa9GWGq2rbNj7eq9OGRrTu0BjeuVr\ndK98DSzO5kugAQAA2ghxBnRQFXXNWr69Vsu312rFjjot316r1eX1ikStpNiVFIf3yNMlo7prRM88\nDe+Rq7wMv8NTAwAAJC/iDEhygVBE6yoatGLHvghbvr1WlfXBvesU56RpUNdsnTGoSMN75GlEjzwV\n56Y5ODUAAEDHQ5wBSaKmMaQ1FfVaW16vNRX1WlMeu23e3SgbOxgmf4pHx3XJ0sTSIg3qmqNBXbM1\nqDhH+ZkcEQMAAHAacQa0I4FQRFt2N2rjrthtXeWeCGtQZX3z3vX8KR717ZypYT1y9dVR3dW/KEsD\ni7PVt3Mm54gBAAC4FHEGuExNU0ibdjVqY1VDPMJi95uqGrWjNrD3KJgkZaelqH9RliaVFqp/UZb6\nF2WpX2GWehZkyOvhy50BAADaE+IMOIbCkajK65q1rbpJW6ubtK06oG3VTfs9b1JtIHzANp2zUtWr\nU4ZO6NtJJZ0y1KtThkoKMtWrU4Y6ZfplDBEGAACQDIgzoI00BsMqr21WeV2zdtYGVF7XrPK6gHbU\n7AmwgHbUBvZeDXGP3HSfuuWlq0d+usb1KVD3vHT16pQZj7AMZfK9YQAAAB0C/+oDvkBzOKKqhqB2\n1QdV1RC7VcSja/8Iq6htVl1z+HPb+70eFeWkqnteusb3KVC3vPT4LU3d89LVNS+dL20GAACAJOIM\nHUgkarWrvlk1TSHVNIVU3RRSVTy6djUEVdXQvPfxnhirbyG4JCnN51FRdpqKslM1sDhbpw4oVFFO\nqoqy09Qlfl+Unaq8DB8fOwQAAECrJCzOjDHfkzRJ0hhJxZLutdb+8jC2P1nSryWNklQj6VlJP7XW\nNrX9tGgPIlGrhmBY9YGw6pvjt0B4b2ztvTXuia+gaprCqo0vr28OS2+90+J7+7xGBZl+FWSmqlOm\nXyUlGSrI9KtTfFlBpl+dsvwqyPSrc1aqctJSiC4AAAC0qUQeObtdUq2klyV9/XA2NMYMl/S2pH9J\nukBSH0m/kdRd0pVtOyYSwVqr5nBUTcGImkIRNQYjCoT2PW4KRtQYDKuhOay65s8H197H+z1vDEa+\n9Of6UzzKS/cpN92nvAyfuufFvlw5L92v3Tu3aMSgAcrN8Ckv3a+cdF8svrL8yk4ltgAAAOCsRMbZ\nEGtt1BiTosOMM0n3Stoi6XJrbUiSjDFBSU8aYx601s5v41mTWjRqFYxE1RyOqjkcUTAcfxyKxpaH\nIvH72PJgJLLv8X7bBMJRNQbDagpG1RQK7w2vzwVY/PlB1734Qikeo6y0FGWl7rsVZPrVsyBD2fHn\nmakpyk7b93jP+rnxGMtN9ynN5z3kzygrK9fEk/q0wX+iAAAAQNtLWJxZa6NHsp0xxifpXEkP7Qmz\nuOckTZV0saSExJm1VpGoVThqFY0/jkalcDSqiN33OBqVItYqEo0qEo193C4StfFl+25RG3uvUDiq\ncDSqUMTG7sNWoWhU4YhVKBJfHokqFI3fR/atG47Ewiq8Z9v4Nnu2DUcPeo9IdF98xeMqGDmi/yoO\nYIyUluJVht+rNF/sPj3+OD/Tr26+2PN0X+yW4fcqze9Vhm/fehn+lNjr/n3r7Ams1BQPR64AAADQ\nobnxgiD9JKVJWrL/QmttwBizVtLgL3uDrfVRnfnQ+4rGgykc2RdbXxRR9jCO9CSKz2uU4vEoxWvk\n83qU4ond+7xGKfHn/pTYfYrXozSfR9lpKUrxxNbxeT1KTfHIn+JRaopXqT6P/F6PUn2x57Hl+9+8\nrVo/xWOIJwAAACCB3BhnBfH73S28VrXf64fk8xgd1yVLHmOU4jHyeIy8xijFa+QxRl5P/Lb/Y89B\nrx30uscTey+v2ffY87n3UPxneuTxaO/PjEXWvng6ILxSPPLFY4wAAgAAADquVsWZMeZMxS7Q8WXe\nt9ZOPKqJjpAxZrKkyZJUWFioK7rXHd0b2vjtCD4R2PLF1+G0+vp6lZWVOT0GXIh9Ay1hv8ChsG+g\nJewXaAutPXL2saRBrViv8Shm2WPPEbP8Fl4rkLS0pY2stVMkTZGk0tJSO3HixDYYBcmkrKxM7Bdo\nCfsGWsJ+gUNh30BL2C/QFloVZ9baRkkrEjzLHmslNUsasv9CY0yapL6Snj9GcwAAAADAMeNxeoCD\nWWuDkt6UdEX8Mvx7XCYpVdIrjgwGAAAAAAmUsAuCGGPGSOqtfQE42BhzWfzxG/GjcTLGPCbpRmvt\n/rP8UtJsSc8ZY/4Sf5/fSHrBWjsvUTMDAAAAgFMSebXGuyTduN/zy+M3SeojaUP8sTd+28ta+5kx\n5mxJD0p6XVKNpKck/SSB8wIAAACAYxL5JdQ3SbrpSNez1n4g6YQ2HgsAAAAAXMl155wBAAAAQEdE\nnAEAAACACxBnAAAAAOACxBkAAAAAuABxBgAAAAAuQJwBAAAAgAsQZwAAAADgAsQZAAAAALgAcQYA\nAAAALkCcAQAAAIALEGcAAAAA4ALEGQAAAAC4AHEGAAAAAC5AnAEAAACACxBnAAAAAOACxBkAAAAA\nuABxBgAAAAAuQJwBAAAAgAsQZwAAAADgAsQZAAAAALgAcQYAAAAALkCcAQAAAIALEGcAAAAA4ALE\nGQAAAAC4AHEGAAAAAC5AnAEAAACACxBnAAAAAOACxBkAAAAAuABxBgAAAAAuQJwBAAAAgAsQZwAA\nAADgAsQZAAAAALgAcQYAAAAALkCcAQAAAIALEGcAAAAA4ALEGQAAAAC4AHEGAAAAAC5AnAEAAACA\nCxBnAAAAAOACxBkAAAAAuABxBgAAAAAuQJwBAAAAgAsQZwAAAADgAsQZAAAAALgAcQYAAAAALkCc\nAQAAAIALEGcAAAAA/n979x5iaV3Hcfz9qU2wlMhLEpaulpprEdUGS0WauZp/ZAVbf1QqQZqVYYqU\nhtBamBR5iaUiNTMrumClLpmgmRJ5a0syraytFBeUvItubF6+/fGcyWE6Z3Zmd878npl5v2A4t+c5\n81n2t2ef7/nd1AMWZ5IkSZLUAxZnkiRJktQDFmeSJEmS1AMWZ5IkSZLUAxZnkiRJktQDFmeSJEmS\n1AMWZ5IkSZLUA2MrzpKckmR9kvuSVJK1szh37eCcqT+XjyuvJEmSJLW0bIzvfRzwOHA5cMI2vsdb\ngWcmPX54e0NJkiRJUh+Nszg7qKqeTbKMbS/Obqmqp+cylCRJkiT10diGNVbVs+N6b0mSJElabPq+\nIMi9SZ5Jck+SLyXZsXUgSZIkSRqHcQ5r3B4bgdOA24ACDgdOBt4ArG6YS5IkSZLGYkbFWZLDgGtm\ncOgNVXXIdiUCqup7U566Jskm4Pwkh1XVtUMyHg8cP3i4Jckd25tDi85uwIOtQ6iXbBsaxnahUWwb\nGsZ2oWEOmM3BM+05uxE4cAbHbZ7NL5+lHwDnA28C/q84q6oLgAsAkmyoqpVjzKIFyHahUWwbGsZ2\noVFsGxrGdqFhkmyYzfEzKs6qajPwl21KNPeqdQBJkiRJmmt9XxBksg8Obm9tmkKSJEmSxmBsC4Ik\nWQks57kCcEWSNYP7Vw1640jyLeDYqlo26dzbgEuBu+h6ylYDnwSurqrrZvDrL5iTP4QWG9uFRrFt\naBjbhUaxbWgY24WGmVW7SNV4RgkmuQQ4dsTL+1TV3ZOPq6pMOveHdHPLXkZX3P2Dbs7Zl6tqy1gC\nS5IkSVJDYyvOJEmSJEkzt5DmnM1akp2T/DjJxiRPJnk0ya1JPtQ6m9pJsn+Srya5PckTSe5LcmWS\n17XOpvaSnJJk/aBdVJK1rTNp/iR5RZLLkjyW5PEkP02yV+tcaivJy5OsS3JTks2Dz4blrXOprSRr\nkvwkyT1J/p3kriRnJ9m5dTa1leSIJNcluT/JliSbBjXJiq2du6iLM2AH4GngbOAo4APAn4HvJjm5\nZTA1dTjwduA7wLuAjwO7AzcneWPLYOqF44CXApe3DqL5leSFwHXAq+mG5R8N7Af8KsmLWmZTc68C\n3g88Avy6cRb1x6nAM8BngXcC3wA+Rrc/72K/xtb0dgF+B5xId915OnAQ3bXm3tOduCSHNSa5Cdip\nql7bOovmX5LdgIdqUuNP8mLgbmB9VR3TKpvaS/K8qno2yTLgKeDMqlrbOJbmQZKTgHOBA6pq4+C5\nfYC/AZ+uqnNb5lM7E58Lg/sfAS5k0vx5LU1Jdq+qB6Y8dwzdl7/vmOEidloikhxAtzXZqVV1zqjj\nlmpV/xBdj5qWoKp6sKZ8K1FVjwF/BfZsk0p9MXEBpiXpKODmicIMoKr+CfwGeHezVGrOzwUNM7Uw\nG/jt4NbrCU310OB22hpkSRRn6SxLsmuS44EjgPNa51J/JNkFeA3dsFdJS9NBwB1Dnr8T2Oo8AUkC\nDh7cej0hkjw/yQ5J9gO+CdxPtwL9SGPb56xnPgGsG9x/Cjipqi5tmEf9sw4IcH7rIJKa2YVuTtFU\nDwMvmecskhaYJHsCnweuraoNrfOoF24BJtYz2AgcWlX/mu6EBdVzluSwwQpJW/u5fsqpP6LbN+1I\n4CJgXZKPznd+jcd2tIuJ80+nWyzmxMnDmbTwbW/bkCRpJpLsBFxBN2Ttw43jqD+OBlbRXWc+TrdY\nzPLpTlhoPWc3AgfO4LjNkx8MxgRPjAu+erAi11eSXFxVT81xRs2/bWoXAElOAL4InFFVF891MDW3\nzW1DS9IjDO8hG9WjJkkk2RFYD+wLHFxVmxpHUk9U1cTw1luS/IJu8bnTgBNGnbOgirOq2ky3ysn2\n2kC3TPIegP+AFrhtbRdJjga+DpxTVWfNeTA1N4efGVoa7qSbdzbVCuBP85xF0gKQ5AXAZcBKYHVV\n/bFxJPVUVT2aZCPd1hwjLahhjXPoYOAJYNoxn1q8krwX+DZwUVWd2jqPpF64EliVZN+JJwbDT94y\neE2S/mewl9n3gUOB91TVzY0jqceS7EG3j+bfpztuQfWczdZgXtkq4Fq6HrJd6TaRXAOcVlX/aRhP\njSR5G91KOX8ALkmyatLLW6rqtjbJ1AdJVgLLee7LqxVJ1gzuXzXojdPidCHdhqFXJDkDKOALwL10\nq2xpCZv0OTAxuf/IJA8AD1TVDY1iqa2vAe8DzgKenHI9scnhjUtXkp8Bvwdup5trtj9wMt2cxJF7\nnMEi34Q6yZuBM4DX080ZeJBuadPzqurnLbOpnSRrgc+NePmeqlo+f2nUN0kuoRv2PIybzi5ySfai\n22plNd0Krr8EPuXfu5KMumC6oaoOmc8s6ockdwN7j3j5zKpaO39p1CdJPkPXIfRKYAe6L/muB87e\n2v8ni7o4kyRJkqSFYqnOOZMkSZKkXrE4kyRJkqQesDiTJEmSpB6wOJMkSZKkHrA4kyRJkqQesDiT\nJEmSpB6wOJMkSZKkHrA4kyRJkqQesDiTJEmSpB74L+ilnK9daulxAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f87f1c714d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(lambda x: math.tanh(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### ReLU (_Rectifier Linear Unit_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "$$ g(x) = \\max(0, x) $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### ReLU – zalety\n",
    "* Mniej podatna na problem zanikającego gradientu (_vanishing gradient_) niż funkcje sigmoidalne, dzięki czemu SGD jest szybciej zbieżna.\n",
    "* Prostsze obliczanie gradientu.\n",
    "* Dzięki zerowaniu ujemnych wartości, wygasza neurony, „rozrzedzając” sieć (_sparsity_), co przyspiesza obliczenia."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "#### ReLU – wady\n",
    "* Dla dużych wartości gradient może „eksplodować”.\n",
    "* „Wygaszanie” neuronów."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### ReLU – wykres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2cAAAG9CAYAAACRcQ4FAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAHy9JREFUeJzt3XuUrXdd3/HPNzmQcLFK9BBcSm4E\njyTeKrELq9VD5NpioosAVcBohSPWC8iiFYSlUVFI1UgXlS4SLyC4xEAVCKRYFCZa4RhuShNKQmgS\nkzZZBBMgJwm5cH79Y++RyWTPycyZ/cz+7b1fr7Vmzcyzn2efb+CXybzPs59nV2stAAAAzNYRsx4A\nAAAAcQYAANAFcQYAANABcQYAANABcQYAANABcQYAANCBweKsqr6+ql5bVR+sqturqlXVCZs89prx\n/us/fmCoeQEAAGZp14DPfXKSZyb5SJK/TvKkLR7/50nOWbftiu2PBQAA0J8h4+yvWmvHJklVPS9b\nj7PPttb2T38sAACA/gz2ssbW2sGhnhsAAGDR9HxDkO8fX6t2Z1Xtd70ZAACwyHqNs4uS/EySJyd5\ndpIvJvmzqnrOTKcCAAAYSLXWhv9DRtecXZDkxNbaNYdx/JFJ9id5RGvtkRvssy/JviQ5+uijH3vc\ncccd/sAspIMHD+aII3r9+whmydpgEuuCjSz72rj7YHLDbQdTSR7xkCPygOX9n+Jeln1dMNmVV175\n2dba7s3uP+QNQaamtfalqnprknOr6mtbazdM2Of8JOcnyZ49e9oVV7ixI/e2srKSvXv3znoMOmRt\nMIl1wUaWeW1cceOt+aEL9mf3kZW37PvOnPg1D5n1SN1Y5nXBxqrq2q3sP495P/ypPgAA7mU1zB4g\nzGAwcxFnVbUrybOS/ENr7cZZzwMAsEyEGeyMQV/WWFVnjb987PjzU6vqpiQ3tdYuGe9zT5I3ttZ+\nfPz9DyU5M8nFSa5LcmySn0ry7Ul+aMh5AQC4N2EGO2foa87euu77140/X5Jk7/jrI8cfq65O8vAk\nv5HkmCS3Jflwkqe01v58sEkBALgXYQY7a9A4a63VVvdpre1PcvpgQwEAcL+EGey8ubjmDACAnSPM\nYDbEGQAA/0SYweyIMwAAkggzmDVxBgCAMIMOiDMAgCUnzKAP4gwAYIkJM+iHOAMAWFLCDPoizgAA\nlpAwg/6IMwCAJSPMoE/iDABgiQgz6Jc4AwBYEsIM+ibOAACWgDCD/okzAIAFJ8xgPogzAIAFJsxg\nfogzAIAFJcxgvogzAIAFJMxg/ogzAIAFI8xgPokzAIAFIsxgfokzAIAFIcxgvokzAIAFIMxg/okz\nAIA5J8xgMYgzAIA5JsxgcYgzAIA5JcxgsYgzAIA5JMxg8YgzAIA5I8xgMYkzAIA5IsxgcYkzAIA5\nIcxgsYkzAIA5IMxg8YkzAIDOCTNYDuIMAKBjwgyWhzgDAOiUMIPlIs4AADokzGD5iDMAgM4IM1hO\n4gwAoCPCDJaXOAMA6IQwg+UmzgAAOiDMAHEGADBjwgxIxBkAwEwJM2CVOAMAmBFhBqwlzgAAZkCY\nAeuJMwCAHSbMgEnEGQDADhJmwEbEGQDADhFmwKGIMwCAHSDMgPsjzgAABibMgM0QZwAAAxJmwGaJ\nMwCAgQgzYCvEGQDAAIQZsFXiDABgyoQZcDjEGQDAFAkz4HCJMwCAKRFmwHaIMwCAKRBmwHaJMwCA\nbRJmwDSIMwCAbRBmwLQMFmdV9fVV9dqq+mBV3V5VrapO2OSxR1TVy6rqmqr6YlX9fVU9fahZAQAO\nhzADpmnIM2cnJ3lmkluS/PUWj/3VJOck+S9Jnppkf5K3VtW/nuaAAACHS5gB07ZrwOf+q9basUlS\nVc9L8qTNHFRVD0/ykiSvbq395njz+6vq5CSvTnLxEMMCAGyWMAOGMNiZs9bawcM89MlJHpjkzeu2\nvznJN1fVidsaDABgG66/9aAwAwbR4w1BTk1yZ5Kr1m2/fPz5lJ0dBwBg5Iobb825l94hzIBB9Bhn\nxyT5XGutrdt+85rHAQB21OpLGY88QpgBwxjymrMdVVX7kuxLkt27d2dlZWW2A9GdAwcOWBdMZG0w\niXXBWtffejDnXnpHjjyi8rPfdDDXXvahXDvroeiKnxlMQ49xdkuSr6qqWnf2bPWM2c0Tjklr7fwk\n5yfJnj172t69ewcdkvmzsrIS64JJrA0msS5YdcWNt+bFF+zPgx90VN6y7ztz7WUfsja4Dz8zmIYe\nX9Z4eZKjkjxq3fbVa80+sbPjAADLyl0ZgZ3UY5y9J8ndSZ69bvtzklzWWrt650cCAJaNMAN22qAv\na6yqs8ZfPnb8+alVdVOSm1prl4z3uSfJG1trP54krbXPVNV5SV5WVbcm+WiSZyU5PckZQ84LAJAI\nM2A2hr7m7K3rvn/d+PMlSfaOvz5y/LHWy5McSPLCJI9IckWSZ7bW3jXMmAAAI8IMmJVB46y1Voez\nT2vtS0leOf4AANgRwgyYpR6vOQMA2HHCDJg1cQYALD1hBvRAnAEAS02YAb0QZwDA0hJmQE/EGQCw\nlIQZ0BtxBgAsHWEG9EicAQBLRZgBvRJnAMDSEGZAz8QZALAUhBnQO3EGACw8YQbMA3EGACw0YQbM\nC3EGACwsYQbME3EGACwkYQbMG3EGACwcYQbMI3EGACwUYQbMK3EGACwMYQbMM3EGACwEYQbMO3EG\nAMw9YQYsAnEGAMw1YQYsCnEGAMwtYQYsEnEGAMwlYQYsGnEGAMwdYQYsInEGAMwVYQYsKnEGAMwN\nYQYsMnEGAMwFYQYsOnEGAHRPmAHLQJwBAF0TZsCyEGcAQLeEGbBMxBkA0CVhBiwbcQYAdEeYActI\nnAEAXRFmwLISZwBAN4QZsMzEGQDQBWEGLDtxBgDMnDADEGcAwIwJM4ARcQYAzIwwA/gycQYAzIQw\nA7g3cQYA7DhhBnBf4gwA2FHCDGAycQYA7BhhBrAxcQYA7AhhBnBo4gwAGJwwA7h/4gwAGJQwA9gc\ncQYADEaYAWyeOAMABiHMALZGnAEAUyfMALZOnAEAUyXMAA6POAMApkaYARw+cQYATIUwA9gecQYA\nbJswA9g+cQYAbIswA5gOcQYAHDZhBjA94gwAOCzCDGC6xBkAsGXCDGD6xBkAsCXCDGAYg8ZZVT2y\nqt5WVZ+vqi9U1Z9W1XGbPLZt8PFtQ84MAGxMmAEMZ9dQT1xVD07yviR3Jjk7SUvyyiTvr6pvaa3d\ntomneUOS16/bduU05wQANkeYAQxrsDhL8vwkJyXZ01q7Kkmq6uNJPpXkJ5Kct4nn+L+ttf3DjQgA\nbIYwAxjekC9rPCPJ/tUwS5LW2tVJ/ibJmQP+uQDAFAkzgJ0xZJydmuSyCdsvT3LKJp/jJ6vqzqq6\nvareV1X/anrjAQD3R5gB7JxqrQ3zxFV3JTmvtfbSddtfmeSlrbVDvqSyqt6U5F1J/l+S45P8h4yi\n7omttZUJ++9Lsi9Jdu/e/dgLL7xwGv8YLJADBw7koQ996KzHoEPWBpNYF8n1tx7MuZfekSOPqLz0\nXxydRzzETZ4Ta4PJrAsmefzjH/+R1tppm92/2zib8HxfkdGZuOtaa999qH337NnTrrjiiq2OzIJb\nWVnJ3r17Zz0GHbI2mGTZ14UzZhtb9rXBZNYFk1TVluJsyL8CuyXJwyZsP2b82Ja01m5N8u4k37HN\nuQCAQxBmALMxZJxdntF1Z+udkuQT23jeYU71AQDCDGCGhoyzdyZ5XFWdtLqhqk5I8l3jx7akqv5Z\nkqcluXRK8wEAawgzgNkaMs4uSHJNkndU1ZlVdUaSdyS5LmveWLqqjq+qe6rqF9dse0lVXVBVP1xV\ne6vq7Ixuwf+IJC8fcGYAWErCDGD2BnsT6tbabVV1epLfTvKmJJXkL5O8qLV2YM2uleTI3DsUr0jy\ng+OPr0zyhYzi7Mdba86cAcAUCTOAPgwWZ0nSWvuHJE+/n32uySjQ1m67KMlFw00GACTCDKAn3rAE\nAJaUMAPoizgDgCUkzAD6I84AYMkIM4A+iTMAWCLCDKBf4gwAloQwA+ibOAOAJSDMAPonzgBgwQkz\ngPkgzgBggQkzgPkhzgBgQQkzgPkizgBgAQkzgPkjzgBgwQgzgPkkzgBggQgzgPklzgBgQQgzgPkm\nzgBgAQgzgPknzgBgzgkzgMUgzgBgjgkzgMUhzgBgTgkzgMUizgBgDgkzgMUjzgBgzggzgMUkzgBg\njggzgMUlzgBgTggzgMUmzgBgDggzgMUnzgCgc8IMYDmIMwDomDADWB7iDAA6JcwAlos4A4AOCTOA\n5SPOAKAzwgxgOYkzAOiIMANYXuIMADohzACWmzgDgA4IMwDEGQDMmDADIBFnADBTwgyAVeIMAGZE\nmAGwljgDgBkQZgCsJ84AYIcJMwAmEWcAsIOEGQAbEWcAsEOEGQCHIs4AYAcIMwDujzgDgIEJMwA2\nQ5wBwICEGQCbJc4AYCDCDICtEGcAMABhBsBWiTMAmDJhBsDhEGcAMEXCDIDDJc4AYEqEGQDbIc4A\nYAqEGQDbJc4AYJuEGQDTIM4AYBuEGQDTIs4A4DAJMwCmSZwBwGEQZgBMmzgDgC0SZgAMQZwBwBYI\nMwCGIs4AYJOEGQBDEmcAsAnCDIChDRpnVfXIqnpbVX2+qr5QVX9aVcdt8tijq+o3quqGqrqjqj5Y\nVd8z5LwAMIkwA2AnDBZnVfXgJO9L8o1Jzk7y3CSPTvL+qtrMf9V+L8nzk/xikqcluSHJn1fVtw0z\nMQDclzADYKfsGvC5n5/kpCR7WmtXJUlVfTzJp5L8RJLzNjqwqr41yQ8n+XettT8Yb7skyeVJfiXJ\nGQPODQBJhBkAO2vIlzWekWT/apglSWvt6iR/k+TMTRx7d5I/WXPsPUnekuTJVXXU9McFgC+7/taD\nwgyAHTXkmbNTk7xjwvbLkzxjE8de3Vq7fcKxD0xy8vjriW7+Yssr3/WJLYzKMrju+jvzPw9YF9yX\ntcF6LcmFl96RBz/oKGEGwI4ZMs6OSXLLhO03J3nYNo5dffxeqmpfkn1J8sBjH5U3f/DqzU/KUmhp\nqeusC+7L2mCSrzqq5UXfekSuvexDuXbWw9CVAwcOZGVlZdZj0BnrgmkYMs52VGvt/CTnJ8mePXva\nJ3/t38x4InqzsrKSvXv3znoMOmRtMIl1wUasDSaxLpiGIa85uyWTz5BtdFZss8cmXz6DBgAAsBCG\njLPLM7p2bL1TktzfxR2XJzlxfDv+9cfeleSq+x4CAAAwv4aMs3cmeVxVnbS6oapOSPJd48cO5aIk\nD8iaG4dU1a4kz0ryP1prd057WAAAgFkaMs4uSHJNkndU1ZlVdUZGd2+8LsnrV3eqquOr6p6q+sXV\nba21j2V0G/3XVNXzqur7MrqN/olJfmnAmQEAAGZisDhrrd2W5PQkVyZ5U5I/SnJ1ktNbawfW7FpJ\njpwwy48l+YMkr0zy7iSPTPKU1tpHh5oZAABgVga9W2Nr7R+SPP1+9rkmo0Bbv/2OJC8efwAAACy0\nIV/WCAAAwCaJMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6I\nMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAA\ngA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6I\nMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAA\ngA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6I\nMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA6IMwAAgA4MFmdVdURVvayqrqmqL1bV\n31fV0zd57Buqqk34eM1Q8wIAAMzSrgGf+1eTvCTJy5N8JMm/TfLWqnpaa+3iTRx/U5Iz1m27Yboj\nAgAA9GGQOKuqh2cUZq9urf3mePP7q+rkJK9Ospk4u6u1tn+I+QAAAHoz1Msan5zkgUnevG77m5N8\nc1WdONCfCwAAMJeGirNTk9yZ5Kp12y8ffz5lE8/x8Kr6bFXdU1VXVtXPV9WRU50SAACgE0Ndc3ZM\nks+11tq67TevefxQ/i6j69QuT3J0kh9M8qokj07yvEkHVNW+JPuSZPfu3VlZWTmswVlcBw4csC6Y\nyNpgEuuCjVgbTGJdMA2birOqekKS925i10taa3u3NVGS1tr6uzJeXFUHkryoqs5trX1qwjHnJzk/\nSfbs2dP27t32GCyYlZWVWBdMYm0wiXXBRqwNJrEumIbNnjn7QJLHbGK/28efb0nyVVVV686erZ4x\nuzlb98dJXpTktCT3iTMAAIB5tqk4a63dnuSTW3jey5McleRRufd1Z6vXmn1iC891n3G2cSwAAECX\nhrohyHuS3J3k2eu2PyfJZa21qw/jOZ+dUZh9aJuzAQAAdGeQG4K01j5TVecleVlV3Zrko0meleT0\nrHtj6ar6yyTHt9ZOHn9/fJI3JXlLRmfdjsrohiA/muT1rbVPDzEzAADALA11t8YkeXmSA0lemOQR\nSa5I8szW2rvW7XfkujluzeiatJ9PcmySgxm9pPJnk7xuwHkBAABmZrA4a619Kckrxx+H2m/vuu9v\nTvIDQ80FAADQo6GuOQMAAGALxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkA\nAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAH\nxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkA\nAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAH\nxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkA\nAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHxBkAAEAHBouzqnpx\nVV1UVTdUVauqc7Z4/HdX1Qeq6o6qurGqzquqBw00LgAAwEwNeebs+UkenuTtWz2wqr4lyXuTfCbJ\n05K8IsmPJXnDFOcDAADoxq4Bn/vU1trBqtqV5AVbPPaXk1yf5BmttbuTpKruSvLGqjq3tfbRKc8K\nAAAwU4OdOWutHTyc46rqAUmekuTC1TAbuzDJXUnOnMJ4AAAAXenxhiCPSnJ0ksvWbmytfTHJp5Oc\nMouhAAAAhtRjnB0z/nzLhMduXvM4AADAwtjUNWdV9YSMbtBxfy5pre3d1kSHqar2JdmXJLt3787K\nysosxqBjBw4csC6YyNpgEuuCjVgbTGJdMA2bvSHIB5I8ZhP73b6NWVatnjF72ITHjkly+aSDWmvn\nJzk/Sfbs2dP27t07hVFYJCsrK7EumMTaYBLrgo1YG0xiXTANm4qz1trtST458CyrPp3kziSnrt1Y\nVUcnOSnJW3doDgAAgB3T3TVnrbW7krwnyTPHt+FfdVaSo5K8cyaDAQAADGiw9zmrqtOSnJAvB+Ap\nVXXW+OuLx2fjUlW/l+Ts1traWc5Jsj/JhVX1O+Pn+Y0kb2utfWSomQEAAGZlyDeh/ukkZ6/5/hnj\njyQ5Mck146+PHH/8k9ba31XVk5Kcm+TdST6f5A+T/MKA8wIAAMzMYHHWWvvRJD96uPu11v4qyXdO\neSwAAIAudXfNGQAAwDISZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0Q\nZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAA\nAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0Q\nZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAA\nAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0Q\nZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0QZwAAAB0YLM6q6sVVdVFV\n3VBVrarO2cKx54yPWf/x9qHmBQAAmKVdAz7385N8Icnbk7zgMJ/ju5N8ac33N293KAAAgB4NGWen\nttYOVtWuHH6c/W1r7Z5pDgUAANCjwV7W2Fo7ONRzAwAALJrebwhyXVV9qaqurapzq+pBsx4IAABg\nCEO+rHE7rkry0iQfS9KSPCnJzyX59iRPnOFcAAAAg9hUnFXVE5K8dxO7XtJa27utiZK01t68btN7\nq+r6JK+pqie01v5iwoz7kuwbf3tnVV223TlYOF+T5LOzHoIuWRtMYl2wEWuDSawLJtmzlZ03e+bs\nA0kes4n9bt/KH75Ff5zkNUm+I8l94qy1dn6S85Okqj7cWjttwFmYQ9YFG7E2mMS6YCPWBpNYF0xS\nVR/eyv6birPW2u1JPnlYE01fm/UAAAAA09b7DUHWevb486UznQIAAGAAg90QpKpOS3JCvhyAp1TV\nWeOvLx6fjUtV/V6Ss1tru9Yc+7Ekf5jkiozOlD0xyc8keU9r7X2b+OPPn8o/BIvGumAj1gaTWBds\nxNpgEuuCSba0Lqq1YV4lWFVvSHL2Bg+f2Fq7Zu1+rbVac+xbMrq27Gszirv/k9E1Z/+ptXbnIAMD\nAADM0GBxBgAAwObN0zVnW1ZVX1FVF1bVVVV1W1V9rqourarnzHo2ZqeqvqGq/nNVfbyqDlTVDVX1\nzqr61lnPxuxV1Yur6qLxumhVdc6sZ2LnVNUjq+ptVfX5qvpCVf1pVR0367mYrar6+qp6bVV9sKpu\nH/9sOGHWczFbVXVWVf23qrq2qu6oqiuq6lVV9RWzno3ZqqonV9X7qurGqrqzqq4fN8kp93fsQsdZ\nkgcmuSfJq5KckeSHk/zvJG+qqp+b5WDM1JOSPD7JG5N8f5J/n2R3kv1V9dhZDkYXnp/k4UnePutB\n2FlV9eAk70vyjRm9LP+5SR6d5P1V9ZBZzsbMnZzkmUluSfLXM56FfrwkyZeS/EKSpyT5r0l+MqP3\n513037E5tGOSfCTJT2f0e+fLkpya0e+axx/qwKV8WWNVfTDJQ1tr3zzrWdh5VfU1Sf6xrVn8VfWV\nSa5JclFr7UdmNRuzV1VHtNYOVtWuJHcn+eXW2jkzHosdUFUvTHJekj2ttavG205M8qkk/7G1dt4s\n52N2Vn8ujL9+XpILsub6eZZTVe1urd20btuPZPSXv9+3yZvYsSSqak9Gb032ktbab22037JW/T9m\ndEaNJdRa+2xb97cSrbXPJ7kyydfNZip6sfoLGEvpjCT7V8MsSVprVyf5myRnzmwqZs7PBSZZH2Zj\nHxp/9vsE6/3j+PMhG2Qp4qxGdlXVV1fVviRPTvLbs56LflTVMUm+KaOXvQLL6dQkl03YfnmS+71O\nACDJ944/+32CVNWRVfXAqnp0ktcnuTGjO9BvaLD3OevMTyV57fjru5O8sLX2hzOch/68Nkklec2s\nBwFm5piMrila7+YkD9vhWYA5U1Vfl+RXkvxFa+3Ds56HLvxtktX7GVyV5PTW2mcOdcBcnTmrqieM\n75B0fx8r6w79k4zeN+2pSX43yWur6id2en6GsY11sXr8yzK6WcxPr305E/Nvu2sDADajqh6a5B0Z\nvWTtx2Y8Dv14bpLHZfR75hcyulnMCYc6YN7OnH0gyWM2sd/ta78ZvyZ49XXB7xnfkes3q+r3W2t3\nT3lGdt5hrYskqaoXJPn1JK9orf3+tAdj5g57bbCUbsnkM2QbnVEDSFU9KMlFSU5K8r2ttetnPBKd\naK2tvrz1b6vqv2d087mXJnnBRsfMVZy11m7P6C4n2/XhjG6TfGwS/wLNucNdF1X13CSvS/JbrbVf\nm/pgzNwUf2awHC7P6Lqz9U5J8okdngWYA1X1gCRvS3Jakie21v7XjEeiU621z1XVVRm9NceG5upl\njVP0vUkOJDnkaz5ZXFX1g0n+IMnvttZeMut5gC68M8njquqk1Q3jl5981/gxgH8yfi+zP0pyepIf\naK3tn/FIdKyqjs3ofTQ/faj95urM2VaNryt7XJK/yOgM2Vdn9CaSZyV5aWvtrhmOx4xU1fdkdKec\nv0/yhqp63JqH72ytfWw2k9GDqjotyQn58l9enVJVZ42/vnh8No7FdEFGbxj6jqp6RZKW5FeTXJfR\nXbZYYmt+Dqxe3P/UqropyU2ttUtmNBaz9TtJnpHk15Lctu73ieu9vHF5VdWfJfloko9ndK3ZNyT5\nuYyuSdzwPc6SBX8T6qr6l0lekeSfZ3TNwGczurXpb7fW3j3L2ZidqjonyS9t8PC1rbUTdm4aelNV\nb8joZc+TeNPZBVdVx2X0VitPzOgOrn+Z5EX+f6eqNvqF6ZLW2t6dnIU+VNU1SY7f4OFfbq2ds3PT\n0JOq+vmMTgg9KskDM/pLvpUkr7q//54sdJwBAADMi2W95gwAAKAr4gwAAKAD4gwAAKAD4gwAAKAD\n4gwAAKAD4gwAAKAD4gwAAKAD4gwAAKAD4gwAAKAD/x+GxLe9ETqZ0QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f87f13c0d10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(lambda x: max(0, x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Softplus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "$$ g(x) = \\log(1 + e^{x}) $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "* Wygładzona wersja ReLU."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### Softplus – wykres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2cAAAG9CAYAAACRcQ4FAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3Xl8XWWB//HPkz1N0jZt032nbbrR\nQkEWQakFBAVRQXAUHVfQ+SmKjuP+G0H56TAqowPqgCsuqIgLiw4oS9kXgbZAaUv3vU2XpNn35/dH\nUig1hbTN7TlJPu/X677uveeec/uF10OaL885zwkxRiRJkiRJycpKOoAkSZIkyXImSZIkSalgOZMk\nSZKkFLCcSZIkSVIKWM4kSZIkKQUsZ5IkSZKUAhkrZyGEsSGEa0MIj4YQ6kMIMYQwsZvHruvcf//H\n2zKVV5IkSZKSlJPB754CXAQ8BTwIvPEgj78LuGK/bSsOP5YkSZIkpU8my9kDMcYRACGED3Pw5Wxn\njPGxno8lSZIkSemTsdMaY4ztmfpuSZIkSepr0rwgyFs6r1VrCiE85vVmkiRJkvqytJaz24HLgLOA\ni4FG4I8hhPckmkqSJEmSMiTEGDP/h3Rcc/ZDYFKMcd0hHJ8NPAaMjDGOO8A+lwKXAhQUFBw3fvz4\nQw+sPqm9vZ2srLT+/wglybGhrjgudCBJjY09TZHKpkhuFgwfkEWuwzNV/Jmhrrzwwgs7Y4xl3d0/\nkwuC9JgYY1sI4XfA1SGEUTHGrV3scwNwA0B5eXlcscKFHfVyCxcuZP78+UnHUAo5NtQVx4UO5EiP\njT31LXz65sXcs7yCS+aM4j8umENxfq/4Fa5f8WeGuhJCWH8w+/fG/7IzP9UnSZKUAs9squL//Opp\ntlc3cuV5s/jnkycQQkg6lqQM6RXlLISQA7wT2BBj3JZ0HkmSpEyKMXLTExu48rbnGVacx80fOZlj\nx5cmHUtShmW0nIUQ3tH58rjO5zeFEHYAO2KM93fu0wrcGGP8UOf7dwFvBf4CbARGAB8D5gHvymRe\nSZKkpNU3t/KlPz7HHxdt5rRpZXznncdQWpSXdCxJR0CmZ85+t9/773c+3w/M73yd3fnYay0wHPgm\nMASoA54Ezo4x3pWxpJIkSQlbVVHLv/zyKVbtqOVfz5zGx94whawsT2OU+ouMlrMY46v+NNl/nxjj\nY8CCjIWSJElKoduWbOHzv3+GwtxsfvHBEzl16rCkI0k6wnrFNWeSJEl9VVNrG1//8zJufHQ9x08o\n5bp3z2PkoIKkY0lKgOVMkiQpIZsq6/nYTYtYsrGKS143ic+ePZ3cbO+VJfVXljNJkqQE3Leigk/9\ndjFtbZH/ec88zp49KulIkhJmOZMkSTqC2toj37n7Ba69dxUzRg3kBxfPY+KwoqRjSUoBy5kkSdIR\nUlHTyOW/Wcwjq3fxzuPHceVbZ1GQm/3qB0rqFyxnkiRJR8BDK3dy+W8XUdvUyjffMYcLjx+XdCRJ\nKWM5kyRJyqDWtna+e89KrrtvFVPKivn1JScxdURJ0rEkpZDlTJIkKUO2Vzdy2a8X8cTa3Vx0/Fiu\nPG82hXmexiipa5YzSZKkDLj/hR186reLaWhu45qL5nL+vLFJR5KUcpYzSZKkHtTa1s63//YCP1i4\nmvIRJXzv4nlMGV6cdCxJvYDlTJIkqYdsqWrgE79exJPrK3nXCeP4yltcjVFS91nOJEmSesC9y7fz\n6ZuX0NLaznf/6RjeesyYpCNJ6mUsZ5IkSYehpa2db961ghseWMOMUQP53ruPZXKZpzFKOniWM0mS\npEO0qbKey369iEUbqrj4xPH833NnehqjpENmOZMkSToEdz63lc/9/lna2iPXvftYzp0zOulIkno5\ny5kkSdJBaG6LfOmPz/Krxzdw9JhBXPuuY5k4rCjpWJL6AMuZJElSN63YVsOVjzawuXYDl75+Mp95\nYzl5OVlJx5LUR1jOJEmSXkWMkV8+voGr7nie/KzIjR88gdOmlSUdS1IfYzmTJEl6BVX1zXzu989w\n19LtvH5aGReMqbOYScoI5+ElSZIO4PE1u3jTdx/k3uUVfOnNM/jZ+1/DoPyQdCxJfZQzZ5IkSftp\nbWvn2ntXce29Kxk/ZAC//5fXMmfs4KRjSerjLGeSJEn72FzVwKd+s5gn1u3m/GPH8NW3zaY431+Z\nJGWeP2kkSZI67b13WWtbO//1zrm8/dixSUeS1I9YziRJUr/X2NLG1+54nl89voE5Ywfx3//kvcsk\nHXmWM0mS1K8t3bKHT/5mMasqavnI6yfzr967TFJCLGeSJKlfam+P/PDBNXzrrysoHZDHLz50Aq+b\n6hL5kpJjOZMkSf3OlqoG/vXmJTy6ZhdnzxrJN84/mtKivKRjSernLGeSJKlfueOZLXzxD8/S2h75\nzwvmcOHxYwnBe5dJSp7lTJIk9Qs1jS185dal/GHRZo4ZN5jvvPMYF/2QlCqWM0mS1Oc9uW43n7p5\nMZsrG/jE6VO5bMEUcrNd9ENSuljOJElSn9XS1s6196zkuvtWMaa0kN999GSOmzAk6ViS1CXLmSRJ\n6pPW7azjk79dzJKNVbzjuLF85S0zKSnITTqWJB2Q5UySJPUpMUZ++/eNfPWO58nNzuL7F8/jzUeP\nSjqWJL0qy5kkSeozdtU28cU/PstdS7fz2qOG8u2L5jJqUGHSsSSpWyxnkiSpT/jr0m184Q/PUtPY\nypfePIMPnTqJrCyXyJfUe1jOJElSr1bd2MJXb3+eW57axKzRA7npkmMoH1mSdCxJOmiWM0mS1Gs9\nsmon/3bLM2yrbuSyBVO4bMFU8nJcIl9S72Q5kyRJvU5DcxtX37mcnz2yjsnDirjloydz7PjSpGNJ\n0mGxnEmSpF5l8cYqPn3zYtbsqOP9r53I586eTmFedtKxJOmwWc4kSVKv0NzaznX3ruR7C1czoiSf\nX334RE6ZMizpWJLUYyxnkiQp9V7YXsOnfruYpVuquWDeWL5y3kwGekNpSX2M5UySJKVWW3vkxw+t\n4Vt/fYGS/Bz+5z3HcfbskUnHkqSMsJxJkqRU2rCrns/8bglPrNvNmTNH8I3zj2ZYcX7SsSQpYyxn\nkiQpVdrbIz9/dB1X37mCnKzAty6cywXzxhCCN5SW1LdZziRJUmqs21nHZ3//DE+s3c1p08r4xvlH\nM3pwYdKxJOmIsJxJkqTEtbdHfvbIOv7zruXkZmXxn++Yw4XHjXW2TFK/YjmTJEmJWruzjs/esoS/\nr6vkDeVlfP38oxk1yNkySf2P5UySJCWirXO27Jt3LSc3O8tryyT1e5YzSZJ0xK3ZUctnb3mGJ9dX\nsmD6cL7+9qMZOagg6ViSlCjLmSRJOmLa2iM/fXgt37xrBfk5WXz7wrmc72yZJAGWM0mSdISs7pwt\ne2p9JadPH87Xzz+aEQOdLZOkvSxnkiQpo1ra2rn+/tX89z2rKMjN4pqL5vL2Y50tk6T9Wc4kSVLG\nPLOpis/e8gzLt9Xw5qNHcsV5sxhe4myZJHXFciZJknpcQ3Mb1/xtBT9+aC3DivO5/r3HcdaskUnH\nkqRUy8rUF4cQxoYQrg0hPBpCqA8hxBDCxG4emxVC+EIIYV0IoTGEsCSEcEGmskqSpJ7zyKqdnPWd\nB/jhg2t552vG8bdPn2Yxk6RuyFg5A6YAFwGVwIMHeezXgCuA64A3AY8BvwshvLknA0qSpJ6zp76F\nz93yDO/+0eNkBbjpkhP5xvlzGFSYm3Q0SeoVMnla4wMxxhEAIYQPA2/szkEhhOHAZ4D/iDF+q3Pz\nfSGEKcB/AH/JRFhJknTo7nxuK//31qXsrmvmI6dN5lNnTKMgNzvpWJLUq2SsnMUY2w/x0LOAPOCX\n+23/JfCTEMKkGOPawwonSZJ6REV1I/9+61LuXLqNmaMG8tP3v4bZYwYlHUuSeqU0LggyC2gCVu23\nfWnn80zAciZJUoLa2yM3P7mRr/9lGY2t7Xz27HIued1kcrMzecWEJPVtaSxnQ4CqGGPcb/vufT6X\nJEkJeWF7DV/8w7M8ub6SEycN4RvnH83ksuKkY0lSr5fGcnZIQgiXApcClJWVsXDhwmQDKXVqa2sd\nF+qSY0NdcVz8o+a2yG2rW/jftS0U5MCHZudx6phGNix9kg1JhzuCHBvqiuNCPSGN5awSGBxCCPvN\nnu2dMdvdxTHEGG8AbgAoLy+P8+fPz2hI9T4LFy7EcaGuODbUFcfFy93/wg6u+tNzbNjdwgXzxvLF\nN09naHF+0rES4dhQVxwX6glpLGdLgXzgKF5+3dnMzufnj3giSZL6qYqaRr52xzJuX7KFycOKuOmS\nE3ntUcOSjiVJfVIay9mdQAtwMXDlPtvfAzznSo2SJGVee3vkpic2cPWdy2lqaefyM6byL/OPIj/H\n5fElKVMyWs5CCO/ofHlc5/ObQgg7gB0xxvs792kFbowxfgggxlgRQrgG+EIIoQZ4GngnsAA4L5N5\nJUkSLN9WzRf/8CxPb6ji5MlDuertsznKBT8kKeMyPXP2u/3ef7/z+X5gfufr7M7Hvr4E1AKfBEYC\nK4CLYox3ZCamJElqaG7ju/es5EcPrqGkIIdvXziX8+eNIYSQdDRJ6hcyWs5ijK/607yrfWKMbcBV\nnQ9JkpRBMUb++vx2vnr782yuauDC48byhTfPYEhRXtLRJKlfSeM1Z5Ik6QhZv6uOK25byn0rdjBt\nRDG/vfQkTpw8NOlYktQvWc4kSeqHGlva+P7C1fzP/avJzQp8+ZwZvO+1E8nNzko6miT1W5YzSZL6\nmXuWbeeK25eycXcD580dzZfOmcGIgQVJx5Kkfs9yJklSP7Fxdz1X3r6Uu5dVMGV4sfcsk6SUsZxJ\nktTHNba0cf39a/j+wlVkZwW++ObpfOCUSZ7CKEkpYzmTJKkPu29FBVfctpT1u+o5d84ovnTODEYN\nKkw6liSpC5YzSZL6oPW76rjqz8v42/PbOaqsiF99+EROmeIpjJKUZpYzSZL6kNqmVr533yp+/OBa\ncrIDnzt7Oh86dRJ5OZ7CKElpZzmTJKkPaG+P/HHRZq6+czkVNU2cP28Mnzt7uqswSlIvYjmTJKmX\nW7yxiituW8rijVXMHTeY6997HMeOL006liTpIFnOJEnqpSqqG7n6zhX8/ulNlJXk8+0L5/L2Y8eQ\nlRWSjiZJOgSWM0mSepmm1jZ+8tA6rrt3JS1tkY+edhQfXzCF4nz/Wpek3syf4pIk9RIxRu5ZVsFV\nf36edbvqOWPGCL58zgwmDitKOpokqQdYziRJ6gWWb6vm//15GQ+u3MmU4cX8/IMn8PppZUnHkiT1\nIMuZJEkpVlHdyDV/e4Gbn9xISUEu/37uTN578gRys10aX5L6GsuZJEkpVN/cyg8fWMv1D6ympa2d\nD5wyicsWTGHwgLyko0mSMsRyJklSirS3R37/9Ca+9dcVbK9u4k2zR/K5s6d7XZkk9QOWM0mSUuKR\nVTu56s/LeH5rNXPHDea6d8/jNROHJB1LknSEWM4kSUrYqopavvGXZdyzvIIxgwv57j8dw1vmjPZ+\nZZLUz1jOJElKyK7aJr5z90puemIDA3Kz+dzZ0/nAKRMpyM1OOpokKQGWM0mSjrC6plZ+/NBabnhg\nDQ0tbbz7hPFcfsZUhhbnJx1NkpQgy5kkSUdIS1s7v3liA9+9ZxU7a5t448wRfPbscqYML0k6miQp\nBSxnkiRlWHt75I5nt/Ltv65g/a56Tpg0hOvfexzHTShNOpokKUUsZ5IkZdCDK3dw9Z3LeW5zNdNH\nlvDT97+G+eVlhOBiH5Kkl7OcSZKUAc9squLqO5fz8KpdjBlcyDUXzeWtx4wh2xUYJUkHYDmTJKkH\nrd1Zx7f+uoI/P7OV0gG5/Pu5M7n4pPHk57gCoyTplVnOJEnqAdv2NHLdfSv5zRMbycvJ4hMLpnDJ\n6ydTUpCbdDRJUi9hOZMk6TDsqGniBwtX88vH19PeHnnXCeO57PQpDC8pSDqaJKmXsZxJknQIKuua\nuf6BNdz4yDqaWtu4YN5YPnH6VMYNGZB0NElSL2U5kyTpIFQ3tvCjB9fyk4fWUtfcynlzR/PJ06cy\nuaw46WiSpF7OciZJUjfUNbXys0fWccMDa9jT0MKbZo/k8jOmUT7SG0hLknqG5UySpFfQ2NLGLx5d\nz//cv5pddc0smD6cT585jdljBiUdTZLUx1jOJEnqQnNb5MZH1vG9+1ZRUdPEqVOG8ek3TmPe+NKk\no0mS+ijLmSRJ+2hsaeOmxzfw3w80UNW0lBMmDuG/33UsJ00emnQ0SVIfZzmTJAmob27lV49t4PoH\n1rCztony0iyue88JnDJlKCGEpONJkvoBy5kkqV+rbWrl54+u40cPrmV3XTOnTBnKdQuOpXHDs5w6\ndVjS8SRJ/YjlTJLUL1U3tnDjw+v48cNrqapv4bRpZXzi9CkcN2EIAAs3JBxQktTvWM4kSf3KnvoW\nfvLwWn7y8FpqGls5ffpwLjt9KseMG5x0NElSP2c5kyT1C7tqm/jJw2u58ZH11Da1ctasEVy2YKpL\n4kuSUsNyJknq0zburudHD67ht09upKm1nTfPHsXHF0xhxqiBSUeTJOllLGeSpD5p+bZqrr9/Dbct\n2UJWgLcfO4ZLX38UU4YXJx1NkqQuWc4kSX3Kk+t284OFq7lneQUD8rL5wGsn8qHXTWLUoMKko0mS\n9IosZ5KkXi/GyH0rKvjBwtX8fV0lpQNy+fSZ0/jnkycweEBe0vEkSeoWy5kkqddqbWvnjme28j/3\nr2b5thrGDC7kirfM5KLXjGNAnn/FSZJ6F//mkiT1OrVNrdz894385OG1bKpsYNqIYq65aC5vmTua\n3OyspONJknRILGeSpF5jS1UDNz6yjpue2EBNYyvHTyjlirfMYsH04WRlhaTjSZJ0WCxnkqTUe3bT\nHn744Br+/OxWAN40eyQfOnUSx44vTTiZJEk9x3ImSUql9vbI3cu286OH1vLE2t0U5+fwgddO5P2n\nTGRs6YCk40mS1OMsZ5KkVKlvbuX3T23ixw+tZd2uesYMLuTL58zgna8ZR0lBbtLxJEnKGMuZJCkV\ntu1p5BePreNXj2+gqr6FueMGc91Z5Zw9ayQ5LvIhSeoHLGeSpMTEGPn7ukpufGQddy7dRnuMnDVz\nJB9+3SSOm1BKCC7yIUnqPyxnkqQjrqG5jVsXb+bGR9ezbGs1Awty+OApE3nvSRMZP9TrySRJ/ZPl\nTJJ0xGzcXc8vH1vPb/6+kT0NLUwfWcI3zj+atx0zhsK87KTjSZKUKMuZJCmjYow8vGoXP3tkHfcs\n305WCJw1awTvO3kiJ0wa4qmLkiR1spxJkjKitqmVPzy9iRsfWcfqHXUMLcrjY/On8O4TxzN6cGHS\n8SRJSp2MlrMQwjjgv4AzgQDcDVweY9zQjWPjAT46Nsa4uOdSSpJ60rOb9nDTE+u5dfEW6pvbmDN2\nEN++cC7nzBlFQa6nLkqSdCAZK2chhAHAvUAT8D4gAlcB94UQ5sQY67rxNT8Drt9v2ws9mVOSdPjq\nmlq5bckWbnp8A89u3kNBbhZvmTOad584nmPHlyYdT5KkXiGTM2eXAJOB8hjjKoAQwjPASuAjwDXd\n+I7NMcbHMhdRknQ4lm7Zw02Pb+DWxVuobWqlfEQJV543i7cdO4ZBhd4wWpKkg5HJcnYe8NjeYgYQ\nY1wbQngYeCvdK2eSpJSpb27ljiVb+dUTG1iysYr8nCzOmTOKi08cz7zx3ptMkqRDlclyNgu4tYvt\nS4ELu/kd/xJC+DegDXgM+EqM8cEeyidJOgjLtlbzmyc28IdFm6lpbOWosiL+77kzuWDeGAYPyEs6\nniRJvV6I8UDrbhzmF4fQDFwTY/z8ftuvAj4fY3zFYhhC+AVwB7AFmAD8GzATODPGuLCL/S8FLgUo\nKys77uabb+6Jfwz1IbW1tRQXFycdQynk2Diw2ubIY1tbeXBzK+ur28kJcPzIbN4wLpdppVl9epbM\ncaEDcWyoK44LdeUNb3jDUzHG47u7f2rLWRffVwI8B2yMMZ76SvuWl5fHFStWHGxk9XELFy5k/vz5\nScdQCjk2Xq6tPfLwqp3c/ORG/vr8dppb25kxaiAXHT+Wtx4zhiFF/WOWzHGhA3FsqCuOC3UlhHBQ\n5SyTpzVWAl0t0TWk87ODEmOsCSH8GfjQ4QaTJP2j9bvquOWpTdzy1Ca27mlkUGEu7z5hPO84biyz\nxwxKOp4kSX1eJsvZUjquO9vfTOD5w/jezEz1SVI/VN/cyv8+u42bn9zI42t3EwK8fmoZXz5nJmfM\nHE5+jvclkyTpSMlkObsN+FYIYXKMcQ1ACGEicArw+Vc4rkshhIHAucATPZhRkvqd9vbIY2t38adF\nm/nLs9uobWpl4tAB/NtZ5Zw/bwyjBhUmHVGSpH4pk+Xsh8DHgVtDCF+mY8bra8BG9rmxdAhhArAa\n+GqM8aud2z4DlAP38dKCIJ8BRgIXZzCzJPVZK7bV8MdFm7l18Wa27mmkKC+bNx09iouOH8drJroE\nviRJSctYOYsx1oUQFgD/BfwCCMA9wOUxxtp9dg1ANpC1z7YVwNs7H4OAauBh4EMxRmfOJKmbtlc3\ncuvizfxx0RaWba0mOytw2rQyvvDmGZw5YwSFeZ62KElSWmRy5owY4wbgglfZZx0dBW3fbbcDt2cu\nmST1XbVNrdz53Db+tGgzD6/eSYxwzLjBXHneLM6ZM4phxflJR5QkSV3IaDmTJB0Zza3tPLhyB39a\nvIW/Pb+NxpZ2xg8ZwGULpvK2Y0Yzucx770iSlHaWM0nqpVra2nlk9S7uWLKFu5Zuo7qxldIBuVx4\n3DjeduwY5o0f7HVkkiT1IpYzSepF2tojj6/Zxe3PbOXO57ZSWd9CSX4OZ84awVvmjOaUKcPIy8l6\n9S+SJEmpYzmTpJRrb4/8fd1u7nhmK//73FZ21jYzIC+bM2aM4Nw5o3j9tDIKcl3YQ5Kk3s5yJkkp\n1N4eWbSxkjue2cpfnt3K9uomCnKzWDB9OOfOGc0byoe70qIkSX2M5UySUqKlrZ0n1u7mzue2cdfS\nbVTUNJGXncVp5WWcO2cUZ8wYQVG+P7YlSeqr/FtekhLU2NLGgyt3cudz27hn+Xaq6lsozM1mfnkZ\nZ80ayYIZwxlYkJt0TEmSdARYziTpCKtpbOG+FTu467lt3LeigvrmNgYW5HDGjBGcNXskr59a5imL\nkiT1Q5YzSToCKmoauW95BXct3c5DK3fS3NbOsOJ83n7sGM6ePZKTJg8lN9tVFiVJ6s8sZ5KUATFG\nlm2t4Z5l27l7eQVLNlYBMLa0kH8+eQJnzx7JseNLyc7yPmSSJKmD5UySekhjSxuPrdnFPcsquGfZ\ndrbsaSQEmDt2MJ954zQWTB/BjFEl3hhakiR1yXImSYdhR00T9y2v4O5l23lo1U7qm9sozM3mdVOH\ncfkZ03jD9OGUleQnHVOSJPUCljNJOght7ZFnNlVx/ws7WLhiB0s2VREjjBpUwPnzxnD6jBGcPHmo\nN4WWJEkHzXImSa+ioqaRB1/YycIXdvDgyh1U1bcQAswZO5hPnTGN02cMZ+aogZ6uKEmSDovlTJL2\n09LWztPrK7n/hR3c/8IOlm6pBmBYcT4Lpg9nfvlwXjdlGKVFeQknlSRJfYnlTJKAjbvreXDlTh54\nYQcPr9pJTVMr2VmB4yaU8m9nlXPatDJmjhpIlqsrSpKkDLGcSeqXdtU28eiaXTy8aid3P1vPjjvv\nAzquHTt37ihOm1bGa6cMY2BBbsJJJUlSf2E5k9Qv1De38sTa3Ty8aicPr9rF81s7TlUsyc9hyqAs\nPnZGOadMGcaU4cVeOyZJkhJhOZPUJ7W0tfPMpioeXrWLh1btZNGGSlraInnZWcyb0HHfsVOmDOPo\nMYN46MEHmH/KpKQjS5Kkfs5yJqlPaGptY8nGPTy+ZhePr93NU+sraWhpIwSYNXogHzx1EqccNYzX\nTBxCYZ7L3EuSpPSxnEnqlRpb2nh6QyWPr9nN42t3sWhDFU2t7QBMH1nCRceP5cTJQzl58lBXVZQk\nSb2C5UxSr1Df3MpT618qY0s27qG5rZ0QYOaogVx84gROnDyEEyYOsYxJkqReyXImKZUqqht5an0l\nT66v5Kn1lTy3eQ+t7ZHsrMDs0QN5/ykTOXHSEI6fOIRBha6oKEmSej/LmaTEtbVHVmyr4an1HdeK\nPbWhko27GwDIz8li7tjBXPL6yS+WseJ8f3RJkqS+x99wJB1xNY0tLNpQ1VHE1leyaEMldc1tAJSV\n5HP8hFLed/JEjptQyqzRg8jLyUo4sSRJUuZZziRlVGtbOy9sr2XJpiqWbKxi8cYqVmyvIUbIClA+\nciDnzxvLcRNKOW5CKWNLC73PmCRJ6pcsZ5J6TIyRTZUNLN7YUcSWbKri2c17aGzpWEVxUGEuc8YO\n4uzZIzl+whDmjhtESYHXi0mSJIHlTNJh2F3X/OKMWEcZ28PuumYA8nKymD16IO86YTzHjBvM3LGD\nmTB0gLNikiRJB2A5k9QtFTWNLN1SzdLNe3huczXPbdnDpsqORTtCgGnDSzhjxnDmdhax8pEl5GZ7\nrZgkSVJ3Wc4kvUyMkc1VDS8VsS3VPLd5DxU1TS/uM2lYEceMG8x7TprAMeMGM3vMIFdQlCRJOkz+\nNiX1Y61t7azdWceybTUs3bKHpZ0zYlX1LUDHgh1Th5dw6tRhzB49iFmjBzJz9ECvE5MkScoAy5nU\nT+yoaWL5tmqWb61h+bYalm+rZmVFLc2tHYt15GVnUT6yhDfNHsmsziI2feRACvOyE04uSZLUP1jO\npD6msaWNVRW1LNta/WIJW7Gthp21zS/uM7wkn+mjBnLqlGFMH1VC+YiBTBle7P3EJEmSEmQ5k3qp\nptY21u6sY+X2WlZVdDyWb6tm7c462mPHPvk5HbNhC6YPZ/rIgUwfVcL0kQMZUpSXbHhJkiT9A8uZ\nlHL1za2s2VHHyooaVm6vZWVFLasralm366USFgKMHzKAqcNLOOfoUUwfNZDpI0uYMLSI7CyXrpck\nSeoNLGdSCsQYqaxvYe3OWlZX1LFqRy0rt9ewsqL2xeXqAXKyAhOHFVE+soRz5oxiyvBipg4vYXJZ\nEQW5XhsmSZLUm1nOpCOorqlLyrXLAAAX4UlEQVSVdbvqWLuzjrU7Op7X7Ox43tPQ8uJ+eTlZTB5W\nxLHjS7no+HFMHV7MlOHFTBha5HVhkiRJfZTlTOphza3tbKys36981bJ2Zx3bq5tetu+oQQVMGlbE\nuXNGMWlYEZPLipg0rJjxQwZ4OqIkSVI/YzmTDsGehhY27q5nw76PXR3Pm6saaNt7MRhQOiCXScOK\nOHVKGZOGDWDSsGImDSti4rABDMjzP0FJkiR18DdDqQutbe1s3dP48vK1u/7FQrb3Js17lQ7IZfyQ\nAcwZO4i3zB3F5GHFTCorYtLQIkpdGVGSJEndYDlTv9Tc2s62PY1srmroeFQ2sKXz9cbKejZXNtC6\nz+xXTlZgbGkh44YM4JyjRzFh6ADGDxnAuM7HwILcBP9pJEmS1BdYztQn1Ta1srmygc1V9WyuamRz\nZQOLVjRy7bJH2FzZwPaaRmJ8+TFlJfmMHlzI0WMGvVjAxg3pKGGjBhV6DZgkSZIyynKmXqe2qZVt\nexrZXt3I1s7nbXs6XnfMgtVT3dj6smNyswOD82DKqCxOnTqM0YMLGTu4kDGlhYweXMioQQUuRS9J\nkqREWc6UGu3tkV11zS+Wrm3VjWzvfN62z/uaptZ/OHZQYS4jBxYwprSQ4yeUMrqzeI0ZXMjY0kLK\nivN54IH7mT//pAT+ySRJkqRXZzlTxjW3trOrromK6iZ21DSxo7bzuaaJippGdtQ0sb2643VL28vP\nNczOCpQV5zNyUAFTyoo5dcowRg4qYOTAAkYMLHjxdWGes16SJEnq3SxnOiQxRvY0tHQWrJfK1t7i\ntbd07ahponK/lQ33Gjwgl+El+ZSV5HPCpCEvK12jBnUUr2HF+V7rJUmSpH7Bciag45TC6sYWdtY2\ns7uumd11Teyqa2ZX5/tde7fVdryurGt+2WqGe+XnZFFWks/wknwmDSvihElDKCsueHFbWedjaHEe\n+TnOdkmSJEl7Wc76oBgjDS1tVNW3dD6aqWpo6ShZtS8Vr73vd9U1U1nf/LIbJ++rJD+HocV5DCnK\nY2zpAOaOHfzi++EDCygrzmf4wI7SVZKfQwjOdEmSJEkHy3KWcg3NbVQ1NFNV30JlfTN76luoatjn\ndef2qobOEtb5eXNr+wG/c2BBDkOL8xlSlMf4oQOYN2EwQ4ryGFKUz9CivBeL19CifEqLcp3hkiRJ\nko4Ay1mGNbe2U9PYQnVja8dzQyvVjS1UN7RQ0/jS6+rG1he37WloebGQNb1CycrLyaJ0QC6lA/IY\nVJjLpGFFHa8H5DK4MI/SAbkMHpDLoMI8Sos69isdkEdeTtYR/DcgSZIkqTssZwew99TA2qZW6pra\nqG1s7Xzd8Vzb1PoP5aqmi6LV0NL2in9OVoCSglwGFuYwsCCXgQW5TBg6gGMGDO4oVp3la3DhPq87\ny5crFEqSJEl9R58sZ20RVlXUUNvURl1niapraqWueZ/XTa3UNrVR29TSUb6aXl6+6ppaOcAlWC+T\nmx06SlVhLgMLchhYmMvIQQUv2/ay8lWYS0nBS6+L8rK9RkuSJElS3yxnG2vaOeOaBw74eXZWoDg/\n58VHUX42JQU5jB5cQFFeDkX5OZQUdDwX5edQnJ9NcX4uRfnZLztuYGEu+TlZlitJkiRJh61PlrOh\nBYHv/tMxHQVrv7JVnJ9joZIkSZKUOhktZyGEccB/AWcCAbgbuDzGuKEbxxYAXwPeAwwGFgOfizEe\neEqsU0le4K3HjDmc6JIkSZJ0RGVs2b4QwgDgXmA68D7gvcBU4L4QQlE3vuLHwCXAvwPnAluBu0II\nx2QmsSRJkiQlJ5MzZ5cAk4HyGOMqgBDCM8BK4CPANQc6MIQwF3g38MEY4087t90PLAW+CpyXwdyS\nJEmSdMRl8oZX5wGP7S1mADHGtcDDwFu7cWwL8Nt9jm0FfgOcFULI7/m4kiRJkpScTJazWcBzXWxf\nCszsxrFrY4z1XRybB0w5/HiSJEmSlB6ZPK1xCFDZxfbdQOlhHLv385cJIVwKXApQVlbGwoULux1U\n/UNtba3jQl1ybKgrjgsdiGNDXXFcqCf0maX0Y4w3ADcAlJeXx/nz5ycbSKmzcOFCHBfqimNDXXFc\n6EAcG+qK40I9IZOnNVbS9QzZgWbFunssvDSDJkmSJEl9QibL2VI6rh3b30zg+W4cO6lzOf79j20G\nVv3jIZIkSZLUe2WynN0GnBRCmLx3QwhhInBK52ev5HYgF7hwn2NzgHcCf40xNvV0WEmSJElKUibL\n2Q+BdcCtIYS3hhDOA24FNgLX790phDAhhNAaQvj3vdtijIvoWEb/OyGED4cQTqdjGf1JwFcymFmS\nJEmSEpGxchZjrAMWAC8AvwB+BawFFsQYa/fZNQDZXWT5APBT4Crgz8A44OwY49OZyixJkiRJScno\nao0xxg3ABa+yzzo6Ctr+2xuAT3c+JEmSJKlPy+RpjZIkSZKkbrKcSZIkSVIKWM4kSZIkKQUsZ5Ik\nSZKUApYzSZIkSUoBy5kkSZIkpYDlTJIkSZJSwHImSZIkSSlgOZMkSZKkFLCcSZIkSVIKWM4kSZIk\nKQUsZ5IkSZKUApYzSZIkSUoBy5kkSZIkpYDlTJIkSZJSwHImSZIkSSlgOZMkSZKkFLCcSZIkSVIK\nWM4kSZIkKQUsZ5IkSZKUApYzSZIkSUoBy5kkSZIkpYDlTJIkSZJSwHImSZIkSSlgOZMkSZKkFLCc\nSZIkSVIKWM4kSZIkKQUsZ5IkSZKUApYzSZIkSUoBy5kkSZIkpYDlTJIkSZJSwHImSZIkSSlgOZMk\nSZKkFLCcSZIkSVIKWM4kSZIkKQUsZ5IkSZKUApYzSZIkSUoBy5kkSZIkpYDlTJIkSZJSwHImSZIk\nSSlgOZMkSZKkFLCcSZIkSVIKWM4kSZIkKQUsZ5IkSZKUApYzSZIkSUoBy5kkSZIkpYDlTJIkSZJS\nwHImSZIkSSlgOZMkSZKkFLCcSZIkSVIKWM4kSZIkKQUsZ5IkSZKUApYzSZIkSUoBy5kkSZIkpUDG\nylkIISuE8IUQwroQQmMIYUkI4YJuHvuzEELs4vGdTOWVJEmSpCTlZPC7vwZ8BvgS8BTwT8DvQgjn\nxhj/0o3jdwDn7bdta89GlCRJkqR0yEg5CyEMp6OY/UeM8Vudm+8LIUwB/gPoTjlrjjE+lol8kiRJ\nkpQ2mTqt8SwgD/jlftt/CRwdQpiUoT9XkiRJknqlTJWzWUATsGq/7Us7n2d24zuGhxB2hhBaQwgv\nhBA+F0LI7tGUkiRJkpQSmbrmbAhQFWOM+23fvc/nr2QxHdepLQUKgLcD3wCmAh/u6oAQwqXApQBl\nZWUsXLjwkIKr76qtrXVcqEuODXXFcaEDcWyoK44L9YRulbMQwhnA37qx6/0xxvmHlQiIMe6/KuNf\nQgi1wOUhhKtjjCu7OOYG4AaA8vLyOH/+YcdQH7Nw4UIcF+qKY0NdcVzoQBwb6orjQj2huzNnjwAz\nurFffedzJTA4hBD2mz3bO2O2m4P3a+By4HjgH8qZJEmSJPVm3SpnMcZ6YPlBfO9SIB84ipdfd7b3\nWrPnD+K7/iHOYRwrSZIkSamUqQVB7gRagIv32/4e4LkY49pD+M6L6Shmfz/MbJIkSZKUOhlZECTG\nWBFCuAb4QgihBngaeCewgP1uLB1CuAeYEGOc0vl+AvAL4Dd0zLrl07EgyPuB62OMqzORWZIkSZKS\nlKnVGgG+BNQCnwRGAiuAi2KMd+y3X/Z+OWrouCbtc8AIoJ2OUyo/AXw/g3klSZIkKTEZK2cxxjbg\nqs7HK+03f7/3u4G3ZSqXJEmSJKVRpq45kyRJkiQdBMuZJEmSJKWA5UySJEmSUsByJkmSJEkpYDmT\nJEmSpBSwnEmSJElSCljOJEmSJCkFLGeSJEmSlAKWM0mSJElKAcuZJEmSJKWA5UySJEmSUsByJkmS\nJEkpYDmTJEmSpBSwnEmSJElSCljOJEmSJCkFLGeSJEmSlAKWM0mSJElKAcuZJEmSJKWA5UySJEmS\nUsByJkmSJEkpYDmTJEmSpBSwnEmSJElSCljOJEmSJCkFLGeSJEmSlAKWM0mSJElKAcuZJEmSJKWA\n5UySJEmSUsByJkmSJEkpYDmTJEmSpBSwnEmSJElSCljOJEmSJCkFLGeSJEmSlAKWM0mSJElKAcuZ\nJEmSJKWA5UySJEmSUsByJkmSJEkpYDmTJEmSpBSwnEmSJElSCljOJEmSJCkFLGeSJEmSlAKWM0mS\nJElKAcuZJEmSJKWA5UySJEmSUsByJkmSJEkpYDmTJEmSpBSwnEmSJElSCljOJEmSJCkFLGeSJEmS\nlAKWM0mSJElKAcuZJEmSJKWA5UySJEmSUsByJkmSJEkpYDmTJEmSpBSwnEmSJElSCmSsnIUQPh1C\nuD2EsDWEEEMIVxzk8aeGEB4JITSEELaFEK4JIRRmKK4kSZIkJSqTM2eXAMOBPx3sgSGEOcDfgArg\nXODLwAeAn/VgPkmSJElKjZwMfvesGGN7CCEH+OhBHnslsAm4MMbYAhBCaAZuDCFcHWN8uoezSpIk\nSVKiMjZzFmNsP5TjQgi5wNnAzXuLWaebgWbgrT0QT5IkSZJSJY0LghwFFADP7bsxxtgIrAZmJhFK\nkiRJkjIpjeVsSOdzZRef7d7nc0mSJEnqM7p1zVkI4Qw6Fuh4NffHGOcfVqJDFEK4FLgUoKysjIUL\nFyYRQylWW1vruFCXHBvqiuNCB+LYUFccF+oJ3V0Q5BFgRjf2qz+MLHvtnTEr7eKzIcDSrg6KMd4A\n3ABQXl4e58+f3wNR1JcsXLgQx4W64thQVxwXOhDHhrriuFBP6FY5izHWA8sznGWv1UATMGvfjSGE\nAmAy8LsjlEOSJEmSjpjUXXMWY2wG7gQu6lyGf693APnAbYkEkyRJkqQMyth9zkIIxwMTeakAzgwh\nvKPz9V86Z+MIIfwYeF+Mcd8sVwCPATeHEL7X+T3fBG6JMT6VqcySJEmSlJRM3oT648D79nl/YecD\nYBKwrvN1dufjRTHGxSGENwJXA38G9gA/B76YwbySJEmSlJiMlbMY4/uB9x/qfjHGB4CTeziWJEmS\nJKVS6q45kyRJkqT+yHImSZIkSSlgOZMkSZKkFLCcSZIkSVIKWM4kSZIkKQUsZ5IkSZKUApYzSZIk\nSUoBy5kkSZIkpYDlTJIkSZJSwHImSZIkSSlgOZMkSZKkFLCcSZIkSVIKWM4kSZIkKQUsZ5IkSZKU\nApYzSZIkSUoBy5kkSZIkpYDlTJIkSZJSwHImSZIkSSlgOZMkSZKkFLCcSZIkSVIKWM4kSZIkKQUs\nZ5IkSZKUApYzSZIkSUoBy5kkSZIkpYDlTJIkSZJSwHImSZIkSSlgOZMkSZKkFLCcSZIkSVIKWM4k\nSZIkKQUsZ5IkSZKUApYzSZIkSUoBy5kkSZIkpYDlTJIkSZJSwHImSZIkSSlgOZMkSZKkFLCcSZIk\nSVIKWM4kSZIkKQUsZ5IkSZKUApYzSZIkSUoBy5kkSZIkpYDlTJIkSZJSwHImSZIkSSlgOZMkSZKk\nFLCcSZIkSVIKWM4kSZIkKQUsZ5IkSZKUApYzSZIkSUoBy5kkSZIkpYDlTJIkSZJSwHImSZIkSSlg\nOZMkSZKkFLCcSZIkSVIKWM4kSZIkKQUsZ5IkSZKUAhkrZyGET4cQbg8hbA0hxBDCFQdx7BWdx+z/\n+FOm8kqSJElSknIy+N2XANXAn4CPHuJ3nAq07fN+9+GGkiRJkqQ0ymQ5mxVjbA8h5HDo5ezxGGNr\nT4aSJEmSpDTK2GmNMcb2TH23JEmSJPU1aV8QZGMIoS2EsD6EcHUIoTDpQJIkSZKUCZk8rfFwrAI+\nDywCIvBG4FPAPODMBHNJkiRJUkZ0q5yFEM4A/taNXe+PMc4/rERAjPGX+236WwhhE/CdEMIZMca7\nu8h4KXBp59umEMJzh5tDfc4wYGfSIZRKjg11xXGhA3FsqCuOC3Wl/GB27u7M2SPAjG7sV38wf/hB\n+jXwHeA1wD+UsxjjDcANACGEJ2OMx2cwi3ohx4UOxLGhrjgudCCODXXFcaGuhBCePJj9u1XOYoz1\nwPJDStTzYtIBJEmSJKmnpX1BkH1d3Pn8RKIpJEmSJCkDMrYgSAjheGAiLxXAmSGEd3S+/kvnbBwh\nhB8D74sx5uxz7CLg58AKOmbKzgQuA+6MMd7bjT/+hh75h1Bf47jQgTg21BXHhQ7EsaGuOC7UlYMa\nFyHGzJwlGEL4GfC+A3w8Kca4bt/9Yoxhn2N/Q8e1ZaPoKHdr6Ljm7D9jjE0ZCSxJkiRJCcpYOZMk\nSZIkdV9vuubsoIUQSkIIN4cQVoUQ6kIIVSGEJ0II70k6m5ITQpgWQvhuCOGZEEJtCGFrCOG2EMLc\npLMpeSGET4cQbu8cFzGEcEXSmXTkhBDGhRBuCSHsCSFUhxD+EEIYn3QuJSuEMDaEcG0I4dEQQn3n\nz4aJSedSskII7wgh/D6EsD6E0BBCWBFC+EYIoSTpbEpWCOGsEMK9IYRtIYSmEMKmzk4y89WO7dPl\nDMgDWoFvAOcB7waWAb8IIXwqyWBK1BuBNwA3Am8B/g9QBjwWQjguyWBKhUuA4cCfkg6iIyuEMAC4\nF5hOx2n57wWmAveFEIqSzKbETQEuAiqBBxPOovT4DNAGfBE4G/gB8C903J+3r/+OrVc2BHgK+Dgd\nv3d+AZhFx++aE17pwH55WmMI4VGgOMZ4dNJZdOSFEIYBu+I+gz+EMAhYB9weY/znpLIpeSGErBhj\newghB2gBrowxXpFwLB0BIYRPAtcA5THGVZ3bJgErgc/GGK9JMp+Ss/fnQufrDwM/ZJ/r59U/hRDK\nYow79tv2z3T8z9/Tu7mInfqJEEI5Hbcm+0yM8dsH2q+/tvpddMyoqR+KMe6M+/1fiRjjHuAFYEwy\nqZQWe38BU790HvDY3mIGEGNcCzwMvDWxVEqcPxfUlf2LWae/dz77+4T2t6vz+RU7SL8oZ6FDTghh\naAjhUuAs4L+SzqX0CCEMAWbTcdqrpP5pFvBcF9uXAq96nYAkAad1Pvv7hAghZIcQ8kIIU4HrgW10\nrEB/QBm7z1nKfAy4tvN1C/DJGOPPE8yj9LkWCMB3kg4iKTFD6LimaH+7gdIjnEVSLxNCGAN8Fbg7\nxvhk0nmUCo8De9czWAUsiDFWvNIBvWrmLIRwRucKSa/2WLjfob+l475pbwJ+BFwbQvjIkc6vzDiM\ncbH3+C/QsVjMx/c9nUm93+GODUmSuiOEUAzcSscpax9IOI7S473ASXT8nllNx2IxE1/pgN42c/YI\nMKMb+9Xv+6bznOC95wXf2bki17dCCD+JMbb0cEYdeYc0LgBCCB8Fvg58Ocb4k54OpsQd8thQv1RJ\n1zNkB5pRkyRCCIXA7cBk4LQY46aEIyklYox7T299PITwv3QsPvd54KMHOqZXlbMYYz0dq5wcrifp\nWCZ5BOB/QL3coY6LEMJ7ge8D344x/r8eD6bE9eDPDPUPS+m47mx/M4Hnj3AWSb1ACCEXuAU4Hjgz\nxvhswpGUUjHGqhDCKjpuzXFAveq0xh50GlALvOI5n+q7QghvB34K/CjG+Jmk80hKhduAk0IIk/du\n6Dz95JTOzyTpRZ33MvsVsAB4W4zxsYQjKcVCCCPouI/m6lfar1fNnB2szuvKTgLupmOGbCgdN5F8\nB/D5GGNzgvGUkBDC6+lYKWcJ8LMQwv9v745Vq4iCMAD/0/gYJiCksBNSiJWNhb15D61sAomIWFpZ\n2fgGYmG6gHkBBZ9A0S5pLazG4kQM4sXu7vHe76sWTjOwsLv/7s6Z21eWf3T3x2UqYwZVtZ9kN79f\nXt2sqgeXxyeXX+PYTK8yBoa+rarDJJ3kaZKvGbtsscWuXAd+Nfffr6rzJOfdfbZQWSzrZZKDJM+S\nfP/jeeKb3xu3V1W9SfIhyaeMXrO9JI8yehJXzjhLNnwIdVXdSXKY5FZGz8BFxtamL7r73ZK1sZyq\nOk5ytGL5S3fvrq8aZlNVrzN+e/4bQ2c3XFVdzxi1ci9jB9fTJA+dd6pq1QPTWXffXWctzKGqPifZ\nWbH8pLuP11cNM6mqxxkfhG4kuZbxku99kuf/up9sdDgDAAD4X2xrzxkAAMBUhDMAAIAJCGcAAAAT\nEM4AAAAmIJwBAABMQDgDAACYgHAGAAAwAeEMAABgAsIZAADABH4C0NGY2YJ9xskAAAAASUVORK5C\nYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f87f1348e50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(lambda x: math.log(1 + math.exp(x)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Problem zanikającego gradientu (_vanishing gradient problem_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "* Sigmoidalne funkcje aktywacji ograniczają wartości na wyjściach neuronów do niewielkich przedziałów ($(-1, 1)$, $(0, 1)$ itp.).\n",
    "* Jeżeli sieć ma wiele warstw, to podczas propagacji wstecznej mnożymy przez siebie wiele małych wartości → obliczony gradient jest mały.\n",
    "* Im więcej warstw, tym silniejszy efekt zanikania."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### Sposoby na zanikający gradient\n",
    "\n",
    "* Modyfikacja algorytmu optymalizacji (_RProp_, _RMSProp_)\n",
    "* Użycie innej funckji aktywacji (ReLU, softplus)\n",
    "* Dodanie warstw _dropout_\n",
    "* Nowe architektury (LSTM itp.)\n",
    "* Więcej danych, zwiększenie mocy obliczeniowej"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## 5.7. Wielowarstwowe sieci neuronowe w&nbsp;praktyce"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Przykład: MNIST\n",
    "\n",
    "_Modified National Institute of Standards and Technology database_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "* Zbiór cyfr zapisanych pismem odręcznym\n",
    "* 60 000 przykładów uczących, 10 000 przykładów testowych\n",
    "* Rozdzielczość każdego przykładu: 28 × 28 = 784 piksele"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "# źródło: https://github.com/keras-team/keras/examples/minst_mlp.py\n",
    "\n",
    "import keras\n",
    "from keras.datasets import mnist\n",
    "\n",
    "# załaduj dane i podziel je na zbiory uczący i testowy\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "slideshow": {
     "slide_type": "notes"
    }
   },
   "outputs": [],
   "source": [
    "def draw_examples(examples, captions=None):\n",
    "    plt.figure(figsize=(16, 4))\n",
    "    m = len(examples)\n",
    "    for i, example in enumerate(examples):\n",
    "        plt.subplot(100 + m * 10 + i + 1)\n",
    "        plt.imshow(example, cmap=plt.get_cmap('gray'))\n",
    "    plt.show()\n",
    "    if captions is not None:\n",
    "        print(6 * ' ' + (10 * ' ').join(str(captions[i]) for i in range(m)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6oAAACVCAYAAABRuAf7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XmUVNXd7vFno4ggICLEGA2Coigy\nKeAUl6DiFBFBFEMYlEThakTNG7kkhhgcEMVhvYDixBUjsgJGZDIaRUV4nVig0RtQDBoEUSIoMxi5\n6r5/dFd3/bbd1VXdp6pOVX0/a/XqeurUsLt4qNOnq3Zt570XAAAAAABxUS/fAwAAAAAAIBkHqgAA\nAACAWOFAFQAAAAAQKxyoAgAAAABihQNVAAAAAECscKAKAAAAAIiVSA9UnXM/ds495Zzb5pzb7px7\n2jnXKsr7QGGgC0igC5DoASrRBSTQBUj0ANVzUa2j6pxrJOldSV9LGiPJS7pNUiNJnbz3uyK5I8Qe\nXUACXYBED1CJLiCBLkCiB0ht7whv60pJh0tq573/UJKcc/9X0mpJIyTdG+F9Id7oAhLoAiR6gEp0\nAQl0ARI9QApRvqL6kqR9vfc/Cc5fLEne+x6R3BFijy4ggS5AogeoRBeQQBcg0QOkFuUrqsdKmlfF\n+SslXZLODTjnojlqRs54710VZ9epC/SgIH3hvW9Zxfl0ofRU1QX2DyWI/QPKsX9AQuT7B3pQkKp7\nTvieKD9MqbmkLVWcv1nSARHeD+KPLpSetdWcTxdKT1VdoAdIoAulh/0DEtg/QKr+OeF7onxFtVac\nc8MlDc/3OJBf9AAJdAEJdAESPUAlugCJHpSSKA9Ut6jqv3xU95cSSZL3/mFJD0u8fF9EMu4CPSha\ndAES+wdU4jkBCXQBEj1AClG+9Xelyt5nHmov6b0I7wfxRxeQQBcg0QNUogtIoAuQ6AFSiPJAdb6k\nk5xzhyfOcM61lvST8m0oHXQBCXQBEj1AJbqABLoAiR4ghSiXp9lPZQv2fqXKBXtvldREZQv27kzj\nNnj5vsBU9amOde0CPShIb3nvu4Vn0oWS9L0usH8oTewfUI79AxIi3z/Qg4JU5XNCVSJ7RdV7v0vS\nGZL+KWm6pBmS1kg6I51fQlA86AIS6AIkeoBKdAEJdAESPUBqkb2iGgX+KlJ4qlknr07oQUFK+69j\nmaALBYkuQBL7B1TgOQEJkXeBHhSk3L+iCgAAAABAFDhQBQAAAADECgeqAAAAAIBY4UAVAAAAABAr\nHKgCAAAAAGKFA1UAAAAAQKxwoAoAAAAAiJW98z0AoFR17drV5GuuucbkoUOHmvz444+bPHnyZJPf\nfvvtCEcHAACAbJk4caLJ1157bcXpFStWmG29e/c2ee3atdkbWIzwiioAAAAAIFY4UAUAAAAAxApv\n/Y3IXnvtZfL++++f9nXDt3w2atTI5Hbt2pn8q1/9yuS7777b5IEDB5r8n//8x+Q77rij4vTNN9+c\n9jhRN126dDF54cKFJjdt2tRk773JQ4YMMblPnz4mH3jggXUdIorEmWeeafKMGTNM7tGjh8kffPBB\n1seE7BgzZozJ4XN6vXr279E9e/Y0efHixVkZF4BoNGnSxOTGjRubfP7555vcsmVLk++9916Tv/76\n6whHh0y0bt3a5MGDB5v83XffVZw+5phjzLajjz7aZN76CwAAAABAHnCgCgAAAACIFQ5UAQAAAACx\nwhzVcq1atTJ5n332MfmUU04x+dRTTzW5WbNmJvfv3z+ysa1fv97kSZMmmdyvXz+Td+zYYfK7775r\nMnOScueEE06oOD179myzLZzHHM5JDf8d9+zZY3I4J/Wkk04yOVyuJrx+KTjttNMqToeP15w5c3I9\nnJzp3r27ycuWLcvTSBC1yy+/3OTRo0ebnDzHqSrh8wyA/Eueuxj+nz755JNN7tChQ0a3ffDBB5uc\nvAQKcmvTpk0mL1myxOTws0fAK6oAAAAAgJjhQBUAAAAAECscqAIAAAAAYqVk56iGa1q+/PLLJmey\nDmrUwjlG4Tp5O3fuNDlcI3HDhg0mb9myxWTWTIxOuObt8ccfb/ITTzxRcTqcJ1KT1atXmzxhwgST\nZ86cafJrr71mctib8ePHZ3T/xSB5zcgjjzzSbCumOarhWplt2rQx+bDDDjPZOZf1MSE7wn/Lfffd\nN08jQaZOPPFEk5PXUAzXNj722GNT3tYNN9xg8meffWZy+DkayfsiSVq6dGnqwSJS4RqY119/vcmD\nBg2qON2wYUOzLXy+/uSTT0wOP88iXH9zwIABJk+ZMsXkVatWVTdsRGzXrl0ml8paqHXBK6oAAAAA\ngFjhQBUAAAAAECscqAIAAAAAYqVk56iuW7fO5C+//NLkKOeohnNBtm7davLpp59ucrje5fTp0yMb\nC6L10EMPmTxw4MDIbjuc79q4cWOTw/Vwk+djSlKnTp0iG0uhGjp0aMXpN954I48jya5w/vOVV15p\ncjg/jTlJhaNXr14mjxw5MuXlw3/b3r17m/z5559HMzDU6NJLLzV54sSJJrdo0aLidDgP8ZVXXjG5\nZcuWJt91110p7zu8vfD6P/vZz1JeH5kJf2e88847TQ670KRJk7RvO/y8inPOOcfk+vXrmxw+ByT3\nrKqM3GnWrJnJnTt3ztNICgevqAIAAAAAYoUDVQAAAABArHCgCgAAAACIlbTmqDrnDpU0WlI3SZ0l\nNZTUxnv/cXC5fSXdKmmwpGaS3pE02nu/JMIxR2Lz5s0mjxo1yuRwXs/f//53kydNmpTy9t95552K\n02eddZbZFq6jFK6Xdt1116W87ThxznkVeBcy0bVrV5PPP/98k1OtTxnOKV2wYIHJd999t8nhunhh\nB8P1cc8444y0x5IFXZ1zrePWg3B90WI1derUlNvDOU5ZFssuFIpw/ctp06aZXNPnJ4RzF/O5Tl+x\n7x/23tv+CtWtWzeTH3nkEZPDdbeXLKn8cW+99Vaz7dVXXzW5QYMGJj/55JMmn3322SnHunz58pTb\ns6zonxP69etn8hVXXFHr2/roo49MDn+HDNdRbdu2ba3vKw+KvguphM8BrVq1Svu63bt3Nzmci1ys\na7Km+1tcW0kDJG2R9D8pLvd/JF0p6SZJvSVtkPS8c65LXQaJgkQXINEDVKILSKALkOgBKtEFVCnd\nT/1d4r0/SJKcc1dI+t6f7pxznSX9XNIvvPfTys9bLGmlpFsk9YlkxIg9ugCJHqASXUACXYBED1CJ\nLiCVtF5R9d5/l8bF+kj6f5JmJV3vG0kzJZ3jnGtQ3RVRdOgCJHqASnQBCXQBEj1AJbqAakW5juqx\nktZ473cH56+UtI/K3j68MsL7i9TcuXNNfvnll03esWOHyeHaR7/85S9NTp5vGM5JDa1caR+W4cOH\npx5s/BV0F5J16WLfdbJw4UKTmzZtarL33uTnnnuu4nS4xmqPHj1MHjNmjMnhvMNNmzaZ/O6775r8\n3Xf270nh/NlwXda3335bWZbzHoRrxx500EFR3nxs1TRvMextHhTNc0K2XXbZZSb/6Ec/Snn5cL3N\nxx9/POohRa1oujB48GCTa5orHv4/TF5bc/v27SmvG67DWdOc1PXr15v8pz/9KeXl86BoeiBJl1xy\nSUaX//jjj01etmxZxenRo0ebbeGc1NAxxxyT0X3HUFF1IZXws0cee+wxk8eOHVvtdcNtW7duNfm+\n++6ry9BiK8pPGmmusjmsoc1J21Ea6AIkeoBKdAEJdAESPUAluoBqRfmKaq0454ZLKviXEFE39AAJ\ndAEJdAESPUAlugCJHpSSKF9R3SLpgCrOT/wlZHMV2+S9f9h73817362q7ShIGXeBHhQlnhOQQBeQ\nwP4BEs8JqMRzAqoV5SuqKyX1c841Ct5n3l7SHkkfRnhfWVfTfJFt27al3H7llVdWnJ41a5bZFs4l\nLEIF24WjjjrK5HB93XAu4BdffGHyhg0bTE6eF7Rz506z7a9//WvKXFcNGzY0+Te/+Y3JgwYNivT+\nqpDzHvz0pz81OXwMikU497ZNmzYpL//pp59mczjpKNjnhGxr0aKFyb/4xS9MDvcX4byk2267LTsD\ny56C7UK41umNN95ocvgZBVOmTDE5/ByCmn7PSPb73/8+7ctK0rXXXmty+BkHMVCwPahK8u980vc/\na+SFF14w+cMP7Y+3cePGWt93EXwWQ1F1IRPhc0qqOaqlKspXVBdIqi+pYka5c25vSZdKesF7/3WE\n94V4owuQ6AEq0QUk0AVI9ACV6AKqlfYrqs65i8tPdi3/fp5zbpOkTd77xd77vzvnZkn6b+dcfUlr\nJF0lqY2krL90g7yiC5DoASrRBSTQBUj0AJXoAtKWyVt//xLkxHtaFkvqWX56mKRxkm6T1EzSu5LO\n9d5nfR0M5BVdgEQPUIkuIIEuQKIHqEQXkLa0D1S99y6Ny3wl6b/Kv4pa+D7yrl27mpy8RmavXr3M\ntnCuQiGrrheF0oUGDew60snr30rfn/MYrqc7dOhQk5cvX25ynOZItmrVKps3/1ZVH2qQ6x60a9eu\n2m3hesWFLOxpOEfpn//8p8lhb7MsFl2Is9atW1ecnj17dkbXnTx5ssmLFi2KYkhZUej7h5tuusnk\ncE7qnj17TH7++edNDtfD/Oqrr6q9r3333dfkcJ3U8PnbOfvQhnOV582bV+195UHRPyeE62Pmcq7h\nySefnLP7ikDRd6Eu6tWrnJFZAp9nk5Yo56gCAAAAAFBnHKgCAAAAAGKFA1UAAAAAQKxEuY5qSdm1\na5fJ4Rpab79dOf/7kUceMdvCOUXhvMb777/f5HBtNkTnuOOOMzmckxq68MILTV68eHHkY0J2LFu2\nLN9DqFbTpk1NPvfcc00ePHiwyeH8tVC4Nlu49ibyK/nft1OnTikv+9JLL5k8ceLErIwJUrNmzUy+\n+uqrTQ73xeGc1L59+2Z0f23btq04PWPGDLMt/NyL0FNPPWXyhAkTMrpvxEvyurf77bdfRtft2LFj\nyu2vv/66yW+88UZGt4/cSZ6Xyu/+ZXhFFQAAAAAQKxyoAgAAAABihbf+RuSjjz4y+fLLL684PW3a\nNLNtyJAhKXP4to/HH3/c5A0bNtR2mAjce++9Jocf+R++tTfOb/VN/lhziY82DzVv3rxO1+/cubPJ\nYVfCZagOPfRQk/fZZ5+K04MG2TXMw3+7cBmLpUuXmvz111+bvPfe9qn8rbfeEuIjfEvoHXfcUe1l\nX331VZMvu+wyk7dt2xbdwGAk/x+VpBYtWqS8fPLbNSXpBz/4gcnDhg0zuU+fPiZ36NCh4nTjxo3N\ntvBtf2F+4oknTA6nIyG/GjVqZHL79u1N/uMf/2hyqmlHme7bw6Vywh5+++23Ka8PxAmvqAIAAAAA\nYoUDVQAAAABArHCgCgAAAACIFeaoZsmcOXMqTq9evdpsC+dFnnnmmSbffvvtJh922GEmjxs3zuRP\nP/201uMsNb179za5S5cuJofzgObPn5/1MUUlnLcS/izvvPNOLoeTF+HczuTH4MEHHzTbbrzxxoxu\nO1xGJJyj+s0335i8e/duk997772K048++qjZFi5RFc6F/vzzz01ev369yQ0bNjR51apVQv60bt3a\n5NmzZ6d93X/9618mh//2yJ49e/aYvGnTJpNbtmxp8po1a0zOdDmJ5LmE27dvN9sOPvhgk7/44guT\nFyxYkNF9IVr169c3OVzqLvw/H/57hvuq5C6Ey8eEy5WF819D4WcWXHTRRSaHS1yFvQfihFdUAQAA\nAACxwoEqAAAAACBWOFAFAAAAAMQKc1RzYMWKFSYPGDDA5AsuuMDkcN3VESNGmHzkkUeafNZZZ9V1\niCUjnMsXrpu3ceNGk2fNmpX1MaWrQYMGJo8dOzbl5V9++WWTf/e730U9pNi5+uqrTV67dm3F6VNO\nOaVOt71u3TqT586da/L7779v8ptvvlmn+0s2fPhwk8O5cuG8RuTX6NGjTc5kTeNUa6wiu7Zu3Wpy\nuP7tM888Y3K4NnO4nvq8efNMfuyxx0zevHlzxemZM2eabeGcxnA7civ8XSGcN/r000+nvP7NN99s\ncrh/fu211ypOh70KL5u8/m5Vwv3D+PHjTa5pXxau043cSV4zt6b9xmmnnWbyfffdl5Ux5RuvqAIA\nAAAAYoUDVQAAAABArHCgCgAAAACIFeao5kE4D2b69OkmT5061eRwTazwfek9e/Y0+ZVXXqnbAEtY\nODdjw4YNeRrJ9+ekjhkzxuRRo0aZHK6tec8995i8c+fOCEdXGO688858DyES4VrLoUzW6UT0wvWY\nzz777LSvG85j/OCDDyIZE+pu6dKlJodz/+oqeV/eo0cPsy2cn8Y89NwK10kN55iG+9/Qc889Z/Lk\nyZNNDn8PTO7Ws88+a7Z17NjR5HDd0wkTJpgczmG98MILTZ4xY4bJL774osnhfnPLli2qTimsz55L\nyf/va1qXOVwft3379iYnr91eyHhFFQAAAAAQKxyoAgAAAABihQNVAAAAAECsMEc1Bzp16mTyxRdf\nbHL37t1NDuekhsL3nS9ZsqQOo0Oy+fPn5+2+w3lu4RyYSy+91ORwblv//v2zMzDE3pw5c/I9hJL2\nwgsvmHzAAQekvHzyGruXX355NoaEApC8rnc4JzWcn8Y6qtm11157mXzrrbeafMMNN5i8a9cuk3/7\n29+aHP57hXNSu3XrZnLyGpjHHXec2bZ69WqTr7rqKpMXLVpkctOmTU0O1xAfNGiQyX369DF54cKF\nqs4nn3xicps2baq9LDL34IMPVpweMWJERtcN11u//vrrIxlTvvGKKgAAAAAgVjhQBQAAAADECgeq\nAAAAAIBYqXGOqnPuYkkDJXWT9ANJ6yQ9Lel27/2OpMsdIOkuSX0lNZT0hqRfe+//kYVxx067du1M\nvuaaaypOh2sd/fCHP8zotr/99luTw7U9w7ktueScG68C6oJzLmXu27evydddd13WxvLrX//a5D/8\n4Q8m77///iaHa58NHTo0OwOrnQ6F1gVkzSHOuSal1IMDDzzQ5Jqek6dMmVJxupjXN+Y5IbXnn38+\n30PIldjvH8L5feGc1N27d5sczh8M56mfdNJJJg8bNszk8847z+Tk+cq33HKL2TZt2jSTw3mioe3b\nt5v8t7/9LWUeOHCgyT//+c+rve3w95ZaKLn9QyZWrVqV7yHETjqvqN4g6VtJN0o6V9IDkq6StNA5\nV0+SXNlv+wvKt4+U1F9SfUmLnHOHZmHciA+6AEnaKLqAMi1FD1CG5wRI7B9Qif0DMpLOgeoF3vsB\n3vsZ3vvF3vv/lnStpBMl9Sy/TB9JP5E0xHv/Z+/938rPqyfpf2dh3IgPugCp7BcRugBJ+kT0AGV4\nToDE/gGV2D8gIzUeqHrvN1Vx9rLy74eUf+8j6TPvfcVnZHvvt6nsryQX1nWQiDW6gAS6AElKrNtA\nD8BzAhLoAiT2D8hQbddR7VH+/f3y78dKWlHF5VZKGuqca+y9L+iJOOG80vA9/clzUiWpdevWtb6v\n5cuXmzxu3DiT87nWZxUKqgvh2nRhDv+dJ02aZPKjjz5q8pdffmlyOC9lyJAhFac7d+5sth16qH1X\ny7p160wO5y8lz2uLqYLqQiEL51YfddRRJiev05kHTcq/F20Pwjlj9epl9rmEr7/+epTDiTOeE1I4\n55xz8j2EXIp1F2666aaU28N1VsN1zseOHWty27ZtM7r/5OuPHz/ebAs/pyRqf/7zn1PmiBX9/qEu\nJk+eXHF65MiRZtsRRxyR8rrhZ6ok35YkffTRR3UcXX5k/Km/zrlDJN0i6UXvfeKIqrmkLVVcfHP5\n99Srn6OQ0QVIZXNK6AIk6UeiByjDcwIk9g+oxP4BGcnoFVXnXGNJ8yR9I2lYDRdP9zaHSxpe4wUR\nV5F0gR4UvLaStokuQPJi/4Ay7B8gsX9ApUj2D/SgdKR9oOqca6iy94wfLqmH93590uYtqvqvHs2T\ntlfJe/+wpIfL78NXdznE1jlRdIEeFLwGogsos5r9A8rxnACJ/QMqRbJ/oAelI60DVedcfUlPqWwt\n1bOqWNtopaSzq7hqe0nrCuH95QcddJDJ7du3N/m+++4z+eijj671fS1dutTku+66y+R58+aZnM91\nUmtSbF0I56FcffXVJvfv39/kcL2yI488Mu37CuepLVq0yOSa5szEzOpi60KchXOrM50jmWVfBbng\ne9ClSxeTe/XqZXL4HL1nzx6T77//fpM///zzCEcXXzwnpHb44Yfnewi5Evv9w7///W+TW7ZsaXKD\nBg1MDj9zIvTss8+avGTJEpPnzp1r8scff1xxOttzUvOs6PYP2bJy5UqTa3q+iPOxQl3U+NtN+VpH\nMySdIamv976qT+mYr7JFfHskXa+ppAvKt6F00IXStKuK8+gCJHqASnShNLF/QHXoAVJK5xXV+yVd\nImmcpF3OueSPNV1f/hL+fElvSHrCOTdKZS/V/06SkzQh2iEjTpL6QBdK2350AeUSXaAHJY7nBJRj\n/4AE9g/ISDrvFzuv/PvvVVam5K8rJMl7/52k3pIWSpoiaY6kbyWd7r3/JOIxI17oAiTpaNEFlEl0\ngR6A5wRI7B9Qif0DMuLCuU75lM0J0c2bNzf5oYceMjmcg1TXuSPJ8w/vuecesy1cH/Orr8K37BcO\n772r+VKZyWYPwrVL//KXv5jcvXv3lNcP16+s6f9P8jqrM2fONNvCNa8K3Fve+25R3ygfklBm1qxZ\nJl9yySUmP/LIIyaPGDEi62NKoei60LNnT5MXLlxocjhHeM2aNSZnuqZisSi0/UOudejQoeL0P/5h\np3CG883CNb43bdqUvYFFL/bPCU2aNDG5b9++Jh9//PEmb9y40eRwjfUtW+xnAIXz1ktY5F0opueE\nZOedd57JCxYsSHn5mtZXj9k6qmn3IFafwAEAAAAAAAeqAAAAAIBY4UAVAAAAABAraa2jWihOPPHE\nitOjRo0y20444QSTDznkkDrd1+7du02eNGmSybfffnvF6V27qvpkduTD+vXrTb7oootMDuf2jRkz\nJqPbnzhxoskPPPBAxekPP/wwo9sCqhPORQFQeFasWFFxevXq1WZb+DkZRxxxhMkFNkc19nbs2GHy\n9OnTU2Yg29577z2T33//fZOPOeaYXA4nb3hFFQAAAAAQKxyoAgAAAABipaje+tuvX78qT6cjfIn9\nmWeeMfmbb74xOVxyZuvWrRndH+Jhw4YNJo8dOzZlBvLhueeeMzlcngbZtWrVKpOTlx+TpFNPPTWX\nw0ERSp4uJElTp041edy4cSaPHDnS5PB3GACFbe3atSZ37NgxTyPJL15RBQAAAADECgeqAAAAAIBY\n4UAVAAAAABArznuf7zFUcM7FZzBIi/c+8nUy6EFBest73y3qG6ULBYkuQBL7h0w0bdrU5CeffNLk\nXr16mfz000+bPGzYMJNjtiwezwlIiLwL9KAgpd0DXlEFAAAAAMQKB6oAAAAAgFjhQBUAAAAAECtF\ntY4qAABAodm+fbvJAwYMMDlcR/Wqq64yOVzzm3VVARQDXlEFAAAAAMQKB6oAAAAAgFjhQBUAAAAA\nECuso4o6YZ08lGOdPCTQBUhi/4AKPCcggXVUIbGOKgAAAACgUHGgCgAAAACIFQ5UAQAAAACxErd1\nVL+QtFZSi/LTSF8+HrPDsnS79KBu6AISiq0Lu0QPaqPYesBzQu3RBSQUSxfoQd3Eugex+jClBOfc\n8mxMvC9mxfiYFePPlAvF+LgV48+UC8X2uBXbz5Mrxfi4FePPlAvF+LgV48+UC8X2uBXbz5MrcX/c\neOsvAAAAACBWOFAFAAAAAMRKXA9UH873AApQMT5mxfgz5UIxPm7F+DPlQrE9bsX28+RKMT5uxfgz\n5UIxPm7F+DPlQrE9bsX28+RKrB+3WM5RBQAAAACUrri+ogoAAAAAKFGxOFB1zv3YOfeUc26bc267\nc+5p51yrfI8rLpxzFzvnZjvn1jrnvnLOfeCcG++caxJc7gDn3FTn3BfOuV3OuRedcx3zNe7aoAup\n0QW6kFAqXaAHqZVKDyS6UBO6QBcSSqUL9CC1YuhB3t/665xrJOldSV9LGiPJS7pNUiNJnbz3u/I4\nvFhwzr0paZ2keZLWSzpO0lhJqySd4r3/zjnnJP2PpNaSRknaIul3ko6V1MV7vz73I88MXagZXaAL\nCaXQBXpQs1LogUQX0kEX6EJCKXSBHtSsKHrgvc/rl6TrJH0rqW3SeW0kfSPpv/I9vjh8SWpZxXlD\nVfaf8ozyfGF5Pj3pMvtL2ixpUr5/BrpAF+gCXaAH9IAu0AW6QBfoAT1I9ysOb/3tI+lN7/2HiTO8\n92skvaayB6/kee83VXH2svLvh5R/7yPpM+/9oqTrbZO0QIXzONKFGtAFupBQIl2gBzUokR5IdKFG\ndIEuJJRIF+hBDYqhB3E4UD1W0ooqzl8pqX2Ox1JIepR/f7/8e6rHsZVzrnFORlU3dKF26AISiq0L\n9KB2iq0HEl2oLbqAhGLrAj2onYLqQRwOVJur7P3Qoc2SDsjxWAqCc+4QSbdIetF7v7z87FSPo1QY\njyVdyBBdQEKRdoEeZKhIeyDRhYzRBSQUaRfoQYYKsQdxOFBFBsr/sjFPZe/BH5bn4SCP6AIS6AIk\neoBKdAEJdAFS4fZg73wPQGVH8VUdrVd3hF+ynHMNVfae8cMl9fD2k7hSPY6J7XFHF9JEF5BQ5F2g\nB2kq8h5IdCFtdAEJRd4FepCmQu5BHF5RXamy90eH2kt6L8djiS3nXH1JT0nqJumn3vt/BBdJ9Tiu\n897vzPIQo0AX0kAX6EJCCXSBHqShBHog0YW00AW6kFACXaAHaSj0HsThQHW+pJOcc4cnznDOtZb0\nk/JtJc85V0/SDElnSOrrvX+ziovNl3SIc65H0vWaSrpAhfM40oUa0AW6kFAiXaAHNSiRHkh0oUZ0\ngS4klEgX6EENiqEHrny9nPwNwLn9VLZg71eqXLD3VklNVLZgb9z/opN1zrkHJP0vSeMkPRNsXu+9\nX19exlcl/Vh2wd5Okjp77z/J4ZBrhS7UjC7QhYRS6AI9qFkp9ECiC+mgC3QhoRS6QA9qVhQ9yMfi\nreGXpFaSZkvaLmmHpLmSWufJ7oROAAAAlUlEQVR7XHH5kvSxyv4DVvU1NulyzSU9qrJP6tot6SWV\nlSzvPwNdoAt0gS7QA3pAF+gCXaAL9IAepPuV91dUAQAAAABIFoc5qgAAAAAAVOBAFQAAAAAQKxyo\nAgAAAABihQNVAAAAAECscKAKAAAAAIgVDlQBAAAAALHCgSoAAAAAIFY4UAUAAAAAxAoHqgAAAACA\nWPn/0DbqrzfbMFoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f87f132bed0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      5          0          4          1          9          2          1\n"
     ]
    }
   ],
   "source": [
    "draw_examples(x_train[:7], captions=y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60000 przykładów uczących\n",
      "10000 przykładów testowych\n"
     ]
    }
   ],
   "source": [
    "num_classes = 10\n",
    "\n",
    "x_train = x_train.reshape(60000, 784)  # 784 = 28 * 28\n",
    "x_test = x_test.reshape(10000, 784)\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "print('{} przykładów uczących'.format(x_train.shape[0]))\n",
    "print('{} przykładów testowych'.format(x_test.shape[0]))\n",
    "\n",
    "# przekonwertuj wektory klas na binarne macierze klas\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_4 (Dense)              (None, 512)               401920    \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 512)               262656    \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 10)                5130      \n",
      "=================================================================\n",
      "Total params: 669,706\n",
      "Trainable params: 669,706\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(512, activation='relu', input_shape=(784,)))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(512, activation='relu'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((60000, 784), (60000, 10))\n"
     ]
    }
   ],
   "source": [
    "print(x_train.shape, y_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/5\n",
      "60000/60000 [==============================] - 14s - loss: 0.2457 - acc: 0.9244 - val_loss: 0.1250 - val_acc: 0.9605\n",
      "Epoch 2/5\n",
      "60000/60000 [==============================] - 14s - loss: 0.1021 - acc: 0.9691 - val_loss: 0.0859 - val_acc: 0.9748\n",
      "Epoch 3/5\n",
      "60000/60000 [==============================] - 14s - loss: 0.0746 - acc: 0.9773 - val_loss: 0.0884 - val_acc: 0.9744\n",
      "Epoch 4/5\n",
      "60000/60000 [==============================] - 13s - loss: 0.0619 - acc: 0.9815 - val_loss: 0.0893 - val_acc: 0.9754\n",
      "Epoch 5/5\n",
      "60000/60000 [==============================] - 15s - loss: 0.0507 - acc: 0.9849 - val_loss: 0.0845 - val_acc: 0.9771\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f87e996a2d0>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer=RMSprop(), metrics=['accuracy'])\n",
    "\n",
    "model.fit(x_train, y_train, batch_size=128, epochs=5, verbose=1,\n",
    "          validation_data=(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.0845102945257\n",
      "Test accuracy: 0.9771\n"
     ]
    }
   ],
   "source": [
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "\n",
    "print('Test loss: {}'.format(score[0]))\n",
    "print('Test accuracy: {}'.format(score[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "Warstwa _dropout_ to metoda regularyzacji, służy zapobieganiu nadmiernemu dopasowaniu sieci. Polega na tym, że część węzłów sieci jest usuwana w sposób losowy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "slideshow": {
     "slide_type": "notes"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_7 (Dense)              (None, 512)               401920    \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 512)               262656    \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 10)                5130      \n",
      "=================================================================\n",
      "Total params: 669,706\n",
      "Trainable params: 669,706\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/5\n",
      "60000/60000 [==============================] - 11s - loss: 0.2214 - acc: 0.9314 - val_loss: 0.1048 - val_acc: 0.9668\n",
      "Epoch 2/5\n",
      "60000/60000 [==============================] - 12s - loss: 0.0838 - acc: 0.9739 - val_loss: 0.0842 - val_acc: 0.9752\n",
      "Epoch 3/5\n",
      "60000/60000 [==============================] - 10s - loss: 0.0548 - acc: 0.9829 - val_loss: 0.0806 - val_acc: 0.9773\n",
      "Epoch 4/5\n",
      "60000/60000 [==============================] - 9s - loss: 0.0387 - acc: 0.9878 - val_loss: 0.0713 - val_acc: 0.9804\n",
      "Epoch 5/5\n",
      "60000/60000 [==============================] - 9s - loss: 0.0297 - acc: 0.9911 - val_loss: 0.0847 - val_acc: 0.9787\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f87e82d1350>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Bez warstw Dropout\n",
    "\n",
    "num_classes = 10\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "\n",
    "x_train = x_train.reshape(60000, 784)  # 784 = 28 * 28\n",
    "x_test = x_test.reshape(10000, 784)\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)\n",
    "\n",
    "model_no_dropout = Sequential()\n",
    "model_no_dropout.add(Dense(512, activation='relu', input_shape=(784,)))\n",
    "model_no_dropout.add(Dense(512, activation='relu'))\n",
    "model_no_dropout.add(Dense(num_classes, activation='softmax'))\n",
    "model_no_dropout.summary()\n",
    "\n",
    "model_no_dropout.compile(loss='categorical_crossentropy',\n",
    "                         optimizer=RMSprop(),\n",
    "                         metrics=['accuracy'])\n",
    "\n",
    "model_no_dropout.fit(x_train, y_train,\n",
    "                     batch_size=128,\n",
    "                     epochs=5,\n",
    "                     verbose=1,\n",
    "                     validation_data=(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss (no dropout): 0.0846566448619\n",
      "Test accuracy (no dropout): 0.9787\n"
     ]
    }
   ],
   "source": [
    "# Bez warstw Dropout\n",
    "\n",
    "score = model_no_dropout.evaluate(x_test, y_test, verbose=0)\n",
    "\n",
    "print('Test loss (no dropout): {}'.format(score[0]))\n",
    "print('Test accuracy (no dropout): {}'.format(score[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "slideshow": {
     "slide_type": "notes"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_10 (Dense)             (None, 2500)              1962500   \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 2000)              5002000   \n",
      "_________________________________________________________________\n",
      "dense_12 (Dense)             (None, 1500)              3001500   \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 1000)              1501000   \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 500)               500500    \n",
      "_________________________________________________________________\n",
      "dense_15 (Dense)             (None, 10)                5010      \n",
      "=================================================================\n",
      "Total params: 11,972,510\n",
      "Trainable params: 11,972,510\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "60000/60000 [==============================] - 212s - loss: 0.7388 - acc: 0.7954 - val_loss: 0.2908 - val_acc: 0.9172\n",
      "Epoch 2/10\n",
      "60000/60000 [==============================] - 191s - loss: 0.2390 - acc: 0.9305 - val_loss: 0.1833 - val_acc: 0.9470\n",
      "Epoch 3/10\n",
      "60000/60000 [==============================] - 166s - loss: 0.1688 - acc: 0.9517 - val_loss: 0.1555 - val_acc: 0.9549\n",
      "Epoch 4/10\n",
      "60000/60000 [==============================] - 166s - loss: 0.1344 - acc: 0.9614 - val_loss: 0.1274 - val_acc: 0.9621\n",
      "Epoch 5/10\n",
      "60000/60000 [==============================] - 166s - loss: 0.1074 - acc: 0.9683 - val_loss: 0.1213 - val_acc: 0.9661\n",
      "Epoch 6/10\n",
      "60000/60000 [==============================] - 440s - loss: 0.0924 - acc: 0.9725 - val_loss: 0.1066 - val_acc: 0.9709\n",
      "Epoch 7/10\n",
      "60000/60000 [==============================] - 169s - loss: 0.0768 - acc: 0.9773 - val_loss: 0.1777 - val_acc: 0.9517\n",
      "Epoch 8/10\n",
      "60000/60000 [==============================] - 183s - loss: 0.0657 - acc: 0.9805 - val_loss: 0.1053 - val_acc: 0.9711\n",
      "Epoch 9/10\n",
      "60000/60000 [==============================] - 170s - loss: 0.0572 - acc: 0.9832 - val_loss: 0.1044 - val_acc: 0.9717\n",
      "Epoch 10/10\n",
      "60000/60000 [==============================] - 166s - loss: 0.0493 - acc: 0.9851 - val_loss: 0.0938 - val_acc: 0.9752\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f87f007f610>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Więcej warstw, inna funkcja aktywacji\n",
    "\n",
    "num_classes = 10\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "\n",
    "x_train = x_train.reshape(60000, 784)  # 784 = 28 * 28\n",
    "x_test = x_test.reshape(10000, 784)\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)\n",
    "\n",
    "model3 = Sequential()\n",
    "model3.add(Dense(2500, activation='tanh', input_shape=(784,)))\n",
    "model3.add(Dense(2000, activation='tanh'))\n",
    "model3.add(Dense(1500, activation='tanh'))\n",
    "model3.add(Dense(1000, activation='tanh'))\n",
    "model3.add(Dense(500, activation='tanh'))\n",
    "model3.add(Dense(num_classes, activation='softmax'))\n",
    "model3.summary()\n",
    "\n",
    "model3.compile(loss='categorical_crossentropy',\n",
    "               optimizer=RMSprop(),\n",
    "               metrics=['accuracy'])\n",
    "\n",
    "model3.fit(x_train, y_train,\n",
    "           batch_size=128,\n",
    "           epochs=10,\n",
    "           verbose=1,\n",
    "           validation_data=(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.0937788957049\n",
      "Test accuracy: 0.9752\n"
     ]
    }
   ],
   "source": [
    "# Więcej warstw, inna funkcja aktywacji\n",
    "\n",
    "score = model3.evaluate(x_test, y_test, verbose=0)\n",
    "\n",
    "print('Test loss: {}'.format(score[0]))\n",
    "print('Test accuracy: {}'.format(score[1]))"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15rc1"
  },
  "livereveal": {
   "start_slideshow_at": "selected",
   "theme": "amu"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}