1344 lines
123 KiB
Plaintext
1344 lines
123 KiB
Plaintext
|
{
|
|||
|
|
"cells": [
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "slide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"### Uczenie maszynowe\n",
|
|||
|
|
"# 11. Sieci neuronowe – wprowadzenie"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 1,
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "notes"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"# Przydatne importy\n",
|
|||
|
|
"\n",
|
|||
|
|
"import matplotlib\n",
|
|||
|
|
"import matplotlib.pyplot as plt\n",
|
|||
|
|
"import numpy as np\n",
|
|||
|
|
"\n",
|
|||
|
|
"%matplotlib inline"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"## 11.1. Perceptron"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"https://www.youtube.com/watch?v=cNxadbrN_aI"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"### Pierwszy perceptron liniowy\n",
|
|||
|
|
"\n",
|
|||
|
|
"* Frank Rosenblatt, 1957\n",
|
|||
|
|
"* aparat fotograficzny podłączony do 400 fotokomórek (rozdzielczość obrazu: 20 x 20)\n",
|
|||
|
|
"* wagi – potencjometry aktualizowane za pomocą silniczków"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"### Uczenie perceptronu\n",
|
|||
|
|
"\n",
|
|||
|
|
"Cykl uczenia perceptronu Rosenblatta:\n",
|
|||
|
|
"\n",
|
|||
|
|
"1. Sfotografuj planszę z kolejnym obiektem.\n",
|
|||
|
|
"1. Zaobserwuj, która lampka zapaliła się na wyjściu.\n",
|
|||
|
|
"1. Sprawdź, czy to jest właściwa lampka.\n",
|
|||
|
|
"1. Wyślij sygnał „nagrody” lub „kary”."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"### Funkcja aktywacji perceptronu\n",
|
|||
|
|
"\n",
|
|||
|
|
"Funkcja bipolarna:\n",
|
|||
|
|
"\n",
|
|||
|
|
"$$ g(z) = \\left\\{ \n",
|
|||
|
|
"\\begin{array}{rl}\n",
|
|||
|
|
"1 & \\textrm{gdy $z > \\theta_0$} \\\\\n",
|
|||
|
|
"-1 & \\textrm{wpp.}\n",
|
|||
|
|
"\\end{array}\n",
|
|||
|
|
"\\right. $$\n",
|
|||
|
|
"\n",
|
|||
|
|
"gdzie $z = \\theta_0x_0 + \\ldots + \\theta_nx_n$,<br/>\n",
|
|||
|
|
"$\\theta_0$ to próg aktywacji,<br/>\n",
|
|||
|
|
"$x_0 = 1$. "
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 3,
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "notes"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"def bipolar_plot():\n",
|
|||
|
|
" matplotlib.rcParams.update({'font.size': 16})\n",
|
|||
|
|
"\n",
|
|||
|
|
" plt.figure(figsize=(8,5))\n",
|
|||
|
|
" x = [-1,-.23,1] \n",
|
|||
|
|
" y = [-1, -1, 1]\n",
|
|||
|
|
" plt.ylim(-1.2,1.2)\n",
|
|||
|
|
" plt.xlim(-1.2,1.2)\n",
|
|||
|
|
" plt.plot([-2,2],[1,1], color='black', ls=\"dashed\")\n",
|
|||
|
|
" plt.plot([-2,2],[-1,-1], color='black', ls=\"dashed\")\n",
|
|||
|
|
" plt.step(x, y, lw=3)\n",
|
|||
|
|
" ax = plt.gca()\n",
|
|||
|
|
" ax.spines['right'].set_color('none')\n",
|
|||
|
|
" ax.spines['top'].set_color('none')\n",
|
|||
|
|
" ax.xaxis.set_ticks_position('bottom')\n",
|
|||
|
|
" ax.spines['bottom'].set_position(('data',0))\n",
|
|||
|
|
" ax.yaxis.set_ticks_position('left')\n",
|
|||
|
|
" ax.spines['left'].set_position(('data',0))\n",
|
|||
|
|
"\n",
|
|||
|
|
" plt.annotate(r'$\\theta_0$',\n",
|
|||
|
|
" xy=(-.23,0), xycoords='data',\n",
|
|||
|
|
" xytext=(-50, +50), textcoords='offset points', fontsize=26,\n",
|
|||
|
|
" arrowprops=dict(arrowstyle=\"->\"))\n",
|
|||
|
|
"\n",
|
|||
|
|
" plt.show()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 4,
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"data": {
|
|||
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAcwAAAEeCAYAAAAHLSWiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de1yUZf7/8fcoZzEVT7mmwapYQsUqth5yKaxdUpHWFHTDxNrU1HbxlNpiWlp00KLdVkOtNA+PBH0Y5LElxbJsk3bZDAo1wbTNNHU7iKDA/fvDL/NrHA43h2EYeD0fj3nIXNd9DZ+5He4398x1X1gMwxAAAKhaC2cXAACAKyAwAQAwgcAEAMAEAhMAABMITAAATCAwAQAwwa2afq45AZwgIiJCu3btcnYZQHNlqaiRM0ygEfruu++cXQKAqxCYAACYQGACAGACgQkAgAkEJgAAJhCYAACYQGACAGACgQkAgAkEJgAAJhCYAACYQGACAGACgQkAgAkEJgAAJhCYAACYQGACAGACgQkAgAkEJlCFkydP6pFHHtHAgQPl4+Mji8WigoICU2PLysqUmJgof39/eXl56ZZbbtGWLVscWzAAhyEwgSocPXpUKSkpateunYYMGVKjsQsWLNCiRYs0ffp07dy5UwMGDNCYMWO0Y8cOB1ULwJEshmFU1V9lJ9DUlZWVqUWLK79Xrl69Wg899JDy8/Pl7+9f5bjTp0+rW7dumjdvnp544glr+9ChQ3XmzBl9+umnVY4PDQ1VVlZWnesHUCuWiho5wwSqUB6WNbV7925dunRJsbGxNu2xsbE6dOiQ8vPz66M8AA2IwAQcICcnR56enurZs6dNe1BQkCQpNzfXGWUBqIMq35K9/fbb7Trj4uIUFxen7777TqNHj7Yb8/DDDysmJkYnTpzQ+PHj7fpnzZqlyMhI5eXlafLkyXb9CQkJuvPOO5Wdna34+Hi7/qefflqDBg3Shx9+qMcee8yuPykpSSEhIcrIyNCSJUvs+pOTk9W7d2+9/fbbWrZsmV3/unXr1K1bN23atEkrVqyw69+8ebM6dOigNWvWaM2aNXb9O3bskI+Pj5YvX66UlBS7/szMTEnS0qVLtW3bNps+b29v7dy5U5K0ePFivfvuuzb97du3t04amT9/vg4cOGDTf91112n9+vWSpPj4eGVnZ9v0BwYGauXKlZKkSZMm6fDhwzb9ISEhSkpKknTlTOjkyZM2/QMHDlRiYqIk6d5779XZs2dt+ocOHaoFCxZIku6++25dvHjRpn/EiBGaPXu2JOn222/X1aKjozV16lQVFhZq2LBhdv11fe2FjJ2lf/zXXRculdr1NTbfrI1XlwlJzi4DaBAFzwyX1HiOe5mZmRW+JetWq2cHuKCdX1lUXNb4wxJA48SkHzQb/vO2O7sE0zjDRHNSfobZiHCGCZSrzQ9oTWbJvvHGG5owYYKOHDli8znmmjVrNHHiRB07dkwBAQGVjg/NWKisxncQAZo1Jv0ADhARESEPDw9t2LDBpn39+vUKDg6uMiwBNE6cYQLV2Lx5syTpk08+kSTt3LlTHTt2VMeOHRUWFiZJcnNz04QJE/Tqq69Kkjp16qQZM2YoMTFRrVu3Vt++fbVp0ybt2bNHaWlpznkiAOqEwASqMWbMGJv7U6dOlSSFhYVZZz2XlpaqtNR2QtFTTz0lX19fvfTSSzp16pR69+6tlJQURUZGNkjdAOoXgQlUo5qJcZVu07JlSyUkJCghIcERZQFoYHyGCQCACQQmAAAmEJgAAJhAYAIAYAKBCQCACQQmAAAmEJgAAJhAYAIAYAKBCQCACQQmAAAmEJgAAJhAYAIAYAKBCQCACQQmAAAmEJgAAJhAYAIAYAKBCQCACQQmAAAmEJgAAJhAYAIAYAKBCQCACQQmAAAmEJgAAJhAYAIAYAKBCQCACQQmAAAmEJgAAJhAYAIAYAKBCQCACQQmAAAmEJgAAJhAYAIAYAKBCQCACQQmAAAmEJgAAJhAYAIAYAKBiTr73//+p8WLFyskJEStW7eWn5+fwsPDtWPHDmeXBgD1xs3ZBcC1ZWZm6g9/+IO++eYbm/a9e/cqMzNTK1as0OTJk51UHQDUH84wUWuZmZkaNmyYvvnmG8XGxurgwYM6f/68Pv74Yw0cOFCGYWjmzJk6efKks0sFgDojMFErp0+f1tixY3Xx4kU999xzWrdunUJDQ9W2bVv1799faWlp8vX1VWFhoTZu3OjscgGgzghM1Mrs2bP17bffasSIEZozZ45df8eOHTV48GBJ0r59+xq6PACodwQmauyLL77Qhg0bZLFY9Nxzz1W6XceOHSVJx48fb6jSAMBhCEzUWHJyssrKynTnnXfqxhtvrHS7y5cv2/wLAK6MwESNlJWV6c0335Qk3XfffVVue+7cOUmSt7e3w+tylBMnTmj06NFq06aNrrnmGo0aNUpfffWVqbEWi6XCW3Z2toOrBuAIXFaCGsnOztapU6ckSXFxcYqLi6t2TLdu3RxclWMUFhYqPDxcnp6eWrt2rSwWixISEnTHHXfo008/VatWrap9jLi4OLvLagIDAx1VMgAHIjBRI7WZwNOrVy8HVOJ4q1at0rFjx5SXl6eePXtKkm6++Wb16tVLycnJmjlzZrWP0bVrVw0YMMDRpQJoALwlixr517/+JUn63e9+p4sXL1Z627p1q3VM3759nVVunaSnp2vAgAHWsJSkgIAADR48WGlpaU6sDIAzEJiokcOHD0uSunfvLi8vr0pvH330kXXMb37zG5vH2LRpk/r37y9vb2+1b99e0dHROnbsWIM+DzNycnIUHBxs1x4UFKTc3FxTj7FixQp5enrKx8dH4eHhev/99+u7TAANhMBEjZSv2uPn51fldtu3b5ck3Xjjjerevbu1/ZVXXtHYsWPl7u6uF198UTNnztSePXs0cOBA05NpGsq5c+fUrl07u3Y/Pz+dP3++2vGxsbFavny5MjIytHLlSp09e1bh4eHKzMx0QLUAHI3PMFEjFy9elCR5eXlVus0XX3yhzz77TJI0fvx4a/u5c+f06KOPKiQkRPv27ZO7u7sk6e6771b//v312GOPaf369Q6svuYsFotdm2EYpsauW7fO+vWQIUMUFRWl4OBgJSQkaP/+/Xbbr1y5UitXrpQknTlzppYVA3AUzjBRIx4eHpKkCxcuVLrN8uXLJV25nOTns2jfeust/fjjj/rzn/9sDUvpymec4eHh2rJliwoLCx1TeC20a9fOemnMz50/f77CM8/qtG7dWsOHD9fBgwcr7J80aZKysrKUlZVlXfQBQONBYKJGrr/+eklSXl5ehf35+fnWs6Tp06erS5cu1r6PP/5YkqxL5v3cbbfdpqKiIuuZaWMQFBSknJwcu/bc3Fz16dOnVo9pGEaFZ60AGj8CEzUSFhYmSXrnnXf03//+16bvwoULiomJUXFxsQIDA7Vw4UKb/q+//lqSdN1119k9bnlbY/rLJiNHjtRHH31kMyGpoKBAH3zwgUaOHFnjx/vhhx+0fft2/frXv67PMgE0EAITNTJx4kS5ubmpuLhYI0eOtP5Jr3feeUeDBw/WwYMH5efnp9TUVLsL+8vfbvX09LR73PLVgBrTW7IPPfSQ/P39FRUVpbS0NKWnpysqKkrdunWzWYzg+PHjcnNz05NPPmltW7p0qR566CFt3LhRmZmZWrt2rQYPHqxTp05pyZIlzng6AOqIST+okRtvvFELFy7UggUL9Mknn+jWW2+16f/lL3+pLVu26Oabb7Yb6+PjI0kqLi62Wy6vqKjIZpvGoFWrVtqzZ49mzJih8ePHyzAMDR06VElJSfL19bVuZxiGSktLVVZWZm3r3bu3tm7dqq1bt+r777/XNddco8GDB+vVV1+122cAXAOBiRpLSEhQYGCgkpKSlJOTI8MwFBgYqJiYGE2bNq3S0OvataukK2+7Xr36z4kTJyRV/HatM3Xv3l1btmypcht/f3+7mbORkZGKjIx0ZGkAGhiBiVqJjo5WdHR0jcbceuutSk5O1oEDB+wC84MPPpCXl1e
|
|||
|
|
"text/plain": [
|
|||
|
|
"<Figure size 576x360 with 1 Axes>"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"metadata": {
|
|||
|
|
"needs_background": "light"
|
|||
|
|
},
|
|||
|
|
"output_type": "display_data"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"bipolar_plot()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"### Schemat perceptronu\n",
|
|||
|
|
"\n",
|
|||
|
|
"<img src=\"perceptron.png\" alt=\"Rys. 11.1. Schemat perceptronu\" style=\"height: 80%\"/>"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"### Zasada działania perceptronu\n",
|
|||
|
|
"\n",
|
|||
|
|
"1. Ustal wartości początkowe $\\theta$ (wektor 0 lub liczby losowe blisko 0).\n",
|
|||
|
|
"1. Dla każdego przykładu $(x^{(i)}, y^{(i)})$, dla $i=1,\\ldots,m$\n",
|
|||
|
|
" * Oblicz wartość wyjścia $o^{(i)} = g(\\theta^{T}x^{(i)}) = g(\\sum_{j=0}^{n} \\theta_jx_j^{(i)})$\n",
|
|||
|
|
" * Wykonaj aktualizację wag (tzw. *perceptron rule*):\n",
|
|||
|
|
" $$ \\theta := \\theta + \\Delta \\theta $$\n",
|
|||
|
|
" $$ \\Delta \\theta = \\alpha(y^{(i)}-o^{(i)})x^{(i)} $$"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"$$\\theta_j := \\theta_j + \\Delta \\theta_j $$\n",
|
|||
|
|
"\n",
|
|||
|
|
"Jeżeli przykład został sklasyfikowany **poprawnie**:\n",
|
|||
|
|
"\n",
|
|||
|
|
"* $y^{(i)}=1$ oraz $o^{(i)}=1$ : $$\\Delta\\theta_j = \\alpha(1 - 1)x_j^{(i)} = 0$$\n",
|
|||
|
|
"* $y^{(i)}=-1$ oraz $o^{(i)}=-1$ : $$\\Delta\\theta_j = \\alpha(-1 - -1)x_j^{(i)} = 0$$"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"Czyli: jeżeli trafiłeś, to nic nie zmieniaj."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"$$\\theta_j := \\theta_j + \\Delta \\theta_j $$\n",
|
|||
|
|
"\n",
|
|||
|
|
"Jeżeli przykład został sklasyfikowany **niepoprawnie**:\n",
|
|||
|
|
"\n",
|
|||
|
|
"* $y^{(i)}=1$ oraz $o^{(i)}=-1$ : $$\\Delta\\theta_j = \\alpha(1 - -1)x_j^{(i)} = 2 \\alpha x_j^{(i)}$$\n",
|
|||
|
|
"* $y^{(i)}=-1$ oraz $o^{(i)}=1$ : $$\\Delta\\theta_j = \\alpha(-1 - 1)x_j^{(i)} = -2 \\alpha x_j^{(i)}$$"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"Czyli: przesuń wagi w odpowiednią stronę."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"### Zalety perceptronu\n",
|
|||
|
|
"\n",
|
|||
|
|
"* intuicyjny i prosty\n",
|
|||
|
|
"* łatwy w implementacji\n",
|
|||
|
|
"* jeżeli dane można liniowo oddzielić, algorytm jest zbieżny w skończonym czasie"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"### Wady perceptronu\n",
|
|||
|
|
"\n",
|
|||
|
|
"* jeżeli danych nie można oddzielić liniowo, algorytm nie jest zbieżny"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 5,
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "notes"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"def plot_perceptron():\n",
|
|||
|
|
" plt.figure(figsize=(12,3))\n",
|
|||
|
|
"\n",
|
|||
|
|
" plt.subplot(131)\n",
|
|||
|
|
" plt.ylim(-0.2,1.2)\n",
|
|||
|
|
" plt.xlim(-0.2,1.2)\n",
|
|||
|
|
"\n",
|
|||
|
|
" plt.title('AND')\n",
|
|||
|
|
" plt.plot([1,0,0], [0,1,0], 'ro', markersize=10)\n",
|
|||
|
|
" plt.plot([1], [1], 'go', markersize=10)\n",
|
|||
|
|
"\n",
|
|||
|
|
" ax = plt.gca()\n",
|
|||
|
|
" ax.spines['right'].set_color('none')\n",
|
|||
|
|
" ax.spines['top'].set_color('none')\n",
|
|||
|
|
" ax.xaxis.set_ticks_position('none')\n",
|
|||
|
|
" ax.spines['bottom'].set_position(('data',0))\n",
|
|||
|
|
" ax.yaxis.set_ticks_position('none')\n",
|
|||
|
|
" ax.spines['left'].set_position(('data',0))\n",
|
|||
|
|
"\n",
|
|||
|
|
" plt.xticks(np.arange(0, 2, 1.0))\n",
|
|||
|
|
" plt.yticks(np.arange(0, 2, 1.0))\n",
|
|||
|
|
"\n",
|
|||
|
|
"\n",
|
|||
|
|
" plt.subplot(132)\n",
|
|||
|
|
" plt.ylim(-0.2,1.2)\n",
|
|||
|
|
" plt.xlim(-0.2,1.2)\n",
|
|||
|
|
"\n",
|
|||
|
|
" plt.plot([1,0,1], [0,1,1], 'go', markersize=10)\n",
|
|||
|
|
" plt.plot([0], [0], 'ro', markersize=10)\n",
|
|||
|
|
"\n",
|
|||
|
|
" ax = plt.gca()\n",
|
|||
|
|
" ax.spines['right'].set_color('none')\n",
|
|||
|
|
" ax.spines['top'].set_color('none')\n",
|
|||
|
|
" ax.xaxis.set_ticks_position('none')\n",
|
|||
|
|
" ax.spines['bottom'].set_position(('data',0))\n",
|
|||
|
|
" ax.yaxis.set_ticks_position('none')\n",
|
|||
|
|
" ax.spines['left'].set_position(('data',0))\n",
|
|||
|
|
"\n",
|
|||
|
|
" plt.title('OR')\n",
|
|||
|
|
" plt.xticks(np.arange(0, 2, 1.0))\n",
|
|||
|
|
" plt.yticks(np.arange(0, 2, 1.0))\n",
|
|||
|
|
"\n",
|
|||
|
|
"\n",
|
|||
|
|
" plt.subplot(133)\n",
|
|||
|
|
" plt.ylim(-0.2,1.2)\n",
|
|||
|
|
" plt.xlim(-0.2,1.2)\n",
|
|||
|
|
"\n",
|
|||
|
|
" plt.title('XOR')\n",
|
|||
|
|
" plt.plot([1,0], [0,1], 'go', markersize=10)\n",
|
|||
|
|
" plt.plot([0,1], [0,1], 'ro', markersize=10)\n",
|
|||
|
|
"\n",
|
|||
|
|
" ax = plt.gca()\n",
|
|||
|
|
" ax.spines['right'].set_color('none')\n",
|
|||
|
|
" ax.spines['top'].set_color('none')\n",
|
|||
|
|
" ax.xaxis.set_ticks_position('none')\n",
|
|||
|
|
" ax.spines['bottom'].set_position(('data',0))\n",
|
|||
|
|
" ax.yaxis.set_ticks_position('none')\n",
|
|||
|
|
" ax.spines['left'].set_position(('data',0))\n",
|
|||
|
|
"\n",
|
|||
|
|
" plt.xticks(np.arange(0, 2, 1.0))\n",
|
|||
|
|
" plt.yticks(np.arange(0, 2, 1.0))\n",
|
|||
|
|
"\n",
|
|||
|
|
" plt.show()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 6,
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"data": {
|
|||
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAqwAAADGCAYAAAAXMtIlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAWVUlEQVR4nO3df5DUd33H8dd7cRO5XJYcaUQn1YEh0Qop2IYqmBACiS1YQ1J6RB1Bo2PUOUwnV5z6MzVJUepUClYnJiZq6FG1sM4kqYBOmhFsMFQ5M57lgt4dXEhik4YcJnCXHyu8+8f3e7C3t7e3y+339nN3z8fMdzb3/X73u5/95vPm+9rvfr+fNXcXAAAAEKpUrRsAAAAAlEJgBQAAQNAIrAAAAAgagRUAAABBI7ACAAAgaARWAAAABI3ACgAAgKARWEeJmW03Mzez/ymxjsfTz4ZYfmW8fFPB/HvznutmljOzHjP7VbzsXWY2qdrvCZhozOwdZpY1s6fM7GUzO2JmPzazj5jZq4qs311QmyfM7Fkz22lmy2rxHoDxwMw+GdfU14ZY/iYz6zOzg2Z2TsGyt5lZi5k9bmYvmdlRM/upma01s8lDbO/hIrXcY2a7zOy9SbxHDGT8cEDyzOx1kp5Q9AHBJL3V3X9eZL38/xmN7v79guVXSvqxpK+4+8158++V9AFJd0l6On6djKQ3S1ooabKkn0l6t7t3V+t9ARNFHEbvkvQhSccl/UDSIUnnS1om6fWS9kl6l7s/k/e8bkmvlfSP8ayzFdXlNYrq9APu/q+j8y6A8SM+CbNH0lslLXH3XQXLHpb0tiLLvijp05JelrRT0q8l1Uv6c0kXS+qQ9E537yx4vYclXSZpvaRXJKUlXSTpOklnSfqsu38xgbeKfu7OlPAk6VOSXNKX48evD7GeS/qtpBclPSZpUsHyK+N1NhXMvzee/5Yi2zxfUku8/ICk+lrvDyamsTZJ2hDX0COSXluw7GxJX89b/qq8Zd2Sfldke9fH6x+u9XtjYhqrk6IPfy9JOijpnLz5n4zr66sF6zfnHQtnFixLSbo1Xt4h6dyC5Q/Hy+oL5l8u6YSkXkmTa71PxvPEJQGj4wZJz0v6nKTfSHqPmb16iHX/T9LXJP1R/LwRcffnJL1f0oOS3iTp4yPdJjCRmNkbJd0s6TlJy9396fzl7v6ypCZJP5E0X9G3HcPZpuhM7evN7ILqthiYGNz9MUl/L2mGpC9JkpnNknSbpC5FJ4sUzz9f0j8oCrjXuHtXwbZOuvutkr6r6Mzp35bZhocldUqqU3TcRkIIrAkzs8sUBcWsu78kaYuk8yStKPG09YoC7udLBNuyefQxsP+riutHuj1ggvmAon8rv+HuzxZboaDGPljmdi1+zI2secCEtkHSf0tqMrOrJW1W9BX9h9y9N2+96yWdI2mbu3eU2N4X4scPnUFbqOUEEViT13/waokftyj6WmHIg5q790j6J0XXxa2pUjt+qqiY5ha7OQTAkN4ePz40zHo/kfR7SX9Wxk2O71F08HzM3X83wvYBE5a7n1D0beTLiq4tnyfpX9z9JwWrllXH7r5f0jOS3hDff1KSmS1SdO3rEUWXEiAhBJcExXcmXi/psKKDmdz9kJn9VNISM3uDux8e4umbFH19/2kzu9vdXxhJW9z9FTPrkTRN0lRFlx4AGN5r48cnS63k7i+a2XOKaux8na6xV5vZrfF/ny1plqKbrl5U9T6QAhOWux8wsy2SPqwobH6myGpl1XHeOtMkvU7S/xYs+4yZvaIoP10s6a8knZT08fjyICSEwJqsRknnSvpa/JVhvxZFdxveIOn2Yk90914zW6foetZPKLpOZ6Rs+FUAjEB/jeXX+9mSPl+w3ouSlrn77lFpFTCOmdkMSf1DS01TdDb1P0eyyfix2DBKny74+4Sk97r7thG8HsrAJQHJ6v/af0vB/K2KhsW4wcxKhchvKLr7sdnMXjOShpjZ2YrOrJ6Q1DOSbQETTP9NVn9YaqX4evOpimo7v8aed3dzd5M0RdGB9aSkrWZWcpsASouPod9UdInNzYpuqrrbzOoLVi2rjmMXFjwn37lxLddLeqeimzG/bWZzKm07KkNgTYiZzZR0Rfzn/vwBhxUdzM5SdGfjlUNtw91zis7M1CsaYWAk3q7ojPov3f33I9wWMJE8Ej9eNcx6VyiqsZ/H19UN4u4vuPv3JH1E0msUDYcF4Mw1SVos6R53/4qiY+Z0nR77uF9ZdRyPMjBN0ZBzhZcDnOLuve6+U9JKRWH528OcgMIIEViTc4OirxV+rOjTX+F0f7zecHcUf0dSm6SPKirCisVF1P81xr+fyTaACWyzoq8Gb4yHxhmkoMa+PdwG3f07kvZKepeZLaxWQ4GJJL4U4EuKfphnbTx7g6IfymmKb4jqt1XRpTgr4xNKQ+m//vVb5bQhvrnr+5L+VNHNlEgIgTUBZpZSNBTOCUnvc/cPF06KbsZ6TtJfm1lmqG25+0lJn1V0Rrbis6xmNlXRAfcdin7R446K3xAwgbn7ryV9VdIfSLrfzKblLzezs+LlVyoaXqfcX666LX68tSoNBSaQ+EPitxSd3byx/8bk+NuNDyq6NOebZlYXzz+i6OzrqyX9Rxx287eXMrPPSXqfonFV/7mC5tym6EPtLfHxHwngpqtkXK1oSKrtQ32lEN+1/2+S/kbSuyXdPdTG3P0H8c/CXT7M637MzJ5WdGY3o2gQ40Ua+NOsxyt9MwD0CUXjJ79fUoeZFf406xsktUq6Nr6UZ1ju/kMz+5miEUMujwcgB1CeNYo+JN7j7j/KX+Du7WZ2u6IxVdfp9I8AfFnRB8+/k9RuZjsU/ZhP/0+zvlFRWF3m7sfKbYi7/8rM7lM0YsD1kr43gveFIdjAm9dRDWb2XUVfDax092yJ9d4i6VFJe919QXx96y/d/S1F1r1c0n/Ff37F3W/OW3avBv66zglJxxQNzdEqKStpR3y2FsAZMrO/UHR5znxFB77jii7Z+a6kbxWGVTPrlnSeu583xPb+UtHYkQ+5+9UJNh0YN+Kzo79SdD/IJcWGfYzHG98r6U8kXe7uj+QtW6Bo2MiFiq5XfVHRz7VmJd3h7n1FtvewotF9zi124ifveN4u6Y853lYfgRUAAABB41oLAAAABI3ACgAAgKARWAEAABA0AisAAACCRmAFAABA0IYLrD6a09KlS0f19ZiYzmAK3ajuD2qWKfApdKO6P6hXpjEwDSmoM6xHjhypdRMAVICaBcYO6hVjWVCBFQAAAChEYAUAAEDQCKwAAAAIGoEVAAAAQSOwAgAAIGgEVgAAAASNwAoAAICgEVgBAAAQNAIrAAAAgkZgBQAAQNAIrAAAAAgagRUAAABBI7ACAAAgaARWAAAABI3ACgAAgKBVPbA++eSTuummm7RgwQLV1dXJzNTd3V3tlwFQBdQrMHZQr5jIqh5YOzs7tXXrVjU0NGjhwoXDP6GrS2pqkjIZqbU1emxqiuYDRXT1dKlpe5My6zNK3ZZSZn1GTdub1NVDn6lUxfWqgfu/9bet7H+URL1WD/WKUZGfy1KpYHKZuXup5SUXFnPy5EmlUlEOvueee3TjjTfq0KFDmj59+uCVd+6UGhulXE7K5TRP0j5JSqejKZuVli2rtAkYx3Z27FTjtkblTuSUO5k7NT+dSis9Ka3syqyWXZxon7EkN14FFdVsRfWqIvv/LkkfHdX9jzGEeh0W9YqwFOSyU0Yvlw1Zs1U/w9pfTMPq6op2Sl/fwJ0iRX/39UXLOdOKWFdPlxq3Naov1zfg4CdJuZM59eX61LitkTMHFSi7XsX+R2XoL9VHvSJRgeey2t10tWHD4B1SKJeTNm4cnfYgeBse2aDcidJ9Jncip4176TNJYP+jEvSX2mL/o2KB57LaBdYtW8rbMS0to9MeBG9L25ZBZwoK5U7m1NJGn0kC+x+VoL/UFvsfFQs8l9UusB4/Xt31MO4df6W8vlDueqgM+x+VoL/UFvsfFQs8l9UusNbXV3c9jHv1Z5XXF8pdD5Vh/6MS9JfaYv+jYoHnsto
|
|||
|
|
"text/plain": [
|
|||
|
|
"<Figure size 864x216 with 3 Axes>"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"metadata": {
|
|||
|
|
"needs_background": "light"
|
|||
|
|
},
|
|||
|
|
"output_type": "display_data"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"plot_perceptron()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"### Perceptron – funkcje aktywacji\n",
|
|||
|
|
"\n",
|
|||
|
|
"Zamiast funkcji bipolarnej możemy zastosować funkcję sigmoidalną jako funkcję aktywacji."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 7,
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "notes"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"def plot_activation_functions():\n",
|
|||
|
|
" plt.figure(figsize=(16,7))\n",
|
|||
|
|
" plt.subplot(121)\n",
|
|||
|
|
" x = [-2,-.23,2] \n",
|
|||
|
|
" y = [-1, -1, 1]\n",
|
|||
|
|
" plt.ylim(-1.2,1.2)\n",
|
|||
|
|
" plt.xlim(-2.2,2.2)\n",
|
|||
|
|
" plt.plot([-2,2],[1,1], color='black', ls=\"dashed\")\n",
|
|||
|
|
" plt.plot([-2,2],[-1,-1], color='black', ls=\"dashed\")\n",
|
|||
|
|
" plt.step(x, y, lw=3)\n",
|
|||
|
|
" ax = plt.gca()\n",
|
|||
|
|
" ax.spines['right'].set_color('none')\n",
|
|||
|
|
" ax.spines['top'].set_color('none')\n",
|
|||
|
|
" ax.xaxis.set_ticks_position('bottom')\n",
|
|||
|
|
" ax.spines['bottom'].set_position(('data',0))\n",
|
|||
|
|
" ax.yaxis.set_ticks_position('left')\n",
|
|||
|
|
" ax.spines['left'].set_position(('data',0))\n",
|
|||
|
|
"\n",
|
|||
|
|
" plt.annotate(r'$\\theta_0$',\n",
|
|||
|
|
" xy=(-.23,0), xycoords='data',\n",
|
|||
|
|
" xytext=(-50, +50), textcoords='offset points', fontsize=26,\n",
|
|||
|
|
" arrowprops=dict(arrowstyle=\"->\"))\n",
|
|||
|
|
"\n",
|
|||
|
|
" plt.subplot(122)\n",
|
|||
|
|
" x2 = np.linspace(-2,2,100)\n",
|
|||
|
|
" y2 = np.tanh(x2+ 0.23)\n",
|
|||
|
|
" plt.ylim(-1.2,1.2)\n",
|
|||
|
|
" plt.xlim(-2.2,2.2)\n",
|
|||
|
|
" plt.plot([-2,2],[1,1], color='black', ls=\"dashed\")\n",
|
|||
|
|
" plt.plot([-2,2],[-1,-1], color='black', ls=\"dashed\")\n",
|
|||
|
|
" plt.plot(x2, y2, lw=3)\n",
|
|||
|
|
" ax = plt.gca()\n",
|
|||
|
|
" ax.spines['right'].set_color('none')\n",
|
|||
|
|
" ax.spines['top'].set_color('none')\n",
|
|||
|
|
" ax.xaxis.set_ticks_position('bottom')\n",
|
|||
|
|
" ax.spines['bottom'].set_position(('data',0))\n",
|
|||
|
|
" ax.yaxis.set_ticks_position('left')\n",
|
|||
|
|
" ax.spines['left'].set_position(('data',0))\n",
|
|||
|
|
"\n",
|
|||
|
|
" plt.annotate(r'$\\theta_0$',\n",
|
|||
|
|
" xy=(-.23,0), xycoords='data',\n",
|
|||
|
|
" xytext=(-50, +50), textcoords='offset points', fontsize=26,\n",
|
|||
|
|
" arrowprops=dict(arrowstyle=\"->\"))\n",
|
|||
|
|
"\n",
|
|||
|
|
" plt.show()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 8,
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"data": {
|
|||
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA4sAAAGKCAYAAACl0NTvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3hUZfrG8XvSOxAIEHrokEgNCCKigIIi4KJiWVxAWXApKygqKEVWFHVRwZ+A4NqWoiAugqAioOCKsvSW0CGhk0ACCenJnN8fWQazQwlkMicz8/1cVy7zvmfOzJM45Mmdc857LIZhCAAAAACA3/MyuwAAAAAAQNlDWAQAAAAA2CEsAgAAAADsEBYBAAAAAHYIiwAAAAAAOz7X2c5SqUAxdO/eXd9//73ZZQCuwGJ2AW6A3gwUA70ZKLar9maOLAIOcPbsWbNLAAAAv0NvBkqOsAgAAAAAsENYBAAAAADYISwCAAAAAOwQFgEAAAAAdgiLAAAAAAA7hEUAAAAAgB3CIgAAAADADmERAAAAAGCHsAgAAAAAsENYBAAAAADYISwCAAAAAOwQFgEAAAAAdgiLAAAAAAA7hEUAAAAAgB3CIgAAAADADmERAAAAAGCHsAgAAAAAsENYBAAAAADYISwCAAAAAOwQFgEAAAAAdgiLAAAAAAA7hEUAAAAAgB3CIgAAAADADmERAAAAAGCHsAgAAAAAsENYhNs4fvy4RowYofbt2ysoKEgWi0UJCQnF2tdqtWrKlCmqU6eOAgIC1Lx5c3311VelWzAAAG6O3gy4NsIi3MbBgwe1aNEiVahQQR07dryhfcePH69XXnlFw4cP13fffad27drp4Ycf1rfffltK1QIA4P7ozYBrsxiGca3t19wIlCVWq1VeXoV///jHP/6hP//5zzpy5Ijq1Klzzf2SkpJUs2ZNjRkzRpMmTbLNd+nSRcnJydq5c+d1Xzs2NlabN28uUf2Ah7CYXYAboDfDZdCbAZdw1d7MkUW4jUvN6EatXLlSubm56tevX5H5fv36adeuXTpy5IgjygMAwOPQmwHX5rQji3feeafdXN++fTV06FBlZmbqvvvus9s+YMAADRgwQGfPntVDDz1kt/0vf/mLHnnkER07dkxPPPGE3fbnnntOPXv21L59+zRkyBC77ePGjVPXrl21fft2jRw50m7766+/rttuu02//vqrXnrpJbvt06ZNU4sWLbR69WpNnjzZbvvs2bPVqFEjffPNN3r77bftts+dO1c1a9bUwoULNWvWLLvtixcvVqVKlfTpp5/q008/tdv+7bffKigoSDNnztSiRYvstq9du1aSNHXqVC1fvrzItsDAQH333XeSpFdffVVr1qwpsr1ixYq26wLGjh2r3377rcj2GjVqaN68eZKkkSNHavv27UW2N2zYUHPmzJEkDR48WPv37y+yvUWLFpo2bZqkwh/8x48fL7K9ffv2mjJliiTpwQcf1Llz54ps79Kli8aPHy9Juvfee5WVlVVke0REhBYvXqwjR45owIABtvkLkbE6X6ODDG8/OdKpz0Yqsv80hz4nUBYlvNGjpE/BkcWSozfTm92qN1/y+/de48aNdfz4cd1xxx227QMGDFDTpk116623KiYmRhUrViyy//++97Zs2aLWrVvbtvPeWyvJM997999/v0aPHi3JvX7uXfp/6gAcWQQklUpQBAAAjpWfny8fHx+7+fDwcNv2K5kzZ4569OihLVu2KC8vr1RrBDwB1yzCLV3tuog6Y1aUyutxZBGegiOLZQK9GS7pRq5Z/POf/6zly5fr1KlTReYPHDighg0b6p///OcVj+D8HtcsAsV21d5s/ycbwENc+qX3xRdf1PTp05WVlSWL5fK/lY0bN+rWW2/V8uXL1aPHtX9Bjl09UZtL/ks0AABQ4RHE1NRUGYZRpDenpqbatgPuKCe/QGcv5io5PUdn03N09mKOzmXk6uzFHKVk5ColI1fnLuYqNTNXF3PytXPiPUX+jTgaYREeLzo6Wjk5OTp06JDq169vm4+Pj5ckNW3a1KzSAADwSPRmuBvDMHQ+M08nL2Tp9IVsnbyQrdMXsnQmLUdn0rJ1Ji1bSek5Op95Y6dPZ+YWKNi/9CIdYREer3v37vLz89P8+fM1ceJE2/y8efMUExOjqKgoE6sDAMDz0Jvhii5k5ikxJUPHUrJ0LDVTx1IydTw1SyfOZ+nk+Sxl5hY4/DVTMnIJi0BxLV68WJK0ZcsWSdJ3332niIgIRUREqFOnTlfcp3Llyho1apSmTJmi0NBQtWrVSgsXLtSPP/6opUuXOq12AADcUXF6s4+Pj/r376+PPvpIEr0ZZVdmbr4OJ2fo8NkMHU6+qCNnM5RwLlOJ5zJu+KjglXh7WVQx2E+Vw/xVKaTwo2KInyoF+ys82E/hIX6qGOxX+Hmwn4L8SjfOERbhVh5++OEi46FDh0qSOnXqdM3lhV977TWFhIRo+vTpOn36tBo1aqRFixapZ8+epVkuAABurzi9uaCgQAUFRY+60JthpqzcAu0/k659p9O1/0y6DiRd1MGkizpxPuv6O19FsJ+3IssHKrJcgCLLBahqucLPq4T5q3JogKqEBSg82E/eXmVnLThWQ4VH+f1qqA5Y1dGGFdeAYis7HdB10ZuBYqA3o7iS03O0++QFxZ9MK/w4laaEcxm6dkyy5+/jpdoVg1QrPEg1KgSpZniQalQIVI0KgapePlDlAn1LdTGaEmA1VAAAAACeLT07TzuOXdCO4+e18/h57Tx+QacuZBd7f28vi2pXDFLdSiGqFxGsqEqFH7UrBqtyqL+8ytBRQUcgLAIAAABwO4Zh6GhKpjYeSdHWo6namnhe+5PSi3XE0Msi1akYrEZVQ9WoaqgaVglV/cohqlMxWH4+XqVffBlBWAQAAADg8gzD0JGzGfrt8Dn9duicNiWk6ExaznX3C/D1UtPIMEVXK6foamFqWi1MDauEKsDX2wlVl22ERQAAAAAu6ezFHP1y4Kx+PpCsXw+e0+m0a59S6mWRGlcNU8ta5dW8Rnk1q1lO9SNC5OPtOUcLbwRhEQAAAIBLKLAa2nH8vH7ck6S1+5O0+0TaNR8f6u+j1nUqqE2dcLWqVUHNapQr1fsSuhu+UwAAAADKrKzcAv18IFk/xJ3R2n1JOpeRe9XHhvr76Na6FdW+XkW1qxuuxlXDytStKFwNYREAAABAmXIxJ19r9pzR97tPa+2+ZGXlFVzxcd5eFrWqVV53NIhQx4YRiqkWximlDkRYBAAAAGC67LwCrd2XpGU7TmrNniTl5Fuv+LhKIf7q3DhCnRtXVof6lRQa4OvkSj0HYREAAACAKQzD0KaEVP1r63Gt2HlK6Tn5V3xc/coh6hZdRfc0rapbqpdzu/sZllWERQAAAABOdfpCthZtPqbFW47raErmFR/TuGqoetwSqXtvqar6lUOdXCEkwiIAAAAAJyiwGlq7L0mfbzyqH/cmyWrYP6ZOxSD1al5NPZtXU4MqBESzERYBAAAAlJrzmblauOmY/vlbok6cz7LbHhbgo/ubV9ODrWqoVa3yslg4xbSsICwCAAAAcLiDSen66JcjWrLthLLz7Berua1eRT3atpbuaVpFAb7eJlSI6yEsAgAAAHCISwvWzPn5kFbvSbLbXiHIV31ja+rRtrUUVSnYhApxIwiLAAAAAErEMAz9tC9J7/94UFuPnrfb3jQyTAM61FGv5tU4iuhCCIsAAAAAborVamjVnjP6vx8PaPeJNLvtXZtU0eA76qpNnQpci+iCCIsAAAAAbohhGFq7L1l/X7lP8aeKhkQ/by/1aVVdgzrWVf3KISZVCEcgLAIAAAAoto1HUvT3lXu1KSG1yLy/j5f+eGttDelUV1XCAkyqDo5EWAQAAABwXQeTLmrKt3u0Zm/RhWsCfL30p/Z1NKhjlCqHEhLdCWERAAAAwFWdu5ij6WsOaP5/jqrAatjmfb0terxtLQ3rXJ+Q6KYIiwAAAADs5BdYNXdDot5ZtV/p2fm2eYtF+kOL6hp1d0PVDA8ysUKUNsIiAAAAgCI2JaR
|
|||
|
|
"text/plain": [
|
|||
|
|
"<Figure size 1152x504 with 2 Axes>"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"metadata": {
|
|||
|
|
"needs_background": "light"
|
|||
|
|
},
|
|||
|
|
"output_type": "display_data"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"plot_activation_functions()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"### Perceptron a regresja liniowa\n",
|
|||
|
|
"\n",
|
|||
|
|
"<img src=\"reglin.png\" alt=\"Rys. 11.2. Perceptron a regresja liniowa\" style=\"height: 60%\">"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"#### Uczenie regresji liniowej:\n",
|
|||
|
|
"* Model: $$h_{\\theta}(x) = \\sum_{i=0}^n \\theta_ix_i$$\n",
|
|||
|
|
"* Funkcja kosztu (błąd średniokwadratowy): $$J(\\theta) = \\frac{1}{m} \\sum_{i=1}^{m} (h_{\\theta}(x^{(i)}) - y^{(i)})^2$$\n",
|
|||
|
|
"* Po obliczeniu $\\nabla J(\\theta)$ - zwykły SGD."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"### Perceptron a dwuklasowa regresja logistyczna\n",
|
|||
|
|
"\n",
|
|||
|
|
"<img src=\"reglog.png\" alt=\"Rys. 11.3. Perceptron a dwuklasowa regresja logistyczna\" style=\"height:60%\">"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"#### Uczenie dwuklasowej regresji logistycznej:\n",
|
|||
|
|
"* Model: $h_{\\theta}(x) = \\sigma(\\sum_{i=0}^n \\theta_ix_i) = P(1|x,\\theta)$\n",
|
|||
|
|
"* Funkcja kosztu (entropia krzyżowa): $$\\begin{eqnarray} J(\\theta) &=& -\\frac{1}{m} \\sum_{i=1}^{m} \\big( y^{(i)}\\log P(1|x^{(i)},\\theta) \\\\ && + (1-y^{(i)})\\log(1-P(1|x^{(i)},\\theta)) \\big) \\end{eqnarray}$$\n",
|
|||
|
|
"* Po obliczeniu $\\nabla J(\\theta)$ - zwykły SGD."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"### Perceptron a wieloklasowa regresja logistyczna\n",
|
|||
|
|
"\n",
|
|||
|
|
"<img src=\"multireglog.png\" alt=\"Rys. 11.4. Perceptron a wieloklasowa regresja logistyczna\" style=\"height:60%\">"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"#### Wieloklasowa regresja logistyczna\n",
|
|||
|
|
"* Model (dla $c$ klasyfikatorów binarnych): \n",
|
|||
|
|
"$$\\begin{eqnarray}\n",
|
|||
|
|
"h_{(\\theta^{(1)},\\dots,\\theta^{(c)})}(x) &=& \\mathrm{softmax}(\\sum_{i=0}^n \\theta_{i}^{(1)}x_i, \\ldots, \\sum_{i=0}^n \\theta_i^{(c)}x_i) \\\\ \n",
|
|||
|
|
"&=& \\left[ P(k|x,\\theta^{(1)},\\dots,\\theta^{(c)}) \\right]_{k=1,\\dots,c} \n",
|
|||
|
|
"\\end{eqnarray}$$"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* Funkcja kosztu (**przymując model regresji binarnej**): $$\\begin{eqnarray} J(\\theta^{(k)}) &=& -\\frac{1}{m} \\sum_{i=1}^{m} \\big( y^{(i)}\\log P(k|x^{(i)},\\theta^{(k)}) \\\\ && + (1-y^{(i)})\\log P(\\neg k|x^{(i)},\\theta^{(k)}) \\big) \\end{eqnarray}$$\n",
|
|||
|
|
"* Po obliczeniu $\\nabla J(\\theta)$, **c-krotne** uruchomienie SGD, zastosowanie $\\mathrm{softmax}(X)$ do niezależnie uzyskanych klasyfikatorów binarnych."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* Przyjmijmy: \n",
|
|||
|
|
"$$ \\Theta = (\\theta^{(1)},\\dots,\\theta^{(c)}) $$\n",
|
|||
|
|
"\n",
|
|||
|
|
"$$h_{\\Theta}(x) = \\left[ P(k|x,\\Theta) \\right]_{k=1,\\dots,c}$$\n",
|
|||
|
|
"\n",
|
|||
|
|
"$$\\delta(x,y) = \\left\\{\\begin{array}{cl} 1 & \\textrm{gdy } x=y \\\\ 0 & \\textrm{wpp.}\\end{array}\\right.$$"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* Wieloklasowa funkcja kosztu $J(\\Theta)$ (kategorialna entropia krzyżowa):\n",
|
|||
|
|
"$$ J(\\Theta) = -\\frac{1}{m}\\sum_{i=1}^{m}\\sum_{k=1}^{c} \\delta({y^{(i)},k}) \\log P(k|x^{(i)},\\Theta) $$"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* Gradient $\\nabla J(\\Theta)$:\n",
|
|||
|
|
"$$ \\dfrac{\\partial J(\\Theta)}{\\partial \\Theta_{j,k}} = -\\frac{1}{m}\\sum_{i = 1}^{m} (\\delta({y^{(i)},k}) - P(k|x^{(i)}, \\Theta)) x^{(i)}_j \n",
|
|||
|
|
"$$\n",
|
|||
|
|
"\n",
|
|||
|
|
"* Liczymy wszystkie wagi jednym uruchomieniem SGD"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"### Perceptron – podsumowanie\n",
|
|||
|
|
"\n",
|
|||
|
|
"* W przypadku jednowarstowej sieci neuronowej wystarczy znać gradient funkcji kosztu.\n",
|
|||
|
|
"* Wtedy liczymy tak samo jak w przypadku regresji liniowej, logistycznej, wieloklasowej logistycznej itp. (wymienione modele to szczególne przypadki jednowarstwowych sieci neuronowych).\n",
|
|||
|
|
"* Regresja liniowa i binarna regresja logistyczna to jeden neuron.\n",
|
|||
|
|
"* Wieloklasowa regresja logistyczna to tyle neuronów, ile klas."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"Funkcja aktywacji i funkcja kosztu są **dobierane do problemu**."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "slide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"## 11.2. Funkcje aktywacji"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* Operacje, które każdy neuron wykonuje, to dodawanie i mnożenie. Są to **funkcje liniowe**."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* Złożenie funkcji liniowych jest funkcją liniową."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* Gdyby nie było funkcji aktywacji, cała sieć neuronowa działałaby jak regresja liniowa."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* **Najważniejszym celem** stosowania funkcji aktywacji jest **wprowadzenie nieliniowości**, dzięki której sieć może uczyć się zależności bardziej skomplikowanych niż tylko liniowe."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"Jakie cechy powinna mieć dobra funkcja aktywacji?"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* **różniczkowalna** (co najmniej **przedziałami różniczkowalna**), aby można było obliczyć jej gradient"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* **prosta obliczeniowo**, ponieważ podczas uczenia sieci jej wartość obliczana jest wiele razy"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* odporna na **problem zanikającego gradientu** (wyjaśnienie kilka slajdów dalej)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 1,
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "notes"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"name": "stderr",
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"text": [
|
|||
|
|
"/home/pawel/.local/lib/python2.7/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
|
|||
|
|
" from ._conv import register_converters as _register_converters\n",
|
|||
|
|
"Using TensorFlow backend.\n"
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"%matplotlib inline\n",
|
|||
|
|
"\n",
|
|||
|
|
"import math\n",
|
|||
|
|
"import matplotlib.pyplot as plt\n",
|
|||
|
|
"import numpy as np\n",
|
|||
|
|
"import random\n",
|
|||
|
|
"\n",
|
|||
|
|
"import keras\n",
|
|||
|
|
"from keras.datasets import mnist\n",
|
|||
|
|
"from keras.models import Sequential\n",
|
|||
|
|
"from keras.layers import Dense, Dropout, SimpleRNN, LSTM\n",
|
|||
|
|
"from keras.optimizers import Adagrad, Adam, RMSprop, SGD\n",
|
|||
|
|
"\n",
|
|||
|
|
"from IPython.display import YouTubeVideo"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 2,
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "notes"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"def plot(fun):\n",
|
|||
|
|
" x = np.arange(-3.0, 3.0, 0.01)\n",
|
|||
|
|
" y = [fun(x_i) for x_i in x]\n",
|
|||
|
|
" fig = plt.figure(figsize=(14, 7))\n",
|
|||
|
|
" ax = fig.add_subplot(111)\n",
|
|||
|
|
" fig.subplots_adjust(left=0.1, right=0.9, bottom=0.1, top=0.9)\n",
|
|||
|
|
" ax.set_xlim(-3.0, 3.0)\n",
|
|||
|
|
" ax.set_ylim(-1.5, 1.5)\n",
|
|||
|
|
" ax.grid()\n",
|
|||
|
|
" ax.plot(x, y)\n",
|
|||
|
|
" plt.show()"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "slide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"### Funkcja logistyczna"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"$$ g(x) = \\frac{1}{1 + e^{-x}} $$"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"#### Funkcja logistyczna – wykres"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 3,
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"data": {
|
|||
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA1cAAAG2CAYAAACTRXz+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XmsXudh3/nfw7vvGy/3VRu1Wou1\n2HAWOk4cx0jjpq1bezpN0mXUdhq0M0Axk04GCaaDAl2AWVNMa6RB00GRNJhOWs/UbdI0ZdJ2YFmW\nLVnWZslcxEUSl7vxbrzbmT/el1ekxCvJvke8l+TnA7x43/O+5/I8NB6R/Pqc87ylqqoAAACwPls2\negAAAAA3A3EFAABQA3EFAABQA3EFAABQA3EFAABQA3EFAABQg1riqpTya6WUs6WUb6/x+eFSymQp\n5dnm45fqOC4AAMBm0VrTr/OPk/xKkn/yHvv8h6qqfrKm4wEAAGwqtZy5qqrqD5OM1fFrAQAA3Iiu\n5z1XHy+lPFdK+dellPuu43EBAAA+dHVdFvh+vpFkf1VV06WUzyb5F0nuvNaOpZQnkzyZJJ2dnR/d\nt2/fdRoiN4qVlZVs2WItFq5mXrAWc4NrMS9Yi7nBtXznO985X1XV6PvtV6qqquWApZQDSf7fqqru\n/wD7Hk/yaFVV599rv0OHDlWvvPJKLePj5nHkyJEcPnx4o4fBJmNesBZzg2sxL1iLucG1lFKeqarq\n0ffb77pkeSllRymlNF8/3jzuhetxbAAAgOuhlssCSym/keRwkq2llFNJfjlJW5JUVfUPkvyJJH+5\nlLKUZC7JF6q6TpkBAABsArXEVVVVX3yfz38ljaXaAQAAbkru1gMAAKiBuAIAAKiBuAIAAKiBuAIA\nAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiB\nuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIA\nAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiB\nuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIA\nAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiB\nuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKiBuAIAAKhBLXFVSvm1UsrZUsq31/i8\nlFL+t1LKa6WUb5VSHqnjuAAAAJtFXWeu/nGSz7zH5z+R5M7m48kk/0dNxwUAANgUaomrqqr+MMnY\ne+zyuST/pGr4apLBUsrOOo4NAACwGVyve652Jzl5xfap5nsAAAA3hdaNHsA7lVKeTOPSwYyOjubI\nkSMbOyA2nenpafOCdzEvWIu5wbWYF6zF3GA9rldcnU6y94rtPc333qWqqi8l+VKSHDp0qDp8+PCH\nPjhuLEeOHIl5wTuZF6zF3OBazAvWYm6wHtfrssAvJ/mZ5qqBH0syWVXVG9fp2AAAAB+6Ws5clVJ+\nI8nhJFtLKaeS/HKStiSpquofJPlKks8meS3JbJI/W8dxAQAANota4qqqqi++z+dVkr9Sx7EAAAA2\no023oAUAAMCHpaqqXFpaycylpcwuLGdmYSkzl5Yyc2k5swuN55mFt7d//L4dH/jXFlcAAMCmd2lp\nOdPzS5m+tJSLzefp+aXMLFy9/fbni1e9N7uwvPq8vFJ9oGOWkuwd7v7AYxRXAADAh+bymaKpucVM\nzi1man4xU3NLmZpvbs8t5uIVETQ9v3T1dvP1wvLK+x6rZUtJb0drejta09fZeB7sbs+e4e70tLek\nu73xXndHS3raW9Pd3tLcbl39vKejJT0drelpb01n25aUUvKnPuDvVVwBAADvaXG5EUdT80vvGUlT\n80tXvG5+Prf4vmHU0bplNYZ6m8+7Brve9d7qo7M1fVe+39mavo621RjaKOIKAABuESsrVS7OL2Vi\nbiHjs4sZn13IZPN5fHYxE7MLmWhuT1zx+cVLS+/567ZuKRnoakv/5Udna3YPdTXe62xLf1dr+jvb\n3t6ns3X1dV9nazpaW67T/wIfLnEFAAA3oKqqMjW/lAvTlzI2s5ALMwu5ML2QsZlLzTBqxNL47EIm\n5hYz0dxe63ajUpL+zrYMdbdloLs9I73tuWNbbwa72zLY1Z7B7mtFUuP1Rp8x2izEFQAAbAKXY2ls\nZiEXpi/lwszCNV5fjqhLGZ9dyOLytUupq60lQ91tGexuz1BPW3YOdGWwuy1D3Y1IGuxuf/vz5vNA\nV1tatgik9RBXAADwIamqKhOzizk3fSnnLjYeZy/Or76+MLOQ882zTWMza8dSb0drRnrbM9zTnt2D\nnfnI7oEM97ZnpKe9+X7H6uuh7vZ0tt0cl9ndaMQVAAB8j+YXl5uh1Iym6Us5NzV/VURdfv9awdTZ\ntiWjfR3Z2tuR3YOdeWB3f0Z6G4E03NO++los3VjEFQAANC2tVDk9MZc3J+fz1tT828/N142IunTN\nBR5KSUZ6OjLa15FtfR25c3tfRvs6Mtr79nujzUdvR6t7lG5C4goAgJteVVW5eGkpb02+HUpvR9Ol\nvDU1nzcm53Nh+lKq3/39q362vWVLtg90ZEd/Z+7Z0Z8fuvPtSLocT9v6OzLc3Z7Wli0b9DtkMxBX\nAADc8OYXl3NmYi5nJuZzZmIupyfmGtuTc3ljohFRswvL7/q5we627OjvzPb+zty7sz/z42/mYw/e\nvfrejoHODHW3OcvEByKuAADY1FZWqlyYWXg7mK6Mp2ZMXZhZuOpnSkm29XVk50BX7t7Zlx8+NJod\nzVi6/Ly9v/Nd9zIdOTKWw4/vu56/PW4i4goAgA21vFLlran5nBybzcnxuZwcm70qpM5MzmdhaeWq\nn+lub8nuwa7sGuzK/bsHsnuwM7ua27sHu7K9vzPtrS7R4/oSVwAAfKguL0d+cnw2J8fm8vrYbPP1\nbE6Nz+X0+FwWlt+Op1KS7X2d2T3UlQf2DObH7+9shNTA2/HU32VBCDYfcQUAwLrNLy7n5NhsI5yu\nOAP1ejOgpt+xut5Qd1v2Dnfn3p39+fR927NvuDt7h7qzd7g7uwY709Fq6XFuPOIKAIAPZG5hOSfG\nZnL8/GyOX5jJiQszOXZ+JicuzOaNyfmr9u1s27IaS08cHM7e4cbrxntd6ets26DfBXx4xBUAAKtm\nF5Zy4sJsjp+fyfELs1cF1JtTVwfUcE97Dox05+O3jeTA1p7sH+nOnmY8jfZ2uGyPW464AgC4xSwt\nr+T1sdkcPTeT756bztFzMzl2YSbHz8/k7MVLV+27tbc9+0d68ok7tubASHf2b+3JwZGe7BvpzkCX\ns09wJXEFAHCTmpxdzGvnpnP03HS+e26m+Tyd18dms7hcre63tbc9B7f25IfuGs2Bke4c2NqTAyON\nM1Eu34MPTlwBANzAlpZXcmp8bvUM1JXPV373U1tLyf6RntyxrTefvm9Hbh/tzW2jPbl9a28GugUU\n1EFcAQDcAJaWV3L8wmxefetivvPWdF49ezGvvjWdY+dnrlrGfKSnPbeN9uTH7t3eiKfR3tw22pu9\nQ11pbfG9T/BhElcAAJvI5Yh67Wwjor7zViOijp6fvupSvr3DXblrW18O3z2a20d7m4+eDHa3b+Do\n4dYmrgAANsDS8kpOjL19Juo7b13Ma2cbl/RdeSbqckR98u5tuXNbb+7a3pfbt/Wku90/42Cz8V8l\nAMCH7Pz0pbz8xsW8/OZUXmo+v3p2OgtL746oHz40mru29YkouAH5rxUAoCbzi8t57ex0Xn7zYl5+\nY6rx/OZUzk+/vbDEtr6
|
|||
|
|
"text/plain": [
|
|||
|
|
"<matplotlib.figure.Figure at 0x7fdda9490fd0>"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"metadata": {},
|
|||
|
|
"output_type": "display_data"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"plot(lambda x: 1 / (1 + math.exp(-x)))"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* Funkcja logistyczna przyjmuje wartości z przedziału $(0, 1)$."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "slide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"### Tangens hiperboliczny"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"$$ g(x) = \\tanh x = \\frac{e^{x} - e^{-x}}{e^{x} + e^{-x}} $$"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"#### Tangens hiperboliczny – wykres"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 4,
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"data": {
|
|||
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA1cAAAG2CAYAAACTRXz+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xd0HOW9h/Hvu7vq1bKKZVtyt9xx\nw6ZjUw0pQCgBQjc4cOEGktz0nnADaeSShCQQejVOaCZ0AqLjinuVu+Si3tuW9/6htRFGsmVrpFlJ\nz+ecPTuzO9r5kTNH8pPZnTXWWgEAAAAAOsfj9gAAAAAA0BsQVwAAAADgAOIKAAAAABxAXAEAAACA\nA4grAAAAAHAAcQUAAAAADnAkrowxDxpjio0xa9p5fpYxpsoYsyJ8+6kT+wUAAACASOFz6HUelvQX\nSY8eYpv3rLVfdGh/AAAAABBRHDlzZa19V1K5E68FAAAAAD1Rd37m6nhjzEpjzCvGmPHduF8AAAAA\n6HJOvS3wcJZLGmKtrTXGnCvpeUmj2trQGDNP0jxJio2NnZabm9tNI6KnCIVC8ni4Fgs+i+MC7eHY\nQFs4LtAejg20ZdOmTaXW2ozDbWestY7s0BgzVNK/rbUTOrDtdknTrbWlh9ouLy/Pbty40ZH50Hvk\n5+dr1qxZbo+BCMNxgfZwbKAtHBdoD8cG2mKMWWatnX647boly40xA4wxJrw8I7zfsu7YNwAAAAB0\nB0feFmiMeUrSLEnpxphCST+TFCVJ1tq/S7pI0k3GmICkBkmXWqdOmQEAAABABHAkrqy1lx3m+b+o\n5VLtAAAAANAr8Wk9AAAAAHAAcQUAAAAADiCuAAAAAMABxBUAAAAAOIC4AgAAAAAHEFcAAAAA4ADi\nCgAAAAAcQFwBAAAAgAOIKwAAAABwAHEFAAAAAA4grgAAAADAAcQVAAAAADiAuAIAAAAABxBXAAAA\nAOAA4goAAAAAHEBcAQAAAIADiCsAAAAAcABxBQAAAAAOIK4AAAAAwAHEFQAAAAA4gLgCAAAAAAcQ\nVwAAAADgAOIKAAAAABxAXAEAAACAA4grAAAAAHAAcQUAAAAADiCuAAAAAMABxBUAAAAAOIC4AgAA\nAAAHEFcAAAAA4ADiCgAAAAAcQFwBAAAAgAOIKwAAAABwAHEFAAAAAA4grgAAAADAAcQVAAAAADiA\nuAIAAAAABxBXAAAAAOAA4goAAAAAHEBcAQAAAIADiCsAAAAAcABxBQAAAAAOIK4AAAAAwAHEFQAA\nAAA4gLgCAAAAAAcQVwAAAADgAOIKAAAAABxAXAEAAACAA4grAAAAAHAAcQUAAAAADiCuAAAAAMAB\nxBUAAAAAOIC4AgAAAAAHEFcAAAAA4ADiCgAAAAAcQFwBAAAAgAOIKwAAAABwAHEFAAAAAA5wJK6M\nMQ8aY4qNMWvaed4YY/5kjCkwxqwyxkx1Yr8AAAAAECmcOnP1sKQ5h3j+HEmjwrd5kv7m0H4BAAAA\nICI4ElfW2ncllR9ik/MkPWpbfCwp1RiT7cS+AQAAACAS+LppP4Mk7Wq1Xhh+bE837R8AAABAL2St\nVchKwZBVyFoFQ1ZBaxUKhZfD68GQVSikT5ftp8+HPvPYZ19rdFZSh2fprrjqMGPMPLW8dVAZGRnK\nz893dyBEnNraWo4LfA7HBdrDsYG2cFygPX3x2LDWKmilYEjyh6SAtQqE1OrWsu4Ph4m/jecCISlg\nJX8oHCe25bHQ/tcOv37IhuMmvL7/uQNRc2Bd4SjSZ7YNHfSzofDzXWnuhOgOb9tdcVUkKafV+uDw\nY59jrb1P0n2SlJeXZ2fNmtXlw6Fnyc/PF8cFDsZxgfZwbKAtHBdoTyQcG6GQVYM/qPrmoOqbA6pr\nCqoxEFSjP6gmf0iN/v3r4WV/SA3+oJr8wQPr+7c/sE0g9Jnn/cGQmgMhNQVblq3DgeLzGPm8Rj6P\nJ3zfsuz1GEV5Tfi+Zd3n9SjGYz77Mwf9vNdjFOXxyOs1ivIYeT2eA6/jC697PZLHY+Q1LY97wvcH\nbsa0PO/Rp8/tf2z/zxx4TAceG5qeoJ929L/b2f8Z27VQ0i3GmPmSZkqqstbylkAAAAD0eNZa1TcH\nVdMYUE2jX9WNAVU3+lXTGFB9U0B1zcFP78OxVN/c/uP1zcGjmiPa51Gsz6PYKK9io7yKi/IqNsqj\nmCivUuKiFJsUo9gor2J8HsVEeRTl9Sja51G0t+UWtX+51X3UgXujaJ9HMZ95rO3to7xGxhiH/1fu\nGRyJK2PMU5JmSUo3xhRK+pmkKEmy1v5d0suSzpVUIKle0rVO7BcAAABwgj8YUkV9s4pqQlq0tUwV\n9X5VNTSruuHzwVTT6G95vGn/ekDB0OFP/UR7PYqP8Soh2qf4aK/iY3xKiPYqNT5aCTFexUe3rO9/\nfP99XDiWYsOx9Jlln/dAMHk8fTNoIokjcWWtvewwz1tJNzuxLwAAAOBQgiGr8rpmldQ0qbS2SRX1\nzaqoa1ZFvV+V9S33FfXNqmx1X9sU+PQFPvj4M69njJQY41NybJSSYlvus1NiNTo2UclxLY8lxX56\nn9zqPiHGp4Ron+KivYr2OfUtSIhUEXdBCwAAAOBgoZBVWatg2n9rWW9utdyk8rpmtXciKTnWp34J\n0UqNj1b/xGiNzExUanyU+sVHq198lHZvL9CJ0ye3PJYQ3RJI0T7OCqFDiCsAAAC4yh8MqbimSXur\nGrW3qlF7qhpa7qsbta+qUXuqGrWvulGBNoopxudRRlKM0hNjNLhfvKbkpiojMUbp4cfSE2OUltAS\nTilxUfJ5D332KL9pu04ald5V/6no5YgrAAAAdKlGf1CFFfXaVdGgwvJ6FVY0aFdFvYoqGrSnqlEl\ntU2fu1pdbJRH2SlxGpAcq5nD0jQgJVZZybHKSIo5EFPpidFKjPH12YsnIPIQVwAAAOgUa61Kapq0\ntbRO20vrtLNVQBVWNKikpukz20f7PBrcL06DUuM0ZkCyBqTEHrhlp8QqOzlOyXFEE3oe4goAAAAd\nUtXg17bSOm0rrdW2kjptLa3TtnBQ1bW6fLjXYzQwNVY5/eI1Oy9DOf3ilZMWr5y0OA3uF6+MxBg+\nw4ReibgCAADAZ5TWNmnT3hpt2lejTcW12ryvRltL6lRW13xgG4+RBveL17D0BB07NE3DMxI0tH+C\nhqUnKDsl9rCfbQJ6I+IKAACgj6pu9GvDnnBEHbjVqrxVRKXERWl0VqLOGp+lYekJGpaeqGHpLWei\nYnxeF6cHIg9xBQAA0AeU1DRp7e4qrd1dfeB+R1n9gecTY3walZWos8ZlaVRWkvKykjQ6K1EZSTF8\n9gnoIOIKAACglymuadSKnZVaXfRpTO2r/vSiErlp8Ro/MFkXTxus8QNTNHpAkgamxBJRQCcRVwAA\nAD1YUyCodbur9cnOSn2yq1Kf7KxQYUWDpJbPRY3MTNSJI9I1bmCyxg9M0biByUqJi3J5aqB3Iq4A\nAAB6kH3VjVq8rVzLd1bok52VWre7Ws3BkCRpYEqspuT20zUnDNWU3FSNH5ii2Cg+FwV0F+IKAAAg\nQllrVVjRoEXbyrV4W5kWbSs/8Dmp2CiPJg1O1bUnDdWUnH6akpuqrORYlycG+jbiCgAAIILsKq/X\nBwWlWrStXIu2lml3VaMkKTU+SscOTdOVxw3RjGFpGpudrCgudw5EFOIKAADARdWNfn1YUKb3C0r0\n/uZSbQ+fmUpPjNbMYf114/A0zRiWptGZSXzxLhDhiCsAAIBuFAiGtLKwUu9uKtV7m0u0srBKwZBV\nfLRXxw/vr6tPGKqTR6VrREYiV+8DehjiCgAAoItVN/r1zsYSvbWhWG9vLFZlvV/GSJMGp+qmU0fo\n5FHpmpLbT9E+3uYH9GTEFQAAQBfYUVanN9cX6z/r92nxtnIFQlb94qN0Wl6mThubqZNGpis1Ptrt\nMQE4iLgCAABwgLVW6/Z
|
|||
|
|
"text/plain": [
|
|||
|
|
"<matplotlib.figure.Figure at 0x7fdda93e9590>"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"metadata": {},
|
|||
|
|
"output_type": "display_data"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"plot(lambda x: math.tanh(x))"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* Tangens hiperboliczny przyjmuje wartości z przedziału $(-1, 1)$."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* Powstaje z funkcji logistytcznej przez przeskalowanie i przesunięcie."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* Daje szybszą zbieżność niż funkcja logistyczna dzięki temu, że przedział wartości jest symetryczny względem zera."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* Ze względu na ograniczony przedział wartości, obie funkcje (logistyczna i $\\tanh$) są podatne na problem zanikającego gradientu."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "slide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"### Problem zanikającego gradientu (*vanishing gradient problem*)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* Sigmoidalne funkcje aktywacji ograniczają wartości na wyjściach neuronów do niewielkich przedziałów ($(-1, 1)$, $(0, 1)$ itp.).\n",
|
|||
|
|
"* Jeżeli sieć ma wiele warstw, to podczas propagacji wstecznej mnożymy przez siebie wiele małych wartości → obliczony gradient jest mały.\n",
|
|||
|
|
"* Im więcej warstw, tym silniejszy efekt zanikania."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"#### Sposoby na zanikający gradient\n",
|
|||
|
|
"\n",
|
|||
|
|
"* Modyfikacja algorytmu optymalizacji (*RProp*, *RMSProp*)\n",
|
|||
|
|
"* Użycie innej funckji aktywacji (ReLU, softplus)\n",
|
|||
|
|
"* Dodanie warstw *dropout*\n",
|
|||
|
|
"* Nowe architektury (LSTM itp.)\n",
|
|||
|
|
"* Więcej danych, zwiększenie mocy obliczeniowej"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "slide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"### ReLU (*Rectifier Linear Unit*)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"$$ g(x) = \\max(0, x) $$"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"#### ReLU – wykres"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 5,
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"data": {
|
|||
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA1cAAAG2CAYAAACTRXz+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAGJhJREFUeJzt3X+sZOd91/HPN/6RRQk0AS+NYzvE\npdalpkBLLNcRVbkmTnFMFTelkWwhkQDVtgirBQmBi0UCgUitkABBItpVY9VJo6QWqZul2eIkda7d\nCNzaiZzWXmfdxQR2t4lN7CT0KmnM1g9/7Gx7vZ659+7Oc2fOzLxe0tXOj7PzPH98vfbb58zZaq0F\nAACA6bxk3hsAAABYBuIKAACgA3EFAADQgbgCAADoQFwBAAB0IK4AAAA66BJXVXVnVT1dVY9OeH+9\nqr5WVY+Mft7RY10AAIChuLDT5/x8kvckef82x/x6a+0HOq0HAAAwKF3OXLXWHkjybI/PAgAAWESz\n/M7V66vqc1X1q1X152e4LgAAwJ7rdVngTj6b5M+01jar6qYkv5zkqnEHVtWBJAeSZN++fa97zWte\nM6Mtsiief/75vOQl7sXCC5kLJjEbjGMukme+0fJ7/6/lWy6uvHJfzXs7g2E2GOeJJ574cmtt/07H\nVWuty4JV9dokv9Ja+85dHPuFJNe01r683XFra2vt6NGjXfbH8tjY2Mj6+vq8t8HAmAsmMRuMs8pz\n0VrLP//oo/mFB/93fvSvfltuv/HPpUpcnbHKs8FkVfWZ1to1Ox03kyyvqlfV6J/aqrp2tO4zs1gb\nAIDThBXsrS6XBVbVh5KsJ7mkqk4keWeSi5KktfYzSX44yd+vqlNJvpHkltbrlBkAADsSVrD3usRV\na+3WHd5/T07fqh0AgBkTVjAbvq0HALDEhBXMjrgCAFhSwgpmS1wBACwhYQWzJ64AAJaMsIL5EFcA\nAEtEWMH8iCsAgCUhrGC+xBUAwBIQVjB/4goAYMEJKxgGcQUAsMCEFQyHuAIAWFDCCoZFXAEALCBh\nBcMjrgAAFoywgmESVwAAC0RYwXCJKwCABSGsYNjEFQDAAhBWMHziCgBg4IQVLAZxBQAwYMIKFoe4\nAgAYKGEFi0VcAQAMkLCCxSOuAAAGRljBYhJXAAADIqxgcYkrAICBEFaw2MQVAMAACCtYfOIKAGDO\nhBUsB3EFADBHwgqWh7gCAJgTYQXLRVwBAMyBsILlI64AAGZMWMFyElcAADMkrGB5iSsAgBkRVrDc\nxBUAwAwIK1h+4goAYI8JK1gN4goAYA8JK1gd4goAYI8IK1gt4goAYA8IK1g94goAoDNhBatJXAEA\ndCSsYHWJKwCAToQVrDZxBQDQgbACxBUAwJSEFZCIKwCAqQgr4AxxBQBwnoQVsJW4AgA4D8IKOJu4\nAgA4R8IKGEdcAQCcA2EFTCKuAAB2SVgB2xFXAAC7IKyAnYgrAIAdCCtgN8QVAMA2hBWwW+IKAGAC\nYQWcC3EFADCGsALOlbgCADiLsALOh7gCANhCWAHnS1wBAIwIK2Aa4goAIMIKmJ64AgBWnrACehBX\nAMBKE1ZAL+IKAFhZwgroqUtcVdWdVfV0VT064f2qqv9QVceq6req6i/3WBcA4HwJK6C3Xmeufj7J\njdu8/6YkV41+DiT5T53WBQA4Z8IK2AsX9viQ1toDVfXabQ65Ocn7W2styYNV9YqqurS19sUe6wMA\n7FZrLR848lzuOy6sgL5m9Z2ry5Ic3/L8xOg1AICZOXPG6r7jp4QV0F2XM1c9VdWBnL50MPv378/G\nxsZ8N8TgbG5umgtexFwwidngjD86Y3UqN1zWct2+L+X++5+a97YYGH9mMI1ZxdXJJFdseX756LUX\naa0dTHIwSdbW1tr6+vqeb47FsrGxEXPB2cwFk5gNkq1nrE5fCnjdvi/l+uuvn/e2GCB/ZjCNWV0W\neCjJ3x7dNfC6JF/zfSsAYBbcvAKYlS5nrqrqQ0nWk1xSVSeSvDPJRUnSWvuZJIeT3JTkWJKvJ/k7\nPdYFANiOsAJmqdfdAm/d4f2W5B/0WAsAYDeEFTBrs7osEABgZoQVMA/iCgBYKsIKmBdxBQAsDWEF\nzJO4AgCWgrAC5k1cAQALT1gBQyCuAICFJqyAoRBXAMDCElbAkIgrAGAhCStgaMQVALBwhBUwROIK\nAFgowgoYKnEFACwMYQUMmbgCABaCsAKGTlwBAIMnrIBFIK4AgEETVsCiEFcAwGAJK2CRiCsAYJCE\nFbBoxBUAMDjCClhE4goAGBRhBSwqcQUADIawAhaZuAIABkFYAYtOXAEAcyesgGUgrgCAuRJWwLIQ\nVwDA3AgrYJmIKwBgLoQVsGzEFQAwc8IKWEbiCgCYKWEFLCtxBQDMjLAClpm4AgBmQlgBy05cAQB7\nTlgBq0BcAQB7SlgBq0JcAQB7RlgBq0RcAQB7QlgBq0ZcAQDdCStgFYkrAKArYQWsKnEFAHQjrIBV\nJq4AgC6EFbDqxBUAMDVhBSCuAIApCSuA08QVAHDehBXAHxFXAMB5EVYALySuAIBzJqwAXkxcAQDn\nRFgBjCeuAIBdE1YAk4krAGBXhBXA9sQVALAjYQWwM3EFAGxLWAHsjrgCACYSVgC7J64AgLGEFcC5\nEVcAwIsIK4BzJ64AgBcQVgDnR1wBAH9IWAGcP3EFACQRVgDTElcAgLAC6EBcAcCKE1YAfYgrAFhh\nwgqgH3EFACtKWAH0Ja4AYAUJK4D+xBUArBhhBbA3xBUArBBhBbB3usRVVd1YVUer6lhV3T7m/bdX\n1f+pqkdGPz/SY10AYPeEFcDeunDaD6iqC5K8N8kbk5xI8lBVHWqtHTnr0F9srd027XoAwLkTVgB7\nr8eZq2uTHGutPdlaey7Jh5Pc3OFzAYAOhBXAbEx95irJZUmOb3l+Isn3jDnub1bV9yV5Isk/aq0d\nH3NMqupAkgNJsn///mxsbHTYIstkc3PTXPAi5oJJVn02Wmv5wJHnct/xU7npyoty3b4v5f77n5r3\ntuZu1eeCycwG0+gRV7vxX5J8qLX2zar60SR3Jflr4w5srR1McjBJ1tbW2vr6+oy2yKLY2NiIueBs\n5oJJVnk2zpyxuu+4M1ZnW+W5YHtmg2n0uCzwZJIrtjy/fPTaH2qtPdNa++bo6c8leV2HdQGACVwK\nCDB7PeLqoSRXVdWVVXVxkluSHNp6QFVduuXpm5M83mFdAGAMYQUwH1NfFthaO1VVtyW5N8kFSe5s\nrT1WVe9K8nBr7VCSH6+qNyc5leTZJG+fdl0A4MWEFcD8dPnOVWvtcJLDZ732ji2PfzLJT/ZYCwAY\nT1gBzFeXv0QYAJgvYQUwf+IKABacsAIYBnEFAAtMWAEMh7gCgAUlrACGRVwBwAISVgDDI64AYMEI\nK4BhElcAsECEFcBwiSsAWBDCCmDYxBUALABhBTB84goABk5YASwGcQUAAyasABaHuAKAgRJWAItF\nXAHAAAkrgMUjrgBgYIQVwGISVwAwIMIKYHGJKwAYCGEFsNjEFQAMgLACWHziCgDmTFgBLAdxBQBz\nJKwAloe4AoA5EVYAy0VcAcAcCCuA5SOuAGDGhBXAchJXADBDwgpgeYkrAJgRYQWw3MQVAMyAsAJY\nfuIKAPaYsAJYDeIKAPaQsAJYHeIKAPaIsAJYLeIKAPaAsAJYPeIKADoTVgCrSVwBQEfCCmB1iSsA\n6ERYAaw2cQUAHQgrAMQVAExJWAGQiCsAmIqwAuAMcQUA50lYAbCVuAKA8yCsADibuAKAcySsABhH\nXAHAORBWAEwirgBgl4QVANsRVwCwC8IKgJ2IKwDYgbACYDfEFQBsQ1gBsFviCgAmEFYAnAtxBQBj\nCCsAzpW4AoCzCCsAzoe4AoAthBUA50tcAcCIsAJgGuIKACKsAJieuAJg5QkrAHoQVwCsNGEFQC/i\nCoCVJawA6ElcAbCShBU
|
|||
|
|
"text/plain": [
|
|||
|
|
"<matplotlib.figure.Figure at 0x7fdda936c6d0>"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"metadata": {},
|
|||
|
|
"output_type": "display_data"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"plot(lambda x: max(0, x))"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"#### ReLU – zalety\n",
|
|||
|
|
"* Mniej podatna na problem zanikającego gradientu (*vanishing gradient*) niż funkcje sigmoidalne, dzięki czemu SGD jest szybciej zbieżna.\n",
|
|||
|
|
"* Prostsze obliczanie gradientu.\n",
|
|||
|
|
"* Dzięki zerowaniu ujemnych wartości, wygasza neurony, „rozrzedzając” sieć (*sparsity*), co przyspiesza obliczenia."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"#### ReLU – wady\n",
|
|||
|
|
"* Dla dużych wartości gradient może „eksplodować”.\n",
|
|||
|
|
"* Za dużo „wygaszonych” neuronów może mieć negatywny wpływ na przepływ gradientu do poprzedzających warstw."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "slide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"### Inne funkcje aktywacji\n",
|
|||
|
|
"\n",
|
|||
|
|
"W literaturze zaproponowano też inne funkcje aktywacji, które są lepsze od ReLU pod niektórymi względami:\n",
|
|||
|
|
"* softplus\n",
|
|||
|
|
"* ELU\n",
|
|||
|
|
"* leaky ReLU\n",
|
|||
|
|
"* SiLU / swish\n",
|
|||
|
|
"\n",
|
|||
|
|
"W praktyce jednak ReLU pozostaje najpopularniejszą funkcją aktywacji ze względu na jej prostotę, skuteczność i szybkość obliczania."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "slide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"## 11.3. Wielowarstwowe sieci neuronowe\n",
|
|||
|
|
"\n",
|
|||
|
|
"czyli _Artificial Neural Networks_ (ANN) lub _Multi-Layer Perceptrons_ (MLP)"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"![Rys. 9.5. Wielowarstwowa sieć neuronowa](nn1.png \"Rys. 9.5. Wielowarstwowa sieć neuronowa\")"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"#### Architektura sieci\n",
|
|||
|
|
"\n",
|
|||
|
|
"* Sieć neuronowa jako graf neuronów. \n",
|
|||
|
|
"* Organizacja sieci przez warstwy.\n",
|
|||
|
|
"* Najczęściej stosowane są sieci jednokierunkowe i gęste.\n",
|
|||
|
|
"* $n$-warstwowa sieć neuronowa ma $n+1$ warstw (nie liczymy wejścia).\n",
|
|||
|
|
"* Rozmiary sieci określane poprzez liczbę neuronów lub parametrów."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "slide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"### Sieć neuronowa jednokierunkowa (*feedforward*)\n",
|
|||
|
|
"\n",
|
|||
|
|
"* Mając daną $n$-warstwową sieć neuronową oraz jej parametry $\\Theta^{(1)}, \\ldots, \\Theta^{(L)} $ oraz $\\beta^{(1)}, \\ldots, \\beta^{(L)} $, liczymy:<br/><br/> \n",
|
|||
|
|
"$$a^{(l)} = g^{(l)}\\left( a^{(l-1)} \\Theta^{(l)} + \\beta^{(l)} \\right). $$"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"![Rys. 11.6. Wielowarstwowa sieć neuronowa - feedforward](nn2.png \"Rys. 11.6. Wielowarstwowa sieć neuronowa - feedforward\")"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* Funkcje $g^{(l)}$ to tzw. **funkcje aktywacji**.<br/>\n",
|
|||
|
|
"Dla $i = 0$ przyjmujemy $a^{(0)} = \\mathrm{x}$ (wektor wierszowy cech) oraz $g^{(0)}(x) = x$ (identyczność)."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* Parametry $\\Theta$ to wagi na połączeniach miedzy neuronami dwóch warstw.<br/>\n",
|
|||
|
|
"Rozmiar macierzy $\\Theta^{(l)}$, czyli macierzy wag na połączeniach warstw $a^{(l-1)}$ i $a^{(l)}$, to $\\dim(a^{(l-1)}) \\times \\dim(a^{(l)})$."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* Parametry $\\beta$ zastępują tutaj dodawanie kolumny z jedynkami do macierzy cech.<br/>Macierz $\\beta^{(l)}$ ma rozmiar równy liczbie neuronów w odpowiedniej warstwie, czyli $1 \\times \\dim(a^{(l)})$."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "fragment"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* **Klasyfikacja**: dla ostatniej warstwy $L$ (o rozmiarze równym liczbie klas) przyjmuje się $g^{(L)}(x) = \\mathop{\\mathrm{softmax}}(x)$.\n",
|
|||
|
|
"* **Regresja**: pojedynczy neuron wyjściowy jak na obrazku. Funkcją aktywacji może wtedy być np. funkcja identycznościowa."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* Pozostałe funkcje aktywacji najcześciej mają postać sigmoidy, np. sigmoidalna, tangens hiperboliczny.\n",
|
|||
|
|
"* Mogą mieć też inny kształt, np. ReLU, leaky ReLU, maxout."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "slide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"### Uczenie wielowarstwowych sieci neuronowych"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"Mając algorytm SGD oraz gradienty wszystkich wag, moglibyśmy trenować każdą sieć."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"* Niech:\n",
|
|||
|
|
"$$\\Theta = (\\Theta^{(1)},\\Theta^{(2)},\\Theta^{(3)},\\beta^{(1)},\\beta^{(2)},\\beta^{(3)})$$\n",
|
|||
|
|
"\n",
|
|||
|
|
"* Funkcja sieci neuronowej z grafiki:\n",
|
|||
|
|
"\n",
|
|||
|
|
"$$\\small h_\\Theta(x) = \\tanh(\\tanh(\\tanh(x\\Theta^{(1)}+\\beta^{(1)})\\Theta^{(2)} + \\beta^{(2)})\\Theta^{(3)} + \\beta^{(3)})$$\n",
|
|||
|
|
"* Funkcja kosztu dla regresji:\n",
|
|||
|
|
"$$J(\\Theta) = \\dfrac{1}{2m} \\sum_{i=1}^{m} (h_\\Theta(x^{(i)})- y^{(i)})^2 $$"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"Jak obliczymy gradienty?\n",
|
|||
|
|
"\n",
|
|||
|
|
"$$\\nabla_{\\Theta^{(l)}} J(\\Theta) = ? \\quad \\nabla_{\\beta^{(l)}} J(\\Theta) = ?$$\n",
|
|||
|
|
"\n",
|
|||
|
|
"* Postać funkcji kosztu zależna od wybranej architektury sieci oraz funkcji aktywacji."
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "markdown",
|
|||
|
|
"metadata": {
|
|||
|
|
"slideshow": {
|
|||
|
|
"slide_type": "subslide"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"source": [
|
|||
|
|
"$$\\small J(\\Theta) = \\frac{1}{2}(a^{(L)} - y)^2 $$\n",
|
|||
|
|
"$$\\small \\dfrac{\\partial}{\\partial a^{(L)}} J(\\Theta) = a^{(L)} - y$$\n",
|
|||
|
|
"\n",
|
|||
|
|
"$$\\small \\tanh^{\\prime}(x) = 1 - \\tanh^2(x)$$"
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"metadata": {
|
|||
|
|
"celltoolbar": "Slideshow",
|
|||
|
|
"kernelspec": {
|
|||
|
|
"display_name": "Python 3 (ipykernel)",
|
|||
|
|
"language": "python",
|
|||
|
|
"name": "python3"
|
|||
|
|
},
|
|||
|
|
"language_info": {
|
|||
|
|
"codemirror_mode": {
|
|||
|
|
"name": "ipython",
|
|||
|
|
"version": 3
|
|||
|
|
},
|
|||
|
|
"file_extension": ".py",
|
|||
|
|
"mimetype": "text/x-python",
|
|||
|
|
"name": "python",
|
|||
|
|
"nbconvert_exporter": "python",
|
|||
|
|
"pygments_lexer": "ipython3",
|
|||
|
|
"version": "3.10.6"
|
|||
|
|
},
|
|||
|
|
"livereveal": {
|
|||
|
|
"start_slideshow_at": "selected",
|
|||
|
|
"theme": "white"
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"nbformat": 4,
|
|||
|
|
"nbformat_minor": 4
|
|||
|
|
}
|