{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Chain_rule.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/PacktPublishing/Hands-On-Computer-Vision-with-PyTorch/blob/master/Chapter01/Chain_rule.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:23:27.984719Z",
          "start_time": "2020-09-24T13:23:27.929962Z"
        },
        "colab_type": "code",
        "id": "i5WpgCBbm_Jc",
        "colab": {}
      },
      "source": [
        "from copy import deepcopy\n",
        "import numpy as np\n",
        "def line():\n",
        "    print('='*80)\n",
        "def feed_forward(inputs, outputs, weights):     \n",
        "    pre_hidden = np.dot(inputs,weights[0])+ weights[1]\n",
        "    hidden = 1/(1+np.exp(-pre_hidden))\n",
        "    out = np.dot(hidden, weights[2]) + weights[3]\n",
        "    mean_squared_error = np.mean(np.square(out - outputs))\n",
        "    return mean_squared_error"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:23:27.989715Z",
          "start_time": "2020-09-24T13:23:27.985804Z"
        },
        "colab_type": "code",
        "id": "YKQpTWlSnCwc",
        "colab": {}
      },
      "source": [
        "def update_weights(inputs, outputs, weights, lr):\n",
        "    original_weights = deepcopy(weights)\n",
        "    temp_weights = deepcopy(weights)\n",
        "    updated_weights = deepcopy(weights)\n",
        "    original_loss = feed_forward(inputs, outputs, original_weights)\n",
        "    for i, layer in enumerate(original_weights):\n",
        "        for index, weight in np.ndenumerate(layer):\n",
        "            temp_weights = deepcopy(weights)\n",
        "            temp_weights[i][index] += 0.0001\n",
        "            _loss_plus = feed_forward(inputs, outputs, temp_weights)\n",
        "            grad = (_loss_plus - original_loss)/(0.0001)\n",
        "            updated_weights[i][index] -= grad*lr\n",
        "    return updated_weights"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:23:28.000882Z",
          "start_time": "2020-09-24T13:23:27.991191Z"
        },
        "colab_type": "code",
        "id": "Vp9I03hRnh2G",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        },
        "outputId": "6b284e11-c5a8-411e-ba4b-4fd81e466235"
      },
      "source": [
        "x = np.array([[1,1]]); y = np.array([[0]])  \n",
        "W = [\n",
        "    np.array([[-0.0053, 0.3793],\n",
        "              [-0.5820, -0.5204],\n",
        "              [-0.2723, 0.1896]], dtype=np.float32).T, \n",
        "    np.array([-0.0140, 0.5607, -0.0628], dtype=np.float32), \n",
        "    np.array([[ 0.1528, -0.1745, -0.1135]], dtype=np.float32).T, \n",
        "    np.array([-0.5516], dtype=np.float32)\n",
        "]\n",
        "line()\n",
        "print('Loss:'.upper())\n",
        "print(feed_forward(x,y,W))\n",
        "line()\n",
        "print('Weights:'.upper())\n",
        "[print(w) for w in W]\n",
        "line()\n",
        "print('Updated Weights:'.upper())\n",
        "for epx in range(1):\n",
        "    updated_weights = update_weights(x,y,W,1)\n",
        "[print(w) for w in updated_weights];"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "================================================================================\n",
            "LOSS:\n",
            "0.33455008989960927\n",
            "================================================================================\n",
            "WEIGHTS:\n",
            "[[-0.0053 -0.582  -0.2723]\n",
            " [ 0.3793 -0.5204  0.1896]]\n",
            "[-0.014   0.5607 -0.0628]\n",
            "[[ 0.1528]\n",
            " [-0.1745]\n",
            " [-0.1135]]\n",
            "[-0.5516]\n",
            "================================================================================\n",
            "UPDATED WEIGHTS:\n",
            "[[ 0.03748801 -0.62894595 -0.30494714]\n",
            " [ 0.42208242 -0.5673459   0.156948  ]]\n",
            "[ 0.02878801  0.51375407 -0.09545201]\n",
            "[[0.8341824 ]\n",
            " [0.25095794]\n",
            " [0.4228859 ]]\n",
            "[0.60529804]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "3llIoQD7xMJf"
      },
      "source": [
        "### Chain Rule\n",
        "Calculate the updated weight value using Chain rule"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:23:28.004119Z",
          "start_time": "2020-09-24T13:23:28.001851Z"
        },
        "colab_type": "code",
        "id": "fSSV-hAcufCd",
        "colab": {}
      },
      "source": [
        "pre_hidden = np.dot(x,W[0])+ W[1]\n",
        "hidden = 1/(1+np.exp(-pre_hidden))\n",
        "predicted_value = np.dot(hidden, W[2]) + W[3]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:23:28.008250Z",
          "start_time": "2020-09-24T13:23:28.004921Z"
        },
        "colab_type": "code",
        "id": "juDL0c_fwp0c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "2cbf40fa-41ad-4b42-bc94-56b2496f2dfe"
      },
      "source": [
        "tmp = W[0][0][0] - (-2*(0-(predicted_value[0][0]))*(W[2][0][0])*hidden[0,0]*(1-hidden[0,0])*x[0][0])\n",
        "print(tmp, updated_weights[0][0][0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.03748860333147175 0.037488014\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:23:28.012629Z",
          "start_time": "2020-09-24T13:23:28.009257Z"
        },
        "colab_type": "code",
        "id": "bIzE0_CD4ePv",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "8f64a525-9fc3-4e69-efa6-29538785a885"
      },
      "source": [
        "tmp = W[0][0][1] - (-2*(0-(predicted_value[0][0]))*(W[2][1][0])*hidden[0,1]*(1-hidden[0,1])*x[0][0])\n",
        "print(tmp, updated_weights[0][0][1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-0.6289373468565382 -0.62894595\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:23:28.016777Z",
          "start_time": "2020-09-24T13:23:28.013578Z"
        },
        "colab_type": "code",
        "id": "eBYFpMD58Zg4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "8182e49a-3300-49f1-edf2-5e29d69946e9"
      },
      "source": [
        "tmp = W[0][0][2] - (-2*(0-(predicted_value[0][0]))*(W[2][2][0])*hidden[0,2]*(1-hidden[0,2])*x[0][0])\n",
        "print(tmp, updated_weights[0][0][0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-0.304951263947996 0.037488014\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:23:28.021124Z",
          "start_time": "2020-09-24T13:23:28.018197Z"
        },
        "colab_type": "code",
        "id": "e5bsJzF88ZrM",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "021e7318-fc42-49f9-89d2-5565d362498f"
      },
      "source": [
        "tmp = W[0][1][0] - (-2*(0-(predicted_value[0][0]))*(W[2][0][0])*hidden[0,0]*(1-hidden[0,0])*x[0][1])\n",
        "print(tmp, updated_weights[0][1][0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.42208860145914084 0.42208242\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:23:28.025464Z",
          "start_time": "2020-09-24T13:23:28.022264Z"
        },
        "colab_type": "code",
        "id": "coHgQaMI9R3N",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "021e7318-fc42-49f9-89d2-5565d362498f"
      },
      "source": [
        "tmp = W[0][1][1] - (-2*(0-(predicted_value[0][0]))*(W[2][1][0])*hidden[0,1]*(1-hidden[0,1])*x[0][1])\n",
        "print(tmp, updated_weights[0][1][1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-0.5673373173880019 -0.5673459\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:23:28.030294Z",
          "start_time": "2020-09-24T13:23:28.026871Z"
        },
        "colab_type": "code",
        "id": "XEbiE6xc9R3P",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "021e7318-fc42-49f9-89d2-5565d362498f"
      },
      "source": [
        "tmp = W[0][1][2] - (-2*(0-(predicted_value[0][0]))*(W[2][2][0])*hidden[0,2]*(1-hidden[0,2])*x[0][1])\n",
        "print(tmp, updated_weights[0][1][2])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.15694874675699821 0.156948\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:23:28.035074Z",
          "start_time": "2020-09-24T13:23:28.031407Z"
        },
        "colab_type": "code",
        "id": "sD1NsccW9R3R",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "021e7318-fc42-49f9-89d2-5565d362498f"
      },
      "source": [
        "tmp = W[1][0] - (-2*(0-(predicted_value[0][0]))*hidden[0,0]*(1-hidden[0,0])*W[2][0][0])\n",
        "print(tmp, updated_weights[1][0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.028788602743620932 0.028788012\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:23:28.038822Z",
          "start_time": "2020-09-24T13:23:28.036075Z"
        },
        "colab_type": "code",
        "id": "90aX0GVu9R3U",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "021e7318-fc42-49f9-89d2-5565d362498f"
      },
      "source": [
        "tmp = W[1][1] - (-2*(0-(predicted_value[0][0]))*hidden[0,1]*(1-hidden[0,1])*W[2][1][0])\n",
        "print(tmp, updated_weights[1][1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.5137626696420274 0.51375407\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:23:28.042770Z",
          "start_time": "2020-09-24T13:23:28.039723Z"
        },
        "colab_type": "code",
        "id": "vP3P_DvO9R3W",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "021e7318-fc42-49f9-89d2-5565d362498f"
      },
      "source": [
        "tmp = W[1][2] - (-2*(0-(predicted_value[0][0]))*hidden[0,2]*(1-hidden[0,2])*W[2][2][0])\n",
        "print(tmp, updated_weights[1][2])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-0.0954512566166247 -0.09545201\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:23:28.046332Z",
          "start_time": "2020-09-24T13:23:28.043797Z"
        },
        "colab_type": "code",
        "id": "D0tTvkLV9R3Z",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "021e7318-fc42-49f9-89d2-5565d362498f"
      },
      "source": [
        "tmp = W[2][0][0]-(-2*(0-(predicted_value[0][0]))*hidden[0][0])\n",
        "print(tmp, updated_weights[2][0][0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.8342055621416937 0.8341824\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:23:28.050423Z",
          "start_time": "2020-09-24T13:23:28.047373Z"
        },
        "colab_type": "code",
        "id": "eJkFiIgG9R3b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "021e7318-fc42-49f9-89d2-5565d362498f"
      },
      "source": [
        "tmp = W[2][1][0]-(-2*(0-(predicted_value[0][0]))*hidden[0][1])\n",
        "print(tmp, updated_weights[2][1][0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.2509642654210383 0.25095794\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:23:28.053986Z",
          "start_time": "2020-09-24T13:23:28.051400Z"
        },
        "colab_type": "code",
        "id": "YQ6B8fA-9R3d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "021e7318-fc42-49f9-89d2-5565d362498f"
      },
      "source": [
        "tmp = W[2][2][0]-(-2*(0-(predicted_value[0][0]))*hidden[0][2])\n",
        "print(tmp, updated_weights[2][2][0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.422898309408289 0.4228859\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:23:28.057470Z",
          "start_time": "2020-09-24T13:23:28.054896Z"
        },
        "colab_type": "code",
        "id": "DYInbxAt9R3g",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "021e7318-fc42-49f9-89d2-5565d362498f"
      },
      "source": [
        "tmp = W[3][0]-(-2*(0-(predicted_value[0][0])))\n",
        "print(tmp, updated_weights[3][0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.6052061234525776 0.60529804\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "7LtMWS6c3nBP",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}