Przetwarzanie_tekstu/projekt/T5_sms_spam.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZXsOR6oJOJbd"
      },
      "source": [
        "# Instalacja pakietów"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8l0hzptKNiZS",
        "outputId": "00b4e80b-9d2a-42f1-e087-1412429b63bd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.26.1-py3-none-any.whl (6.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.3/6.3 MB\u001b[0m \u001b[31m39.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting datasets\n",
            "  Downloading datasets-2.9.0-py3-none-any.whl (462 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m462.8/462.8 KB\u001b[0m \u001b[31m22.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.8/dist-packages (1.13.1+cu116)\n",
            "Collecting sentencepiece\n",
            "  Downloading sentencepiece-0.1.97-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m28.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from transformers) (23.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (1.21.6)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m23.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.8/dist-packages (from transformers) (4.64.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from transformers) (3.9.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (2022.6.2)\n",
            "Collecting huggingface-hub<1.0,>=0.11.0\n",
            "  Downloading huggingface_hub-0.12.1-py3-none-any.whl (190 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m190.3/190.3 KB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from transformers) (2.25.1)\n",
            "Collecting responses<0.19\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Collecting multiprocess\n",
            "  Downloading multiprocess-0.70.14-py38-none-any.whl (132 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.0/132.0 KB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.8/dist-packages (from datasets) (9.0.0)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (213 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m213.0/213.0 KB\u001b[0m \u001b[31m14.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: dill<0.3.7 in /usr/local/lib/python3.8/dist-packages (from datasets) (0.3.6)\n",
            "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.8/dist-packages (from datasets) (2023.1.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.8/dist-packages (from datasets) (3.8.4)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.8/dist-packages (from datasets) (1.3.5)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.8/dist-packages (from torch) (4.5.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (3.0.1)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.3.3)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.8.2)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (22.2.0)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (4.0.2)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (6.0.4)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (4.0.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2.10)\n",
            "Collecting urllib3<1.27,>=1.21.1\n",
            "  Downloading urllib3-1.26.14-py2.py3-none-any.whl (140 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m140.6/140.6 KB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets) (2022.7.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.8/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: tokenizers, sentencepiece, xxhash, urllib3, multiprocess, responses, huggingface-hub, transformers, datasets\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 1.24.3\n",
            "    Uninstalling urllib3-1.24.3:\n",
            "      Successfully uninstalled urllib3-1.24.3\n",
            "Successfully installed datasets-2.9.0 huggingface-hub-0.12.1 multiprocess-0.70.14 responses-0.18.0 sentencepiece-0.1.97 tokenizers-0.13.2 transformers-4.26.1 urllib3-1.26.14 xxhash-3.2.0\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers datasets torch sentencepiece"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dhN0rmb5Oi3d"
      },
      "source": [
        "# Załadowanie datasetu"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "tnaDkwZ2Pbnn"
      },
      "outputs": [],
      "source": [
        "from datasets import load_dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 263,
          "referenced_widgets": [
            "c396a3f65bb947ffa33130c424d9d93b",
            "fe2dd0bc42b84c5890d2c3dccaf66992",
            "245239c79fa74387bed598565bbc24a4",
            "926ce941393e4004bec99d38e82ea879",
            "65433acf8e5345d580c6bf8c949c0064",
            "293dc96088f942a3afc9af735d4c7117",
            "a7dc0ef4c814401f8b2ff982063b66cc",
            "790101eedd824ab893a7c7a1c0039163",
            "6889bbe799b54d9eade42250c2e5caa6",
            "5cc03bba39a74eaab7da908c6c24b1fb",
            "db67393d589c4d5bbbdab58adf51f970",
            "2c5a4622661a4465910b5f1f95bea742",
            "3d4560baaec44c40b5d5c27ed8eba68a",
            "dffa3d04bfe548e9aea5d4327ecca77a",
            "01ce10b9b22f48839824dda0a40ec5e8",
            "b839f01904ce41398b5286a801bbf4a7",
            "e36e4ad05a5040a4b67e0a133156358e",
            "892dbc4c003941409202f31523589835",
            "4a82770611cc4ba29aea5c462ce0c5be",
            "a3767cd0d3cd454595230f7933a0b2fe",
            "a77ad82dae14426b97ecc94166511b5c",
            "98e29231304e4d9ba32e0368ec5b3fd5",
            "d64108ba247a4ac5a93b3bdeede7fd9a",
            "aea219a8097b4d989664c09fcff9eb93",
            "b2721b8b11ed42e29c623a10a8c8e13f",
            "3d9d73ddd88446c5aeef5b2362bb878b",
            "b955aea22f9e4ce2882b5e722ae1dda8",
            "2a1ebe1ed1c64921bb7cdb0ed1e57b2b",
            "9980c1ddad4a475e97130c0efc2f3efe",
            "57feff796f4241f292cb4617ed85cfe0",
            "9297c35ec1f7494caab95714d65e34ab",
            "dd110977a96d4f19b37e923c4296fbdd",
            "953c5391bd5e49ad9b3752794929e08c",
            "b8c9fbbce7b84bf4a2f5b90c9d35ce0f",
            "51ab0b4ff03f4bdab8f5ba3fef868d4a",
            "fca8377c5d6e480180d22925109d431c",
            "a8854ee753ff4d179718690c3361b5a0",
            "72918e6a9c7e46f291f85c7adf237eb9",
            "fd3d449f08554666887c54c329baecd9",
            "8c83da8d3d54491a9f4c535bdb5611e1",
            "fb3f2df569ff42c89dea846f0df4b62f",
            "e298b598c6474c3fa2b7bada616110cc",
            "7838193eee55441d865d6f056454b841",
            "9ff537b0b2c745968c7d22e90cbe3894",
            "6b52b5d926bc43a3b70238c4fbfad7da",
            "733b55697ed8437c8d1d180756f7d3bb",
            "0d67db8eb2504316819ba0b4691aecb0",
            "6c402b861e4b49ad8b7d07c053e069bd",
            "0ea8f8e157634292b17babbdee9f46e0",
            "5016ba950fa447ae8d0f3a8191c8a34c",
            "8af86d8fcbbc41bd9e4bfee243bdb759",
            "16ddb3f4068040c9bbe5fa74c1b0177c",
            "f1424f2fde924cb88e05fc4a98d8e354",
            "afd518d5660e49b599d930c7f6040f87",
            "8a7c3caf8629499fbff1966e7544fc85",
            "bca3b484a8374617836c78a5a7247f19",
            "a6ad21ce2df942cc87a7009a76913bd7",
            "991524a5133c4ae9a1e6067be42f6ee4",
            "d4ed92b329c74cda9e38dc380fee1b71",
            "0e7dd3e98f64447dbe54df8d80fed381",
            "3034d7c759324b91bf84b704a023dd77",
            "4c3563f49ed0497ab2dc8739e4570d5b",
            "5788c4c9888e4def886297d645c670a3",
            "b713a8211c324197a5b66b05b4bfbb10",
            "9ab2665c20b8408490cc3fa43762b225",
            "b640800aca9f431883d8c309dd3b1daf"
          ]
        },
        "id": "cCiAuRqrOkvV",
        "outputId": "f0e3ddd0-5cc7-47e2-9910-8b6b84cbd896"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/3.21k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c396a3f65bb947ffa33130c424d9d93b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading metadata:   0%|          | 0.00/1.69k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2c5a4622661a4465910b5f1f95bea742"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading readme:   0%|          | 0.00/4.87k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d64108ba247a4ac5a93b3bdeede7fd9a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading and preparing dataset sms_spam/plain_text to /root/.cache/huggingface/datasets/sms_spam/plain_text/1.0.0/53f051d3b5f62d99d61792c91acefe4f1577ad3e4c216fb0ad39e30b9f20019c...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/203k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b8c9fbbce7b84bf4a2f5b90c9d35ce0f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split:   0%|          | 0/5574 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6b52b5d926bc43a3b70238c4fbfad7da"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset sms_spam downloaded and prepared to /root/.cache/huggingface/datasets/sms_spam/plain_text/1.0.0/53f051d3b5f62d99d61792c91acefe4f1577ad3e4c216fb0ad39e30b9f20019c. Subsequent calls will reuse this data.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bca3b484a8374617836c78a5a7247f19"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "dataset = load_dataset(\"sms_spam\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JKFHPko3OnAV",
        "outputId": "6c5513f7-90f2-4977-a938-539c6f623aaa"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'sms': 'Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...\\n',\n",
              " 'label': 0}"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "dataset['train'][0]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l140vJrgYxPr"
      },
      "source": [
        "# Modyfikacja datasetu - klasyfikacja"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1boUF-YiY3_y",
        "outputId": "23bb86a0-9015-46b4-b36e-84007cad246e"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'sms': 'binary classification: Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...',\n",
              " 'label': '0'}"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "parsed_dataset = []\n",
        "\n",
        "for row in dataset['train']:\n",
        "  text = \"binary classification: \" + row['sms'].replace(\"\\n\", \"\")\n",
        "  new_row = {}\n",
        "  new_row['sms'] = text\n",
        "  if row['label'] == 0:\n",
        "    new_row['label'] = \"0\"\n",
        "  else:\n",
        "    new_row['label'] = \"1\"\n",
        "  parsed_dataset.append(new_row)\n",
        "\n",
        "parsed_dataset[0]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O-J-jBDxPJcn"
      },
      "source": [
        "# Tokenizer T5"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "P23AYPX1PZ6g"
      },
      "outputs": [],
      "source": [
        "from transformers import T5Tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203,
          "referenced_widgets": [
            "8263f80bfd30477389a7d24450a41aa9",
            "d46c7aa929e24e5da4a3508e1ec82795",
            "84a7f048c34541708b01a64726037e94",
            "c545335c355d43c2b158eb7ea1032b68",
            "45f93d1c8d574efc8573cd9c30be2fa4",
            "6bc7141208cf4f2787890daa6dd900b5",
            "5fd95d4231824740a4df2c7d7cb015a0",
            "154741e5c910415f9e87da6cb5e1c578",
            "7dda689c19df4ff9b854cede266b804b",
            "863b386cb073419e96e8ae0d01554a36",
            "663902a693b5405c85be17bbe46e0650",
            "e76d348dd73d4e88994fa53449b69a0c",
            "bd65017865934166878adf8aa6c352c9",
            "0dc9cdebd0bd480d86fd2b7151f8617f",
            "ac095333a156479bbba127d424b48943",
            "2b2a144c8b434a0eb6b91c532965a956",
            "9674fadc0e6448f1ad34f2e47d6dec14",
            "1ed148605a1c4d97a8ff1bbab36b0f8e",
            "3ed7755f2175486d8407dea96ecd8898",
            "9afc3775ccc440c4b58e606e5daa9e75",
            "9d94d64ee8c4465489f4b645080030e5",
            "4fdd5febab8d42e6b79438a69d39622e"
          ]
        },
        "id": "q5Jz0E_oPMBr",
        "outputId": "1c5a4105-22c9-41d1-9d46-19120868ae9e"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)ve/main/spiece.model:   0%|          | 0.00/792k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8263f80bfd30477389a7d24450a41aa9"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/1.21k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e76d348dd73d4e88994fa53449b69a0c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/transformers/models/t5/tokenization_t5.py:163: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5.\n",
            "For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`.\n",
            "- Be aware that you SHOULD NOT rely on t5-base automatically truncating your input to 512 when padding/encoding.\n",
            "- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding.\n",
            "- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value.\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "tokenizer = T5Tokenizer.from_pretrained('t5-base')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dfxJQpoePsvI",
        "outputId": "a4b4cfa8-5334-4be6-c3ec-124840ecdcfa"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original:  binary classification: Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...\n",
            "Tokenized:  ['▁binary', '▁classification', ':', '▁Go', '▁until', '▁jur', 'ong', '▁point', ',', '▁crazy', '.', '.', '▁Available', '▁only', '▁in', '▁bug', 'is', '▁', 'n', '▁great', '▁world', '▁la', '▁', 'e', '▁buffet', '...', '▁Cine', '▁there', '▁got', '▁', 'a', 'more', '▁wa', 't', '...']\n",
            "Token IDs:  [14865, 13774, 10, 1263, 552, 10081, 2444, 500, 6, 6139, 5, 5, 8144, 163, 16, 8143, 159, 3, 29, 248, 296, 50, 3, 15, 15385, 233, 17270, 132, 530, 3, 9, 3706, 8036, 17, 233]\n"
          ]
        }
      ],
      "source": [
        "sms = parsed_dataset[0]['sms']\n",
        "print('Original: ', sms)\n",
        "print('Tokenized: ', tokenizer.tokenize(sms))\n",
        "print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sms)))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UpluhM8cU5Ir"
      },
      "source": [
        "# Check maximum lenght of a sentence"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7uNUkixPU85O",
        "outputId": "b34a2f27-9478-4fc9-cdbb-23081472ec92"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Max sentence length:  341\n"
          ]
        }
      ],
      "source": [
        "max_len = 0\n",
        "\n",
        "for sentence in parsed_dataset:\n",
        "    input_ids = tokenizer.encode(sentence['sms'], add_special_tokens=True)\n",
        "    max_len = max(max_len, len(input_ids))\n",
        "\n",
        "print('Max sentence length: ', max_len)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lj0issBznZfK",
        "outputId": "d406d5a6-e278-47aa-b03e-8ee33c5871ac"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Max sentence length:  3\n"
          ]
        }
      ],
      "source": [
        "max_label_len = 0\n",
        "\n",
        "for sentence in parsed_dataset:\n",
        "    input_ids = tokenizer.encode(sentence['label'], add_special_tokens=True)\n",
        "    max_label_len = max(max_label_len, len(input_ids))\n",
        "\n",
        "print('Max sentence length: ', max_label_len)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nfw62HdgSERb"
      },
      "source": [
        "# Pre train tokenization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "KTXYalS1VLqH"
      },
      "outputs": [],
      "source": [
        "import torch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z28QYfLnSGxR",
        "outputId": "aa3c2dce-488c-48a5-f47d-18026ac678d6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original:  {'sms': 'binary classification: Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...', 'label': '0'}\n",
            "Token IDs: tensor([14865, 13774,    10,  1263,   552, 10081,  2444,   500,     6,  6139,\n",
            "            5,     5,  8144,   163,    16,  8143,   159,     3,    29,   248,\n",
            "          296,    50,     3,    15, 15385,   233, 17270,   132,   530,     3,\n",
            "            9,  3706,  8036,    17,   233,     1,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0])\n",
            "Label token IDs: tensor([  3, 632,   1])\n"
          ]
        }
      ],
      "source": [
        "input_ids = []\n",
        "target_ids = []\n",
        "attention_masks = []\n",
        "\n",
        "for sentence in parsed_dataset:\n",
        "    encoded_dict = tokenizer.encode_plus(\n",
        "                        sentence['sms'],\n",
        "                        add_special_tokens = True,\n",
        "                        max_length = 341,\n",
        "                        padding = 'max_length',\n",
        "                        truncation=True,\n",
        "                        return_attention_mask = True,\n",
        "                        return_tensors = 'pt',\n",
        "                   )\n",
        "    \n",
        "    encoded_target_dict = tokenizer.encode_plus(\n",
        "                        sentence['label'],\n",
        "                        add_special_tokens = True,\n",
        "                        max_length = 3,\n",
        "                        padding = 'max_length',\n",
        "                        truncation=True,\n",
        "                        return_attention_mask = True,\n",
        "                        return_tensors = 'pt',\n",
        "                   )\n",
        "      \n",
        "    input_ids.append(encoded_dict['input_ids'])\n",
        "    target_ids.append(encoded_target_dict['input_ids'])\n",
        "    attention_masks.append(encoded_dict['attention_mask'])\n",
        "\n",
        "input_ids = torch.cat(input_ids, dim=0)\n",
        "target_ids = torch.cat(target_ids, dim=0)\n",
        "attention_masks = torch.cat(attention_masks, dim=0)\n",
        "\n",
        "print('Original: ', parsed_dataset[0])\n",
        "print('Token IDs:', input_ids[0])\n",
        "print('Label token IDs:', target_ids[0])"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('Label token IDs:', target_ids[123])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ld1xH-BD0G-M",
        "outputId": "67aca9e1-5dca-48d7-97b7-26f9781bf51f"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Label token IDs: tensor([209,   1,   0])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qD_t0y0KVVSy"
      },
      "source": [
        "# Split dataset\n",
        "Class balance ratio should be similar to base dataset ratio."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "vN_SatRIVa4c"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import TensorDataset, random_split"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def check_class_balance(dataset):\n",
        "  spam_count = 0.0\n",
        "  not_spam_count = 0.0\n",
        "  for row in dataset:\n",
        "    if row[2][1].item() == 1:\n",
        "      spam_count += 1.0\n",
        "    else:\n",
        "      not_spam_count += 1.0\n",
        "  return spam_count / not_spam_count "
      ],
      "metadata": {
        "id": "oo9C8ATt0dTq"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Mm6vc6lLVW3l",
        "outputId": "e7223b64-86a7-459d-b681-1ea1e0db02d8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Spam to not spam messages ratio: 0.15475450590428838\n",
            "\n",
            "1,000 test samples\n",
            "Ratio: 0.15074798619102417\n",
            "\n",
            "4,116 training samples\n",
            "Ratio: 0.15455820476858345\n",
            "\n",
            "  458 validation samples\n",
            "Ratio: 0.16539440203562342\n",
            "\n"
          ]
        }
      ],
      "source": [
        "dataset = TensorDataset(input_ids, attention_masks, target_ids)\n",
        "print(\"Spam to not spam messages ratio: {}\\n\".format(check_class_balance(dataset)))\n",
        "\n",
        "test_size = 1000\n",
        "dataset_len = len(dataset)\n",
        "train_size = int(0.9 * (dataset_len-test_size))\n",
        "val_size = (dataset_len-test_size) - train_size\n",
        "\n",
        "test_dataset, train_dataset, val_dataset = random_split(dataset, [test_size, train_size, val_size])\n",
        "\n",
        "print('{:>5,} test samples'.format(test_size))\n",
        "print(\"Ratio: {}\\n\".format(check_class_balance(test_dataset)))\n",
        "print('{:>5,} training samples'.format(train_size))\n",
        "print(\"Ratio: {}\\n\".format(check_class_balance(train_dataset)))\n",
        "print('{:>5,} validation samples'.format(val_size))\n",
        "print(\"Ratio: {}\\n\".format(check_class_balance(val_dataset)))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bmgQOP4EVfA1"
      },
      "source": [
        "# Create train and validation loaders"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CxnQ3cmIVlNh"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0hcpO_onVjEC"
      },
      "outputs": [],
      "source": [
        "batch_size = 16\n",
        "\n",
        "train_dataloader = DataLoader(\n",
        "            train_dataset,\n",
        "            sampler = RandomSampler(train_dataset),\n",
        "            batch_size = batch_size\n",
        "        )\n",
        "\n",
        "validation_dataloader = DataLoader(\n",
        "            val_dataset,\n",
        "            sampler = SequentialSampler(val_dataset),\n",
        "            batch_size = batch_size\n",
        "        )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "efwhqLyyVu9z"
      },
      "source": [
        "# Device check"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ANBCfNGnVwVk",
        "outputId": "ff2ff959-f0e9-47f3-d504-9daa45f870c2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla T4\n"
          ]
        }
      ],
      "source": [
        "if torch.cuda.is_available():     \n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "okTx_ynMV0rH"
      },
      "source": [
        "# Load T5 model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Eu-7Eed8WgN0"
      },
      "outputs": [],
      "source": [
        "from transformers import T5ForConditionalGeneration"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "68418b4f08654a2c8a19bdefa31ef7e2",
            "f59f1fe74df84329baa0137729651d7e",
            "4e6666f32de94c14973b2f5895c4f4ec",
            "9a8b0e9cf614453789dceff586f47682",
            "a4e1407e1a42416087a3138812851afa",
            "1813bc00d8db4de5a7bb7cd276346312",
            "ab6b0613a4934f34aad4d28cd855362d",
            "7514dfc8c5c34f29ab9a246ba6b45dc2",
            "017b00a3a26743d3a761a5b05f72fe73",
            "1cfe23326f964bb0a2925456aea14ad5",
            "384aac4ea3274eebbb43ea847036793a",
            "17986d272156460f8e9bcee2559088d9",
            "f1c7c8e7770848dabf155be27b342c6f",
            "719b8ebc46884edd9b36829f49680c98",
            "f28050af08f947678a41e1ea5611067f",
            "2ff5d9e91bf64330a2747c9c518ba31c",
            "85bd410d586b4ac98b8df72f980c0194",
            "feb7905c359e4acd9c9f848fb63d5d55",
            "b1d4154a8b054c8380a9ac70c311755b",
            "2fee9e3e54ae41c8977beaae6802010f",
            "42dc0f0578ed4105abeee4362667a98a",
            "04bb3488deec4565a0864049b122437d"
          ]
        },
        "id": "JKv9O8kfV2zZ",
        "outputId": "ad88a39b-bdc7-4325-b588-ed5feb453c3e"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)\"pytorch_model.bin\";:   0%|          | 0.00/892M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "68418b4f08654a2c8a19bdefa31ef7e2"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)neration_config.json:   0%|          | 0.00/147 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "17986d272156460f8e9bcee2559088d9"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "T5ForConditionalGeneration(\n",
              "  (shared): Embedding(32128, 768)\n",
              "  (encoder): T5Stack(\n",
              "    (embed_tokens): Embedding(32128, 768)\n",
              "    (block): ModuleList(\n",
              "      (0): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (relative_attention_bias): Embedding(32, 12)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (1): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (2): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (3): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (4): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (5): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (6): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (7): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (8): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (9): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (10): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (11): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (final_layer_norm): T5LayerNorm()\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "  )\n",
              "  (decoder): T5Stack(\n",
              "    (embed_tokens): Embedding(32128, 768)\n",
              "    (block): ModuleList(\n",
              "      (0): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (relative_attention_bias): Embedding(32, 12)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (1): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (2): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (3): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (4): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (5): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (6): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (7): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (8): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (9): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (10): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (11): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (final_layer_norm): T5LayerNorm()\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "  )\n",
              "  (lm_head): Linear(in_features=768, out_features=32128, bias=False)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ],
      "source": [
        "model = T5ForConditionalGeneration.from_pretrained('t5-base')\n",
        "\n",
        "model.cuda()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F_SDAwxoawDy"
      },
      "source": [
        "# Helper functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "s-q6_F38bLVA"
      },
      "outputs": [],
      "source": [
        "import datetime\n",
        "import numpy as np"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FzUi8908ax61"
      },
      "outputs": [],
      "source": [
        "def calculate_accuracy(preds, target):\n",
        "  results_ok = 0.0\n",
        "  results_false = 0.0\n",
        "\n",
        "  for idx, pred in enumerate(preds):\n",
        "    if pred == target[idx]:\n",
        "      results_ok += 1.0\n",
        "    else:\n",
        "      results_false += 1.0\n",
        "\n",
        "  return results_ok / (results_ok + results_false)\n",
        "\n",
        "def format_time(elapsed):\n",
        "    '''\n",
        "    Takes a time in seconds and returns a string hh:mm:ss\n",
        "    '''\n",
        "    elapsed_rounded = int(round((elapsed)))\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ucChBa-9bXJy"
      },
      "source": [
        "# Init training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "A7XUF4PNbYy8"
      },
      "outputs": [],
      "source": [
        "optimizer = torch.optim.AdamW(model.parameters(),\n",
        "                  lr = 3e-4,\n",
        "                  eps = 1e-8\n",
        "                )\n",
        "\n",
        "epochs = 4\n",
        "total_steps = len(train_dataloader) * epochs"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DAzQWODja0A3"
      },
      "source": [
        "# Training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Hoa7NlU0bI7G"
      },
      "outputs": [],
      "source": [
        "import random\n",
        "import time"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xsHxfslka1u5",
        "outputId": "e40d00a1-baf8-4554-e5ec-aeb87ee35f66"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "======== Epoch 1 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    258.    Elapsed: 0:01:12.\n",
            "  Batch    80  of    258.    Elapsed: 0:02:20.\n",
            "  Batch   120  of    258.    Elapsed: 0:03:28.\n",
            "  Batch   160  of    258.    Elapsed: 0:04:36.\n",
            "  Batch   200  of    258.    Elapsed: 0:05:45.\n",
            "  Batch   240  of    258.    Elapsed: 0:06:53.\n",
            "\n",
            "  Average training loss: 0.09\n",
            "  Average training acc: 0.81\n",
            "  Training epcoh took: 0:07:23\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.83\n",
            "  Validation took: 0:00:27\n",
            "  Validation Loss: 0.00\n",
            "\n",
            "======== Epoch 2 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    258.    Elapsed: 0:01:09.\n",
            "  Batch    80  of    258.    Elapsed: 0:02:17.\n",
            "  Batch   120  of    258.    Elapsed: 0:03:25.\n",
            "  Batch   160  of    258.    Elapsed: 0:04:33.\n",
            "  Batch   200  of    258.    Elapsed: 0:05:42.\n",
            "  Batch   240  of    258.    Elapsed: 0:06:50.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Average training acc: 0.86\n",
            "  Training epcoh took: 0:07:19\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.83\n",
            "  Validation took: 0:00:26\n",
            "  Validation Loss: 0.00\n",
            "\n",
            "======== Epoch 3 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    258.    Elapsed: 0:01:08.\n",
            "  Batch    80  of    258.    Elapsed: 0:02:16.\n",
            "  Batch   120  of    258.    Elapsed: 0:03:24.\n",
            "  Batch   160  of    258.    Elapsed: 0:04:32.\n",
            "  Batch   200  of    258.    Elapsed: 0:05:41.\n",
            "  Batch   240  of    258.    Elapsed: 0:06:49.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Average training acc: 0.85\n",
            "  Training epcoh took: 0:07:18\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.83\n",
            "  Validation took: 0:00:26\n",
            "  Validation Loss: 0.00\n",
            "\n",
            "======== Epoch 4 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    258.    Elapsed: 0:01:08.\n",
            "  Batch    80  of    258.    Elapsed: 0:02:16.\n",
            "  Batch   120  of    258.    Elapsed: 0:03:24.\n",
            "  Batch   160  of    258.    Elapsed: 0:04:32.\n",
            "  Batch   200  of    258.    Elapsed: 0:05:41.\n",
            "  Batch   240  of    258.    Elapsed: 0:06:49.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Average training acc: 0.86\n",
            "  Training epcoh took: 0:07:18\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.83\n",
            "  Validation took: 0:00:26\n",
            "  Validation Loss: 0.00\n",
            "\n",
            "Training complete!\n",
            "Total training took 0:31:02 (h:mm:ss)\n"
          ]
        }
      ],
      "source": [
        "# This training code is based on the `run_glue.py` script here:\n",
        "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
        "\n",
        "seed_val = 42\n",
        "\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "training_stats = []\n",
        "total_t0 = time.time()\n",
        "\n",
        "for epoch_i in range(0, epochs):\n",
        "    \n",
        "    # ========================================\n",
        "    #               Training\n",
        "    # ========================================\n",
        "\n",
        "    print(\"\")\n",
        "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "    print('Training...')\n",
        "\n",
        "    t0 = time.time()\n",
        "    total_train_loss = 0\n",
        "    total_train_acc = 0\n",
        "\n",
        "    model.train()\n",
        "\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "        if step % 40 == 0 and not step == 0:\n",
        "            elapsed = format_time(time.time() - t0)\n",
        "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
        "\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "\n",
        "        y = batch[2].to(device)\n",
        "        y_ids = y[:, :-1].contiguous()\n",
        "        lm_labels = y[:, 1:].clone().detach()\n",
        "        lm_labels[y[:, 1:] == tokenizer.pad_token_id] = -100      \n",
        "\n",
        "        outputs = model(\n",
        "            input_ids=b_input_ids,\n",
        "            attention_mask=b_input_mask,\n",
        "            decoder_input_ids=y_ids,\n",
        "            labels=lm_labels\n",
        "        )\n",
        "\n",
        "        generated_ids = model.generate(\n",
        "            input_ids = b_input_ids,\n",
        "            attention_mask = b_input_mask, \n",
        "            max_length=3, \n",
        "            num_beams=2,\n",
        "            repetition_penalty=2.5, \n",
        "            length_penalty=1.0, \n",
        "            early_stopping=True\n",
        "        )\n",
        "\n",
        "        preds = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=True) for g in generated_ids]\n",
        "        target = [tokenizer.decode(t, skip_special_tokens=True, clean_up_tokenization_spaces=True) for t in y]\n",
        "        total_train_acc += calculate_accuracy(preds, target)   \n",
        "\n",
        "        loss = outputs[0]\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        total_train_loss += loss.item()\n",
        "\n",
        "    avg_train_loss = total_train_loss / len(train_dataloader)    \n",
        "    avg_train_acc = total_train_acc / len(train_dataloader)          \n",
        "    training_time = format_time(time.time() - t0)\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "    print(\"  Average training acc: {0:.2f}\".format(avg_train_acc))\n",
        "    print(\"  Training epcoh took: {:}\".format(training_time))\n",
        "        \n",
        "    # ========================================\n",
        "    #               Validation\n",
        "    # ========================================\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"Running Validation...\")\n",
        "\n",
        "    t0 = time.time()\n",
        "    model.eval()\n",
        "\n",
        "    total_eval_loss = 0\n",
        "    total_eval_accuracy = 0\n",
        "\n",
        "    for batch in validation_dataloader:\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "\n",
        "        y = batch[2].to(device)\n",
        "        y_ids = y[:, :-1].contiguous()\n",
        "        lm_labels = y[:, 1:].clone().detach()\n",
        "        lm_labels[y[:, 1:] == tokenizer.pad_token_id] = -100\n",
        "        \n",
        "        with torch.no_grad():        \n",
        "\n",
        "          outputs = model(\n",
        "            input_ids=b_input_ids,\n",
        "            attention_mask=b_input_mask,\n",
        "            decoder_input_ids=y_ids,\n",
        "            labels=lm_labels\n",
        "          )\n",
        "\n",
        "          loss = outputs[0]\n",
        "          total_eval_loss += loss.item()\n",
        "\n",
        "          generated_ids = model.generate(\n",
        "            input_ids = b_input_ids,\n",
        "            attention_mask = b_input_mask, \n",
        "            max_length=3, \n",
        "            num_beams=2,\n",
        "            repetition_penalty=2.5, \n",
        "            length_penalty=1.0, \n",
        "            early_stopping=True\n",
        "          )\n",
        "\n",
        "          preds = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=True) for g in generated_ids]\n",
        "          target = [tokenizer.decode(t, skip_special_tokens=True, clean_up_tokenization_spaces=True) for t in y]\n",
        "          total_eval_accuracy += calculate_accuracy(preds, target)        \n",
        "\n",
        "    avg_val_loss = total_eval_loss / len(validation_dataloader)\n",
        "\n",
        "    avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)\n",
        "    print(\"  Accuracy: {0:.2f}\".format(avg_val_accuracy))\n",
        "    \n",
        "    validation_time = format_time(time.time() - t0)\n",
        "    print(\"  Validation took: {:}\".format(validation_time))\n",
        "    print(\"  Validation Loss: {0:.2f}\".format(avg_val_loss))\n",
        "\n",
        "    training_stats.append(\n",
        "        {\n",
        "            'epoch': epoch_i + 1,\n",
        "            'Training Loss': avg_train_loss,\n",
        "            'Training Accur.': avg_train_acc,\n",
        "            'Valid. Loss': avg_val_loss,\n",
        "            'Valid. Accur.': avg_val_accuracy,\n",
        "            'Training Time': training_time,\n",
        "            'Validation Time': validation_time\n",
        "        }\n",
        "    )\n",
        "\n",
        "print(\"\")\n",
        "print(\"Training complete!\")\n",
        "\n",
        "print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xIpFPoRb91Or"
      },
      "source": [
        "# Train summary"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GjYqBrrO93Oh",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "326edb05-56a5-4376-d793-424e5e122507"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       Training Loss  Training Accur.  Valid. Loss  Valid. Accur.  \\\n",
              "epoch                                                               \n",
              "1           9.03e-02             0.81     9.89e-07           0.83   \n",
              "2           1.30e-05             0.86     2.26e-08           0.83   \n",
              "3           3.05e-06             0.85     0.00e+00           0.83   \n",
              "4           5.13e-06             0.86     0.00e+00           0.83   \n",
              "\n",
              "      Training Time Validation Time  \n",
              "epoch                                \n",
              "1           0:07:23         0:00:27  \n",
              "2           0:07:19         0:00:26  \n",
              "3           0:07:18         0:00:26  \n",
              "4           0:07:18         0:00:26  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-970792fa-8287-4ad4-ac66-0d66603dad55\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Training Accur.</th>\n",
              "      <th>Valid. Loss</th>\n",
              "      <th>Valid. Accur.</th>\n",
              "      <th>Training Time</th>\n",
              "      <th>Validation Time</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>epoch</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>9.03e-02</td>\n",
              "      <td>0.81</td>\n",
              "      <td>9.89e-07</td>\n",
              "      <td>0.83</td>\n",
              "      <td>0:07:23</td>\n",
              "      <td>0:00:27</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.30e-05</td>\n",
              "      <td>0.86</td>\n",
              "      <td>2.26e-08</td>\n",
              "      <td>0.83</td>\n",
              "      <td>0:07:19</td>\n",
              "      <td>0:00:26</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3.05e-06</td>\n",
              "      <td>0.85</td>\n",
              "      <td>0.00e+00</td>\n",
              "      <td>0.83</td>\n",
              "      <td>0:07:18</td>\n",
              "      <td>0:00:26</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5.13e-06</td>\n",
              "      <td>0.86</td>\n",
              "      <td>0.00e+00</td>\n",
              "      <td>0.83</td>\n",
              "      <td>0:07:18</td>\n",
              "      <td>0:00:26</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-970792fa-8287-4ad4-ac66-0d66603dad55')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-970792fa-8287-4ad4-ac66-0d66603dad55 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-970792fa-8287-4ad4-ac66-0d66603dad55');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ],
      "source": [
        "import pandas as pd\n",
        "\n",
        "pd.set_option('precision', 2)\n",
        "df_stats = pd.DataFrame(data=training_stats)\n",
        "\n",
        "df_stats = df_stats.set_index('epoch')\n",
        "df_stats"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Xk3gzkeU96v3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 427
        },
        "outputId": "0fab6cbe-dd1c-4bdd-a986-24ae72d30aec"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvAAAAGaCAYAAABpIXfbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeUBTV/428CchBJBdRKGIqCirrC64YFVEQcVqFfcRta61VlunM+pY7fbWTtWq1bZ26ka17gpWBcXd1rrvWhYrKIIiIAoCAiEk7x/+SI0BDRi8AZ7PP52ce8+536Tc6ZObc88VKZVKJYiIiIiIqFYQC10AERERERFpjwGeiIiIiKgWYYAnIiIiIqpFGOCJiIiIiGoRBngiIiIiolqEAZ6IiIiIqBZhgCeiei89PR2urq5YsWJFtceYPXs2XF1ddVhV3VXZ5+3q6orZs2drNcaKFSvg6uqK9PR0ndcXFRUFV1dXnDlzRudjExHpgkToAoiInleVIHz48GE0bdq0BqupfZ48eYIff/wRsbGxyMrKQsOGDdG2bVtMnToVzs7OWo0xffp0xMXFYdeuXXB3d69wH6VSiZ49e+Lx48c4ceIEjI2Ndfk2atSZM2dw9uxZjBkzBhYWFkKXoyE9PR09e/bEqFGjMH/+fKHLISI9wwBPRHpn4cKFaq8vXLiArVu3YtiwYWjbtq3atoYNG77y8RwcHHD16lUYGBhUe4wvvvgCn3322SvXogsff/wxYmJiEBYWhg4dOiA7OxtHjhzBlStXtA7w4eHhiIuLw86dO/Hxxx9XuM/p06dx9+5dDBs2TCfh/erVqxCLX88Pw2fPnsV3332Ht99+WyPADxgwAP369YOhoeFrqYWIqKoY4IlI7wwYMEDtdVlZGbZu3QpfX1+Nbc8rKCiAmZlZlY4nEolgZGRU5TqfpS9hr6ioCPv370dgYCC++eYbVfu0adMgk8m0HicwMBD29vbYs2cP/v3vf0MqlWrsExUVBeBp2NeFV/13oCsGBgav9GWOiKimcQ48EdVaQUFBGD16NOLj4zF+/Hi0bdsWb731FoCnQX7p0qUYMmQIAgIC0KZNG/Tq1QuLFy9GUVGR2jgVzcl+tu3o0aMYPHgwvLy8EBgYiK+//hpyuVxtjIrmwJe35efn45NPPkGnTp3g5eWF4cOH48qVKxrv59GjR5gzZw4CAgLg5+eHiIgIxMfHY/To0QgKCtLqMxGJRBCJRBV+oagohFdGLBbj7bffRm5uLo4cOaKxvaCgAAcOHICLiwu8vb2r9HlXpqI58AqFAv/73/8QFBQELy8vhIWFYffu3RX2T05Oxqeffop+/frBz88PPj4+GDRoELZv36623+zZs/Hdd98BAHr27AlXV1e1f/+VzYF/+PAhPvvsM3Tr1g1t2rRBt27d8Nlnn+HRo0dq+5X3P3XqFNasWYPg4GC0adMGISEhiI6O1uqzqIrExES89957CAgIgJeXF/r27YtVq1ahrKxMbb+MjAzMmTMHPXr0QJs2bdCpUycMHz5crSaFQoHIyEj0798ffn5+8Pf3R0hICP7zn/+gtLRU57UTUfXwCjwR1Wr37t3DmDFjEBoait69e+PJkycAgMzMTOzYsQO9e/dGWFgYJBIJzp49i9WrVyMhIQFr1qzRavzjx49j06ZNGD58OAYPHozDhw9j7dq1sLS0xJQpU7QaY/z48WjYsCHee+895ObmYt26dZg0aRIOHz6s+rVAJpNh3LhxSEhIwKBBg+Dl5YWkpCSMGzcOlpaWWn8exsbGGDhwIHbu3Im9e/ciLCxM677PGzRoEFauXImoqCiEhoaqbYuJiUFxcTEGDx4MQHef9/O++uorrF+/Hu3bt8fYsWORk5ODzz//HI6Ojhr7nj17FufPn0f37t3RtGlT1a8RH3/8MR4+fIjJkycDAIYNG4aCggIcPHgQc+bMgbW1NYAX33uRn5+PESNGIDU1FYMHD4aHhwcSEhKwefNmnD59Gtu3b9f45Wfp0qUoLi7GsGHDIJVKsXnzZsyePRvNmjXTmApWXdeuXcPo0aMhkUgwatQoNGrUCEePHsXixYuRmJio+hVGLpdj3LhxyMzMxMiRI9G8eXMUFBQgKSkJ58+fx9tvvw0AWLlyJZYvX44ePXpg+PDhMDAwQHp6Oo4cOQKZTKY3vzQR1XtKIiI9t3PnTqWLi4ty586dau09evRQuri4KLdt26bRp6SkRCmTyTTaly5dqnRxcVFeuXJF1ZaWlqZ0cXFRLl++XKPNx8dHmZaWpmpXKBTKfv36Kbt06aI27qxZs5QuLi4Vtn3yySdq7bGxsUoXFxfl5s2bVW2//PKL0sXFRfnDDz+o7Vve3qNHD433UpH8/HzlxIkTlW3atFF6eHgoY2JitOpXmYiICKW7u7syMzNTrX3o0KFKT09PZU5OjlKpfPXPW6lUKl1cXJSzZs1SvU5OTla6uroqIyIilHK5XNV+/fp1paurq9LFxUXt301hYaHG8cvKypT/+Mc/lP7+/mr1LV++XKN/ufK/t9OnT6valixZonRxcVH+8ssvavuW//tZunSpRv8BAwYoS0pKVO33799Xenp6Kj/88EONYz6v/DP67LPPXrjfsGHDlO7u7sqEhARVm0KhUE6fPl3p4uKiPHnypFKpVCoTEhKULi4uyp9++umF4w0cOFDZp0+fl9ZHRMLiFBoiqtWsrKwwaNAgjXapVKq6WiiXy5GXl4eHDx+ic+fOAFDhFJaK9OzZU22VG5FIhICAAGRnZ6OwsFCrMcaOHav2umPHjgCA1NRUVdvRo0dhYGCAiIgItX2HDBkCc3NzrY6jUCgwY8YMJCYmYt++fXjzzTfx0UcfYc+ePWr7zZs3D56enlrNiQ8PD0dZWRl27dqlaktOTsbly5cRFBSkuolYV5/3sw4fPgylUolx48apzUn39PREly5dNPZv0KCB6n+XlJTg0aNHyM3NRZcuXVBQUICUlJQq11Du4MGDaNiwIYYNG6bWPmzYMDRs2BCHDh3S6DNy5Ei1aUtNmjRBixYtcPv27WrX8aycnBxcunQJQUFBcHNzU7WLRCK8++67qroBqP6Gzpw5g5ycnErHNDMzQ2ZmJs6fP6+TGomoZnAKDRHVao6OjpXecLhx40Zs2bIFN2/ehEKhUNuWl5en9fjPs7KyAgDk5ubC1NS0ymOUT9nIzc1VtaWnp6Nx48Ya40mlUjRt2hSPHz9+6XEOHz6MEydOYNGiRWjatCm+/fZbTJs2Df/+978hl8tV0ySSkpLg5eWl1Zz43r17w8LCAlFRUZg0aRIAYOfOnQCgmj5TThef97PS0tIAAC1bttTY5uzsjBMnTqi1FRYW4rvvvsO+ffuQkZGh0Uebz7Ay6enpaNOmDSQS9f9sSiQSNG/eHPHx8Rp9KvvbuXv3brXreL4mAGjVqpXGtpYtW0IsFqs+QwcHB0yZMgU//fQTAgMD4e7ujo4dOyI0NBTe3t6qfjNnzsR7772HUaNGoXHjxujQoQO6d++OkJCQKt1DQUQ1iwGeiGo1ExOTCtvXrVuH//73vwgMDERERAQaN24MQ0NDZGZmYvbs2VAqlVqN/6LVSF51DG37a6v8psv27dsDeBr+v/vuO7z77ruYM2cO5HI53NzccOXKFXz55ZdajWlkZISwsDBs2rQJFy9ehI+PD3bv3g07Ozt07dpVtZ+uPu9X8c9//hPHjh3D0KFD0b59e1hZWcHAwADHjx9HZGSkxpeKmva6lsTU1ocffojw8HAcO3YM58+fx44dO7BmzRpMmDAB//rXvwAAfn5+OHjwIE6cOIEzZ87gzJkz2Lt3L1auXIlNmzapvrwSkbAY4ImoTvr111/h4OCAVatWqQWp3377TcCqKufg4IBTp06hsLBQ7Sp8aWkp0tPTtXrYUPn7vHv3Luzt7QE8DfE//PADpkyZgnnz5sHBwQEuLi4YOHCg1rWFh4dj06ZNiIqKQl5eHrKzszFlyhS1z7UmPu/yK9gpKSlo1qyZ2rbk5GS1148fP8axY8cwYMAAfP7552rbTp48qTG2S
          },
          "metadata": {}
        }
      ],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "import seaborn as sns\n",
        "\n",
        "sns.set(style='darkgrid')\n",
        "\n",
        "sns.set(font_scale=1.5)\n",
        "plt.rcParams[\"figure.figsize\"] = (12,6)\n",
        "\n",
        "plt.plot(df_stats['Training Loss'], 'b-o', label=\"Training\")\n",
        "plt.plot(df_stats['Valid. Loss'], 'g-o', label=\"Validation\")\n",
        "\n",
        "plt.title(\"Training & Validation Loss\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.legend()\n",
        "plt.xticks([1, 2, 3, 4])\n",
        "\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UJlKxl0r-W-m"
      },
      "source": [
        "# Create test loader"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eQGsEEDh-YxG"
      },
      "outputs": [],
      "source": [
        "prediction_dataloader = DataLoader(\n",
        "            test_dataset,\n",
        "            sampler = SequentialSampler(test_dataset),\n",
        "            batch_size = batch_size\n",
        "        )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gHSDNWvA-aq9"
      },
      "source": [
        "# Evaluate on test dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OPcQkHnJ-c9A",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d5f3a63f-f9f6-4e68-b9d8-39e826ac1a3d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Predicting labels for 1,000 test sentences...\n",
            "    DONE.\n"
          ]
        }
      ],
      "source": [
        "print('Predicting labels for {:,} test sentences...'.format(len(test_dataset)))\n",
        "\n",
        "model.eval()\n",
        "predictions , true_labels = [], []\n",
        "total_test_acc = 0\n",
        "\n",
        "for batch in prediction_dataloader:\n",
        "\n",
        "    b_input_ids = batch[0].to(device)\n",
        "    b_input_mask = batch[1].to(device)\n",
        "    y = batch[2].to(device)\n",
        "    \n",
        "    with torch.no_grad():        \n",
        "\n",
        "      generated_ids = model.generate(\n",
        "        input_ids = b_input_ids,\n",
        "        attention_mask = b_input_mask, \n",
        "        max_length=3, \n",
        "        num_beams=2,\n",
        "        repetition_penalty=2.5, \n",
        "        length_penalty=1.0, \n",
        "        early_stopping=True\n",
        "      )\n",
        "        \n",
        "    preds = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=True) for g in generated_ids]\n",
        "    target = [tokenizer.decode(t, skip_special_tokens=True, clean_up_tokenization_spaces=True) for t in y]\n",
        "    total_test_acc += calculate_accuracy(preds, target)  \n",
        "\n",
        "    predictions.append(preds)\n",
        "    true_labels.append(target)\n",
        "\n",
        "print('    DONE.')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ifz56jYW-zBN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7f68ebc4-1205-4522-f5b5-74fb49d63565"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.873015873015873"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ],
      "source": [
        "avg_test_accuracy = total_test_acc / len(prediction_dataloader)\n",
        "avg_test_accuracy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1LqVo4wW-2g-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9c4820aa-4d8d-41dd-d3c4-d751927fcac0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sample prediction: 0, expected: 0\n"
          ]
        }
      ],
      "source": [
        "print(\"Sample prediction: {}, expected: {}\".format(predictions[2][10], true_labels[2][10]))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dLYc9WXz_B1o"
      },
      "source": [
        "# MCC Score"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hPEPpXXX_DXR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5ce9215b-0d24-4126-f80e-4bfa831b48bb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Calculating Matthews Corr. Coef. for each batch...\n"
          ]
        }
      ],
      "source": [
        "from sklearn.metrics import matthews_corrcoef\n",
        "\n",
        "matthews_set = []\n",
        "print('Calculating Matthews Corr. Coef. for each batch...')\n",
        "\n",
        "for i in range(len(true_labels)):\n",
        "  matthews = matthews_corrcoef(true_labels[i], predictions[i])                \n",
        "  matthews_set.append(matthews)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qjtAYcme_EyM",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 427
        },
        "outputId": "4728121a-b291-4ff8-ff2e-2197d04810ce"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvUAAAGaCAYAAACPCLyfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXyNd97/8fcJWWQhoYk9qkgskdhqa9BiIrXHVkuldCytpqP6MIPb6K+jnWpJi7FMUfsuJNIyljL3Xa2qJVWpCiWUkHs4REJCJCPn94c7Z3okOTnhRJx6PR8Pj5l8r+/yuZJO553L9/oeg8lkMgkAAACAw3Iq6wIAAAAAPBxCPQAAAODgCPUAAACAgyPUAwAAAA6OUA8AAAA4OEI9AAAA4OAI9QAAOJjhw4erc+fOZV0GgMdI+bIuAAAelYMHDyoyMlKSNGzYML3zzjsF+ly7dk2dOnVSbm6uWrdurdWrVxfo8+OPP2rt2rU6fPiwjEajnJycVKtWLbVr106DBw9WvXr1LPrfvn1bGzdu1O7du3XmzBllZWWpUqVKatKkiV588UX17t1b5ctb/9fxzZs3tXr1au3atUuXLl3S3bt35ePjo4YNG+qFF17QwIEDH+I7g/t17txZly5dMn9tMBhUpUoV1a1bV0OGDFGPHj0eeO49e/YoKSlJb775pj1KBQBJhHoATyBXV1dt27ZNkydPlouLi8W1+Ph4mUymIkP2/PnzNX/+fPn4+Khnz56qX7++8vLydObMGe3YsUNr167VoUOH5OnpKUk6f/68xowZo19++UXt27fXmDFj5OPjo2vXrunAgQOaMmWKzpw5oz/96U9F1puZmakBAwYoJSVF3bp1U//+/eXs7KyUlBR9//33WrVqFaG+FFSrVk1vv/22JCkvL0+XL19WXFyc3n77bRmNRo0YMeKB5t2zZ4/i4uII9QDsilAP4Inzu9/9Ttu2bdOePXvUvXt3i2uxsbHq2LGjvvvuuwLjNm/erHnz5qlNmzZasGCBvLy8LK7/8Y9/1Pz5881fZ2dna+zYsbp48aLmzZunsLAwi/5jxoxRYmKifvzxR6v1btq0Sb/88ov+67/+S6+88kqB60ajsdh7Lg2ZmZnmX14ciclk0q1bt+Th4WG1n5eXl/r06WPR9tJLL6lDhw6KjY194FAPAKWBPfUAnjiNGzdWYGCgYmNjLdoTExN1+vRp9e/fv8CYnJwczZkzR+7u7pozZ06BQC9Jbm5umjhxojnoxsTE6Ny5cxo5cmSBQJ8vODhYw4YNs1rvL7/8Iklq165dodd9fX0LtJ0/f15TpkxRx44dFRQUpNDQUL3++us6fvy4Rb89e/Zo8ODBatasmZo3b67Bgwdrz549Bebr3Lmzhg8frhMnTuj3v/+9WrZsqd69e1vU+Mc//lGhoaEKCgpS586d9dFHH+nWrVtW7+3++X/66SdFRkaqefPmat26tSZNmqRr164V6J+Tk6NPP/1UPXr0UNOmTdWqVSu99tprOnHihEW/gwcPmn/Wa9euVffu3dW0aVMtW7bMprruV6lSJbm4uMjZ2dmiPTExUZMnT1a3bt0UEhJi/l5++eWXFv2GDx+uuLg4SVJgYKD5z6//WTQajXr//ffVpUsXBQUFqV27dho5cqT2799foJ7Lly/r7bff1rPPPquQkBD9/ve/17lz5x7o3gA4Np7UA3gi9e/fXx9++KEuX76sqlWrSrr3JL5KlSp6/vnnC/T//vvvZTQa1adPH1WuXNmmNXbt2iXp3tPdh+Hv7y/p3t8iTJw4sdj99z/++KNGjBihf//73xowYIAaNGigjIwMHTp0SEePHlVQUJAkae3atZo+fbqeeeYZjRs3TpIUFxenN954Q9OnTy9Qd2pqql555RWFh4crLCzMHNiPHz+uV155RRUrVtRLL72kqlWr6uTJk1q9erWOHj2q1atXFwjBhfnXv/6lESNGKCwsTN26ddOJEye0ZcsWHT9+XJs3b1aFChUkSbm5ufr973+vo0ePqk+fPho2bJgyMzO1adMmDRkyRGvWrFHTpk0t5l65cqXS09M1cOBA+fr6qlq1asXWc/fuXaWlpUm6t/3GaDRq1apVysrK0uDBgy36fvnllzp79qzCw8NVs2ZNpaenKy4uTlFRUYqOjlavXr0kSa+99pry8vJ05MgRzZw50zy+RYsWkqSLFy9qyJAhunbtmvr06aOgoCDdvn1bx44d07fffqvnnnvOPObWrVt6+eWXFRISogkTJujixYtatWqVxo0bp23btqlcuXLF3iOA3xATADwhvvvuO1NAQIDps88+M6WlpZmaNGli+vvf/24ymUym27dvm1q2bGn68MMPTSaTydSsWTPTyy+/bB67atUqU0BAgGnZsmU2r9e6dWtTixYtHrru9PR0U6dOnUwBAQGmdu3amd58803TokWLTIcPHzbdvXvXom9eXp6pR48epqCgIFNSUlKBufL7p6enm5o1a2bq2rWr6ebNm+brN2/eNHXp0sXUrFkzU0ZGhrn9hRdeMAUEBJg2bdpUYM5evXqZunXrZjGPyWQy7d692xQQEGDasmVLsfeYP//y5cst2pcvX24KCAgwLVq0qEDbvn37LPrevHnT1KlTJ4ufW/7P/NlnnzVdvXq12Drur+f+P02bNjVt2LChQP+srKwCbbdu3TKFhYWZXnzxRYv2SZMmmQICAgpdd9SoUYXem8lksvhZv/zyy6aAgADT4sWLLfosWbKkyPEAftvYfgPgieTj46POnTubt0Ls3r1bN2/eLHTrjXRv/7ikEu0hz8zMLHbfti0qVaqk2NhYjR49Wl5eXtq1a5c+/vhjDRs2TF27dtU333xj7puUlKTTp0+rX79+atiwYYG5nJzu/Wt///79unXrloYPH25xT56enho+fLhu3bqlb7/91mKst7e3+vXrZ9F26tQpnTp1Sj179lROTo7S0tLMf1q2bCl3d/dCt40UxtPTU0OHDrVoGzp0qDw9PS22sXz++ed65pln1KRJE4v1cnJy1L59eyUkJCg7O9tinj59+qhKlSo21ZGvZs2aWr58uZYvX65ly5bpww8/VEhIiN59911t2bLFoq+7u7v5v9++fVvXr1/X7du31bZtWyUnJ5v/+bEmPT1dX3/9tTp06KAOHToUuJ7/s/v11/mnOeVr27atpHvbrwA8Wdh+A+CJ1b9/f40ZM0ZHjhzRli1bFBwcrPr16xfaNz/4ZmVl2Ty/p6dnifpbU7lyZU2cOFETJ07U9evX9cMPP2jHjh36/PPPFRUVpfj4eNWpU8e8/75x48ZW57t48aIkqUGDBgWu5belpKRYtNeuXbvAlo7k5GRJ0rx58zRv3rxC17p69WrxN/h/899/GpGLi4tq165tUUtycrKys7OLfMdAkq5fv67q1aubv3766adtquHX3N3d1b59e4u2Xr16KSIiQu+//746d+4sHx8fSfeOQp0zZ4727t1b6DsAN27cKPYXwgsXLshkMhX7s8vn5+cnV1dXizZvb29J935BAPBkIdQDeGKFhoaqatWqWrBggQ4ePKh33323yL75Qff+FzGtadCggQ4fPqyUlBTVrl37Ycs18/Hx0QsvvKAXXnhB1atX16effqrt27eb98WXlvw97YV59dVXC326LEkVK1a0ax0mk0kBAQGaMmVKkX3uf+/BWu0lUb58ebVt21arVq1SYmKiOnXqJJPJpFdffVXJycmKjIxUUFCQvLy8VK5cOW3ZskXbtm1TXl6eXdb/NWt75k0mk93XA/B4I9QDeGKVK1dOffv21aJFi+Tm5qaePXsW2bdFixby9fXVnj17dP36dfMTWmvCwsJ0+PBhxcTEmM87t7eQkBBJ905BkaS6detKurcNx5r8XzJOnz5d4In3mTNnLPpYU6dOHUn3toLc/1S7pFJSUpSTk2PxtD4nJ0cpKSl65plnLNa8fv262rZtW2BLyqPw73//W9J//tbm1KlTOnnypN544w394Q9/sOgbExNTYLzBYCh0Xn9/fxkMhmJ/dgBQGPbUA3iiDR48WFFRU
          },
          "metadata": {}
        }
      ],
      "source": [
        "ax = sns.barplot(x=list(range(len(matthews_set))), y=matthews_set, ci=None)\n",
        "\n",
        "plt.title('MCC Score per Batch')\n",
        "plt.ylabel('MCC Score (-1 to +1)')\n",
        "plt.xlabel('Batch #')\n",
        "\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rkonN244_HPz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0f08315b-7d90-4d6e-ad6f-559cf4573fa8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total MCC: 0.042\n"
          ]
        }
      ],
      "source": [
        "flat_predictions = np.concatenate(predictions, axis=0)\n",
        "flat_true_labels = np.concatenate(true_labels, axis=0)\n",
        "\n",
        "mcc = matthews_corrcoef(flat_true_labels, flat_predictions)\n",
        "print('Total MCC: %.3f' % mcc)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GPhCp068_Iwq"
      },
      "source": [
        "# Save model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "avafCMoS_KDF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e7983c91-ecab-4b7c-de48-9c5db5737ca9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive/\n",
            "Saving model to /content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/T5_Model\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('/content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/T5_Model/tokenizer_config.json',\n",
              " '/content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/T5_Model/special_tokens_map.json',\n",
              " '/content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/T5_Model/spiece.model',\n",
              " '/content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/T5_Model/added_tokens.json')"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "\n",
        "drive.mount('/content/gdrive/', force_remount=True)\n",
        "\n",
        "output_dir = '/content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/T5_Model'\n",
        "print(\"Saving model to %s\" % output_dir)\n",
        "\n",
        "model_to_save = model.module if hasattr(model, 'module') else model \n",
        "model_to_save.save_pretrained(output_dir)\n",
        "tokenizer.save_pretrained(output_dir)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wHzm2_nDA6i-"
      },
      "source": [
        "# Bibliografia\n",
        "- https://github.com/Shivanandroy/T5-Finetuning-PyTorch/blob/main/notebook/T5_Fine_tuning_with_PyTorch.ipynb\n",
        "- https://mccormickml.com/2019/07/22/BERT-fine-tuning/#a1-saving--loading-fine-tuned-model\n",
        "- https://huggingface.co/docs/transformers/model_doc/t5#training"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "68418b4f08654a2c8a19bdefa31ef7e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f59f1fe74df84329baa0137729651d7e",
              "IPY_MODEL_4e6666f32de94c14973b2f5895c4f4ec",
              "IPY_MODEL_9a8b0e9cf614453789dceff586f47682"
            ],
            "layout": "IPY_MODEL_a4e1407e1a42416087a3138812851afa"
          }
        },
        "f59f1fe74df84329baa0137729651d7e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1813bc00d8db4de5a7bb7cd276346312",
            "placeholder": "​",
            "style": "IPY_MODEL_ab6b0613a4934f34aad4d28cd855362d",
            "value": "Downloading (…)&quot;pytorch_model.bin&quot;;: 100%"
          }
        },
        "4e6666f32de94c14973b2f5895c4f4ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7514dfc8c5c34f29ab9a246ba6b45dc2",
            "max": 891691430,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_017b00a3a26743d3a761a5b05f72fe73",
            "value": 891691430
          }
        },
        "9a8b0e9cf614453789dceff586f47682": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1cfe23326f964bb0a2925456aea14ad5",
            "placeholder": "​",
            "style": "IPY_MODEL_384aac4ea3274eebbb43ea847036793a",
            "value": " 892M/892M [00:12&lt;00:00, 72.4MB/s]"
          }
        },
        "a4e1407e1a42416087a3138812851afa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1813bc00d8db4de5a7bb7cd276346312": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ab6b0613a4934f34aad4d28cd855362d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7514dfc8c5c34f29ab9a246ba6b45dc2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "017b00a3a26743d3a761a5b05f72fe73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1cfe23326f964bb0a2925456aea14ad5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "384aac4ea3274eebbb43ea847036793a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "17986d272156460f8e9bcee2559088d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f1c7c8e7770848dabf155be27b342c6f",
              "IPY_MODEL_719b8ebc46884edd9b36829f49680c98",
              "IPY_MODEL_f28050af08f947678a41e1ea5611067f"
            ],
            "layout": "IPY_MODEL_2ff5d9e91bf64330a2747c9c518ba31c"
          }
        },
        "f1c7c8e7770848dabf155be27b342c6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_85bd410d586b4ac98b8df72f980c0194",
            "placeholder": "​",
            "style": "IPY_MODEL_feb7905c359e4acd9c9f848fb63d5d55",
            "value": "Downloading (…)neration_config.json: 100%"
          }
        },
        "719b8ebc46884edd9b36829f49680c98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b1d4154a8b054c8380a9ac70c311755b",
            "max": 147,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2fee9e3e54ae41c8977beaae6802010f",
            "value": 147
          }
        },
        "f28050af08f947678a41e1ea5611067f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_42dc0f0578ed4105abeee4362667a98a",
            "placeholder": "​",
            "style": "IPY_MODEL_04bb3488deec4565a0864049b122437d",
            "value": " 147/147 [00:00&lt;00:00, 3.74kB/s]"
          }
        },
        "2ff5d9e91bf64330a2747c9c518ba31c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "85bd410d586b4ac98b8df72f980c0194": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "feb7905c359e4acd9c9f848fb63d5d55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b1d4154a8b054c8380a9ac70c311755b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2fee9e3e54ae41c8977beaae6802010f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "42dc0f0578ed4105abeee4362667a98a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "04bb3488deec4565a0864049b122437d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c396a3f65bb947ffa33130c424d9d93b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_fe2dd0bc42b84c5890d2c3dccaf66992",
              "IPY_MODEL_245239c79fa74387bed598565bbc24a4",
              "IPY_MODEL_926ce941393e4004bec99d38e82ea879"
            ],
            "layout": "IPY_MODEL_65433acf8e5345d580c6bf8c949c0064"
          }
        },
        "fe2dd0bc42b84c5890d2c3dccaf66992": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_293dc96088f942a3afc9af735d4c7117",
            "placeholder": "​",
            "style": "IPY_MODEL_a7dc0ef4c814401f8b2ff982063b66cc",
            "value": "Downloading builder script: 100%"
          }
        },
        "245239c79fa74387bed598565bbc24a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_790101eedd824ab893a7c7a1c0039163",
            "max": 3208,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6889bbe799b54d9eade42250c2e5caa6",
            "value": 3208
          }
        },
        "926ce941393e4004bec99d38e82ea879": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5cc03bba39a74eaab7da908c6c24b1fb",
            "placeholder": "​",
            "style": "IPY_MODEL_db67393d589c4d5bbbdab58adf51f970",
            "value": " 3.21k/3.21k [00:00&lt;00:00, 41.4kB/s]"
          }
        },
        "65433acf8e5345d580c6bf8c949c0064": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "293dc96088f942a3afc9af735d4c7117": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a7dc0ef4c814401f8b2ff982063b66cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "790101eedd824ab893a7c7a1c0039163": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6889bbe799b54d9eade42250c2e5caa6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5cc03bba39a74eaab7da908c6c24b1fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "db67393d589c4d5bbbdab58adf51f970": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2c5a4622661a4465910b5f1f95bea742": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3d4560baaec44c40b5d5c27ed8eba68a",
              "IPY_MODEL_dffa3d04bfe548e9aea5d4327ecca77a",
              "IPY_MODEL_01ce10b9b22f48839824dda0a40ec5e8"
            ],
            "layout": "IPY_MODEL_b839f01904ce41398b5286a801bbf4a7"
          }
        },
        "3d4560baaec44c40b5d5c27ed8eba68a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e36e4ad05a5040a4b67e0a133156358e",
            "placeholder": "​",
            "style": "IPY_MODEL_892dbc4c003941409202f31523589835",
            "value": "Downloading metadata: 100%"
          }
        },
        "dffa3d04bfe548e9aea5d4327ecca77a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4a82770611cc4ba29aea5c462ce0c5be",
            "max": 1687,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a3767cd0d3cd454595230f7933a0b2fe",
            "value": 1687
          }
        },
        "01ce10b9b22f48839824dda0a40ec5e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a77ad82dae14426b97ecc94166511b5c",
            "placeholder": "​",
            "style": "IPY_MODEL_98e29231304e4d9ba32e0368ec5b3fd5",
            "value": " 1.69k/1.69k [00:00&lt;00:00, 24.3kB/s]"
          }
        },
        "b839f01904ce41398b5286a801bbf4a7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e36e4ad05a5040a4b67e0a133156358e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "892dbc4c003941409202f31523589835": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4a82770611cc4ba29aea5c462ce0c5be": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a3767cd0d3cd454595230f7933a0b2fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a77ad82dae14426b97ecc94166511b5c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "98e29231304e4d9ba32e0368ec5b3fd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d64108ba247a4ac5a93b3bdeede7fd9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_aea219a8097b4d989664c09fcff9eb93",
              "IPY_MODEL_b2721b8b11ed42e29c623a10a8c8e13f",
              "IPY_MODEL_3d9d73ddd88446c5aeef5b2362bb878b"
            ],
            "layout": "IPY_MODEL_b955aea22f9e4ce2882b5e722ae1dda8"
          }
        },
        "aea219a8097b4d989664c09fcff9eb93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2a1ebe1ed1c64921bb7cdb0ed1e57b2b",
            "placeholder": "​",
            "style": "IPY_MODEL_9980c1ddad4a475e97130c0efc2f3efe",
            "value": "Downloading readme: 100%"
          }
        },
        "b2721b8b11ed42e29c623a10a8c8e13f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_57feff796f4241f292cb4617ed85cfe0",
            "max": 4872,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9297c35ec1f7494caab95714d65e34ab",
            "value": 4872
          }
        },
        "3d9d73ddd88446c5aeef5b2362bb878b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dd110977a96d4f19b37e923c4296fbdd",
            "placeholder": "​",
            "style": "IPY_MODEL_953c5391bd5e49ad9b3752794929e08c",
            "value": " 4.87k/4.87k [00:00&lt;00:00, 81.3kB/s]"
          }
        },
        "b955aea22f9e4ce2882b5e722ae1dda8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2a1ebe1ed1c64921bb7cdb0ed1e57b2b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9980c1ddad4a475e97130c0efc2f3efe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "57feff796f4241f292cb4617ed85cfe0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9297c35ec1f7494caab95714d65e34ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "dd110977a96d4f19b37e923c4296fbdd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "953c5391bd5e49ad9b3752794929e08c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b8c9fbbce7b84bf4a2f5b90c9d35ce0f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_51ab0b4ff03f4bdab8f5ba3fef868d4a",
              "IPY_MODEL_fca8377c5d6e480180d22925109d431c",
              "IPY_MODEL_a8854ee753ff4d179718690c3361b5a0"
            ],
            "layout": "IPY_MODEL_72918e6a9c7e46f291f85c7adf237eb9"
          }
        },
        "51ab0b4ff03f4bdab8f5ba3fef868d4a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fd3d449f08554666887c54c329baecd9",
            "placeholder": "​",
            "style": "IPY_MODEL_8c83da8d3d54491a9f4c535bdb5611e1",
            "value": "Downloading data: 100%"
          }
        },
        "fca8377c5d6e480180d22925109d431c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fb3f2df569ff42c89dea846f0df4b62f",
            "max": 203415,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e298b598c6474c3fa2b7bada616110cc",
            "value": 203415
          }
        },
        "a8854ee753ff4d179718690c3361b5a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7838193eee55441d865d6f056454b841",
            "placeholder": "​",
            "style": "IPY_MODEL_9ff537b0b2c745968c7d22e90cbe3894",
            "value": " 203k/203k [00:00&lt;00:00, 1.12MB/s]"
          }
        },
        "72918e6a9c7e46f291f85c7adf237eb9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fd3d449f08554666887c54c329baecd9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8c83da8d3d54491a9f4c535bdb5611e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fb3f2df569ff42c89dea846f0df4b62f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e298b598c6474c3fa2b7bada616110cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7838193eee55441d865d6f056454b841": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9ff537b0b2c745968c7d22e90cbe3894": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6b52b5d926bc43a3b70238c4fbfad7da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_733b55697ed8437c8d1d180756f7d3bb",
              "IPY_MODEL_0d67db8eb2504316819ba0b4691aecb0",
              "IPY_MODEL_6c402b861e4b49ad8b7d07c053e069bd"
            ],
            "layout": "IPY_MODEL_0ea8f8e157634292b17babbdee9f46e0"
          }
        },
        "733b55697ed8437c8d1d180756f7d3bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5016ba950fa447ae8d0f3a8191c8a34c",
            "placeholder": "​",
            "style": "IPY_MODEL_8af86d8fcbbc41bd9e4bfee243bdb759",
            "value": "Generating train split: 100%"
          }
        },
        "0d67db8eb2504316819ba0b4691aecb0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_16ddb3f4068040c9bbe5fa74c1b0177c",
            "max": 5574,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f1424f2fde924cb88e05fc4a98d8e354",
            "value": 5574
          }
        },
        "6c402b861e4b49ad8b7d07c053e069bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_afd518d5660e49b599d930c7f6040f87",
            "placeholder": "​",
            "style": "IPY_MODEL_8a7c3caf8629499fbff1966e7544fc85",
            "value": " 5574/5574 [00:01&lt;00:00, 4695.80 examples/s]"
          }
        },
        "0ea8f8e157634292b17babbdee9f46e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "5016ba950fa447ae8d0f3a8191c8a34c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8af86d8fcbbc41bd9e4bfee243bdb759": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "16ddb3f4068040c9bbe5fa74c1b0177c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f1424f2fde924cb88e05fc4a98d8e354": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "afd518d5660e49b599d930c7f6040f87": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8a7c3caf8629499fbff1966e7544fc85": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bca3b484a8374617836c78a5a7247f19": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a6ad21ce2df942cc87a7009a76913bd7",
              "IPY_MODEL_991524a5133c4ae9a1e6067be42f6ee4",
              "IPY_MODEL_d4ed92b329c74cda9e38dc380fee1b71"
            ],
            "layout": "IPY_MODEL_0e7dd3e98f64447dbe54df8d80fed381"
          }
        },
        "a6ad21ce2df942cc87a7009a76913bd7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3034d7c759324b91bf84b704a023dd77",
            "placeholder": "​",
            "style": "IPY_MODEL_4c3563f49ed0497ab2dc8739e4570d5b",
            "value": "100%"
          }
        },
        "991524a5133c4ae9a1e6067be42f6ee4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5788c4c9888e4def886297d645c670a3",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b713a8211c324197a5b66b05b4bfbb10",
            "value": 1
          }
        },
        "d4ed92b329c74cda9e38dc380fee1b71": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9ab2665c20b8408490cc3fa43762b225",
            "placeholder": "​",
            "style": "IPY_MODEL_b640800aca9f431883d8c309dd3b1daf",
            "value": " 1/1 [00:00&lt;00:00,  9.49it/s]"
          }
        },
        "0e7dd3e98f64447dbe54df8d80fed381": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3034d7c759324b91bf84b704a023dd77": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4c3563f49ed0497ab2dc8739e4570d5b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5788c4c9888e4def886297d645c670a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b713a8211c324197a5b66b05b4bfbb10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9ab2665c20b8408490cc3fa43762b225": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b640800aca9f431883d8c309dd3b1daf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8263f80bfd30477389a7d24450a41aa9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d46c7aa929e24e5da4a3508e1ec82795",
              "IPY_MODEL_84a7f048c34541708b01a64726037e94",
              "IPY_MODEL_c545335c355d43c2b158eb7ea1032b68"
            ],
            "layout": "IPY_MODEL_45f93d1c8d574efc8573cd9c30be2fa4"
          }
        },
        "d46c7aa929e24e5da4a3508e1ec82795": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6bc7141208cf4f2787890daa6dd900b5",
            "placeholder": "​",
            "style": "IPY_MODEL_5fd95d4231824740a4df2c7d7cb015a0",
            "value": "Downloading (…)ve/main/spiece.model: 100%"
          }
        },
        "84a7f048c34541708b01a64726037e94": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_154741e5c910415f9e87da6cb5e1c578",
            "max": 791656,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7dda689c19df4ff9b854cede266b804b",
            "value": 791656
          }
        },
        "c545335c355d43c2b158eb7ea1032b68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_863b386cb073419e96e8ae0d01554a36",
            "placeholder": "​",
            "style": "IPY_MODEL_663902a693b5405c85be17bbe46e0650",
            "value": " 792k/792k [00:00&lt;00:00, 2.27MB/s]"
          }
        },
        "45f93d1c8d574efc8573cd9c30be2fa4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6bc7141208cf4f2787890daa6dd900b5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5fd95d4231824740a4df2c7d7cb015a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "154741e5c910415f9e87da6cb5e1c578": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7dda689c19df4ff9b854cede266b804b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "863b386cb073419e96e8ae0d01554a36": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "663902a693b5405c85be17bbe46e0650": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e76d348dd73d4e88994fa53449b69a0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_bd65017865934166878adf8aa6c352c9",
              "IPY_MODEL_0dc9cdebd0bd480d86fd2b7151f8617f",
              "IPY_MODEL_ac095333a156479bbba127d424b48943"
            ],
            "layout": "IPY_MODEL_2b2a144c8b434a0eb6b91c532965a956"
          }
        },
        "bd65017865934166878adf8aa6c352c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9674fadc0e6448f1ad34f2e47d6dec14",
            "placeholder": "​",
            "style": "IPY_MODEL_1ed148605a1c4d97a8ff1bbab36b0f8e",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "0dc9cdebd0bd480d86fd2b7151f8617f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3ed7755f2175486d8407dea96ecd8898",
            "max": 1208,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9afc3775ccc440c4b58e606e5daa9e75",
            "value": 1208
          }
        },
        "ac095333a156479bbba127d424b48943": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9d94d64ee8c4465489f4b645080030e5",
            "placeholder": "​",
            "style": "IPY_MODEL_4fdd5febab8d42e6b79438a69d39622e",
            "value": " 1.21k/1.21k [00:00&lt;00:00, 18.1kB/s]"
          }
        },
        "2b2a144c8b434a0eb6b91c532965a956": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9674fadc0e6448f1ad34f2e47d6dec14": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1ed148605a1c4d97a8ff1bbab36b0f8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3ed7755f2175486d8407dea96ecd8898": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9afc3775ccc440c4b58e606e5daa9e75": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9d94d64ee8c4465489f4b645080030e5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4fdd5febab8d42e6b79438a69d39622e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}