Przetwarzanie_tekstu/projekt/T5_sms_spam.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard",
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "137fa30b14f34f57a0beb8a6c6e60cf5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_755a0a2d6a154feab203d06a6daf9848",
              "IPY_MODEL_c0c63e8ee5e84757a66dc953b11116d7",
              "IPY_MODEL_cf211c4bc94c47e2a9b752d4a9890271"
            ],
            "layout": "IPY_MODEL_070471ec9b8d4523be8e8779c87e7d9e"
          }
        },
        "755a0a2d6a154feab203d06a6daf9848": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c22bd9c427b34a7dad89b1293da20e65",
            "placeholder": "​",
            "style": "IPY_MODEL_39348508617043c8a3cbec645ef49331",
            "value": "Downloading builder script: 100%"
          }
        },
        "c0c63e8ee5e84757a66dc953b11116d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d1f02d9191c6458a9fc42f5ebb2d5961",
            "max": 3208,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_11b4792c447c4b56acbdc5b3f7427c54",
            "value": 3208
          }
        },
        "cf211c4bc94c47e2a9b752d4a9890271": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_633950f724a8490b9425bb2fa2e4e84a",
            "placeholder": "​",
            "style": "IPY_MODEL_fe588dfbdc0f4fabaf54a1c3e02c0be3",
            "value": " 3.21k/3.21k [00:00<00:00, 67.3kB/s]"
          }
        },
        "070471ec9b8d4523be8e8779c87e7d9e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c22bd9c427b34a7dad89b1293da20e65": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "39348508617043c8a3cbec645ef49331": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d1f02d9191c6458a9fc42f5ebb2d5961": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "11b4792c447c4b56acbdc5b3f7427c54": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "633950f724a8490b9425bb2fa2e4e84a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fe588dfbdc0f4fabaf54a1c3e02c0be3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "47463520a286401ba3d5d29fce07ede5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f26179b81ddd477588764f8262560e2f",
              "IPY_MODEL_5d6c567d1f2a430c9632cf063572c8de",
              "IPY_MODEL_25f45b49e5f643ee994fbb5998807676"
            ],
            "layout": "IPY_MODEL_6030ad10c42c4993ac7d6610c8f0d77f"
          }
        },
        "f26179b81ddd477588764f8262560e2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_945360806d6445418fd779166c114994",
            "placeholder": "​",
            "style": "IPY_MODEL_d24b647f4a1542e5a4ebfa59bcf2bb2b",
            "value": "Downloading metadata: 100%"
          }
        },
        "5d6c567d1f2a430c9632cf063572c8de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1bc411a48f1c4f849351440d4e9c646f",
            "max": 1687,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_dd1c430002dd4fd6a08cb5b576bf0290",
            "value": 1687
          }
        },
        "25f45b49e5f643ee994fbb5998807676": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2d551e5c6a4548ae82c34d827f919d18",
            "placeholder": "​",
            "style": "IPY_MODEL_02cb3347a1be49bf8808e95a45a34801",
            "value": " 1.69k/1.69k [00:00<00:00, 36.1kB/s]"
          }
        },
        "6030ad10c42c4993ac7d6610c8f0d77f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "945360806d6445418fd779166c114994": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d24b647f4a1542e5a4ebfa59bcf2bb2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1bc411a48f1c4f849351440d4e9c646f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dd1c430002dd4fd6a08cb5b576bf0290": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2d551e5c6a4548ae82c34d827f919d18": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "02cb3347a1be49bf8808e95a45a34801": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0926a24353a94b82a8e405ec72bef775": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7e44b8cbc06f4d3bb5f073fad6a0b151",
              "IPY_MODEL_bcca70a24e7d4a5b9b2a98cebc5c5eda",
              "IPY_MODEL_0d26ce82d9f1424dbe8d08ce7b7e5b0b"
            ],
            "layout": "IPY_MODEL_9bdccfd08b814f4fb1f53c5a8a0bded0"
          }
        },
        "7e44b8cbc06f4d3bb5f073fad6a0b151": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3977c23fd48348b1b3855c3cc0db6827",
            "placeholder": "​",
            "style": "IPY_MODEL_2573a6dce3d94ca79cadee9a793d9e91",
            "value": "Downloading readme: 100%"
          }
        },
        "bcca70a24e7d4a5b9b2a98cebc5c5eda": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f01fad907cf742f6aae936051f36579d",
            "max": 4872,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_86661cdb45b44daeaf1581ca1b8d4cc6",
            "value": 4872
          }
        },
        "0d26ce82d9f1424dbe8d08ce7b7e5b0b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_04f96b4447a547b8ba8ff13b9bbedc04",
            "placeholder": "​",
            "style": "IPY_MODEL_dbdeaaff21924caca89dc32b633f80da",
            "value": " 4.87k/4.87k [00:00<00:00, 157kB/s]"
          }
        },
        "9bdccfd08b814f4fb1f53c5a8a0bded0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3977c23fd48348b1b3855c3cc0db6827": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2573a6dce3d94ca79cadee9a793d9e91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f01fad907cf742f6aae936051f36579d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "86661cdb45b44daeaf1581ca1b8d4cc6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "04f96b4447a547b8ba8ff13b9bbedc04": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dbdeaaff21924caca89dc32b633f80da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "320ba3e1e74541288c307eedbd5e2754": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e9675016075e4ec89c891033452ec11c",
              "IPY_MODEL_305a6be9c48c4381983307f584c5c6c8",
              "IPY_MODEL_cbfc39d75c494de284e2b8a3c95e6057"
            ],
            "layout": "IPY_MODEL_6bc6798a84944c9e8dad7391d1baa997"
          }
        },
        "e9675016075e4ec89c891033452ec11c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_86c211b9dac34fd0ad51d81385d59fd8",
            "placeholder": "​",
            "style": "IPY_MODEL_21df98aa3cdc403f91acd1c21f4b4e9d",
            "value": "Downloading data: 100%"
          }
        },
        "305a6be9c48c4381983307f584c5c6c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_eeb508fdb9064c09b8152b800fe61214",
            "max": 203415,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ce0d0132b7904f1794037e81880a563a",
            "value": 203415
          }
        },
        "cbfc39d75c494de284e2b8a3c95e6057": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_113ab2100a894a16b74040611824c91d",
            "placeholder": "​",
            "style": "IPY_MODEL_834964317e404dd5a9a28c63a58ce6c9",
            "value": " 203k/203k [00:00<00:00, 1.05MB/s]"
          }
        },
        "6bc6798a84944c9e8dad7391d1baa997": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "86c211b9dac34fd0ad51d81385d59fd8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "21df98aa3cdc403f91acd1c21f4b4e9d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "eeb508fdb9064c09b8152b800fe61214": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ce0d0132b7904f1794037e81880a563a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "113ab2100a894a16b74040611824c91d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "834964317e404dd5a9a28c63a58ce6c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a950c7427a174b22abdd17fb7710ece7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b72789fe47914768ba2808926ad54c86",
              "IPY_MODEL_327a1bc8f6c644d6b3168f708a2c6876",
              "IPY_MODEL_5f8f6e108647479eb92fb44ec1916d0d"
            ],
            "layout": "IPY_MODEL_7fa6a321dd3a4e198451da879be60a9b"
          }
        },
        "b72789fe47914768ba2808926ad54c86": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b179785454d8400fb495f810ae2aede2",
            "placeholder": "​",
            "style": "IPY_MODEL_2b16bccddc0e4be78d83cc012c2224e9",
            "value": "Generating train split:  96%"
          }
        },
        "327a1bc8f6c644d6b3168f708a2c6876": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6d601c96a62c46ff96cbc4fd8e0fbbb6",
            "max": 5574,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_498224e7a7b54c56ba45261a1b39c1c3",
            "value": 5574
          }
        },
        "5f8f6e108647479eb92fb44ec1916d0d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e9319122f48e43a5b15ef55823347507",
            "placeholder": "​",
            "style": "IPY_MODEL_7ea7ab8cba484142b5a3d8019e9c9c84",
            "value": " 5359/5574 [00:00<00:00, 11295.24 examples/s]"
          }
        },
        "7fa6a321dd3a4e198451da879be60a9b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "b179785454d8400fb495f810ae2aede2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2b16bccddc0e4be78d83cc012c2224e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6d601c96a62c46ff96cbc4fd8e0fbbb6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "498224e7a7b54c56ba45261a1b39c1c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e9319122f48e43a5b15ef55823347507": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7ea7ab8cba484142b5a3d8019e9c9c84": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fddd4ee4bc054b0f90ed88018fc3e3a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_55b5df3163a34561a4a5ba27efada434",
              "IPY_MODEL_2823c1041e914dd8887498410baaab43",
              "IPY_MODEL_fba57e030f624bbab5b51b65d7d36722"
            ],
            "layout": "IPY_MODEL_86fa9ce6cd9b4ec9af6d24d943aac75b"
          }
        },
        "55b5df3163a34561a4a5ba27efada434": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_97a0725bbb044103911e1e29bb07360e",
            "placeholder": "​",
            "style": "IPY_MODEL_1cf99cee3c374e30abd95c03e9696bbe",
            "value": "100%"
          }
        },
        "2823c1041e914dd8887498410baaab43": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d17a7d096d964a41a7ee183f5028c037",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c6a51ab4f16649c0899bbfd14fdc9cb8",
            "value": 1
          }
        },
        "fba57e030f624bbab5b51b65d7d36722": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_876c522f96a64809a82d316f4afa1bae",
            "placeholder": "​",
            "style": "IPY_MODEL_1c38d46a00224aa791cad25fdd4d33e2",
            "value": " 1/1 [00:00<00:00, 24.93it/s]"
          }
        },
        "86fa9ce6cd9b4ec9af6d24d943aac75b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "97a0725bbb044103911e1e29bb07360e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1cf99cee3c374e30abd95c03e9696bbe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d17a7d096d964a41a7ee183f5028c037": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c6a51ab4f16649c0899bbfd14fdc9cb8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "876c522f96a64809a82d316f4afa1bae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1c38d46a00224aa791cad25fdd4d33e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6e2b903343ad49c89339a38a1c626619": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b1e96f5c00d048c69c1c0fadeb31dcd9",
              "IPY_MODEL_ec994f6daafb4ef8a08371f2394918dd",
              "IPY_MODEL_699d9f4479854372ada35ab38fe80352"
            ],
            "layout": "IPY_MODEL_cb36ca390eac4c76ad8cfbcbdb5b6950"
          }
        },
        "b1e96f5c00d048c69c1c0fadeb31dcd9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5ffe2a0b8e3342a9b764fbbdf1395f1c",
            "placeholder": "​",
            "style": "IPY_MODEL_ea92e7a968d4479e842c368ece4b60c1",
            "value": "Downloading (…)ve/main/spiece.model: 100%"
          }
        },
        "ec994f6daafb4ef8a08371f2394918dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_95a4496f03414cf8a8d7cf5e6cc3f37b",
            "max": 791656,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7b1d15df592048fe8a8c043e7a8461ad",
            "value": 791656
          }
        },
        "699d9f4479854372ada35ab38fe80352": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cc73442788a3462a8d5d53c9c799df7a",
            "placeholder": "​",
            "style": "IPY_MODEL_ee75218b2c4047fdb265df7a54feea78",
            "value": " 792k/792k [00:00<00:00, 2.55MB/s]"
          }
        },
        "cb36ca390eac4c76ad8cfbcbdb5b6950": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5ffe2a0b8e3342a9b764fbbdf1395f1c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ea92e7a968d4479e842c368ece4b60c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "95a4496f03414cf8a8d7cf5e6cc3f37b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7b1d15df592048fe8a8c043e7a8461ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cc73442788a3462a8d5d53c9c799df7a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ee75218b2c4047fdb265df7a54feea78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e720c7e5ef0849918eb6e7123673c95e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8b2ff14cab9941388b547140d06e1dd5",
              "IPY_MODEL_0eb67bf20ecf4cc0b5f3bd0589440e6b",
              "IPY_MODEL_4e976d7959e640f4b098d9a02320f228"
            ],
            "layout": "IPY_MODEL_48de3465dc194fff9903fd3813aae91a"
          }
        },
        "8b2ff14cab9941388b547140d06e1dd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_29558f7cc7574024879d274548ac4cd7",
            "placeholder": "​",
            "style": "IPY_MODEL_ce5531904574465d84d65365b0fc2951",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "0eb67bf20ecf4cc0b5f3bd0589440e6b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_10cc03c556cf4fb791697277b3deef35",
            "max": 1208,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2113acabfb014e7ab55d099d28845914",
            "value": 1208
          }
        },
        "4e976d7959e640f4b098d9a02320f228": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_504b320daae543b78b8777cebbe65dea",
            "placeholder": "​",
            "style": "IPY_MODEL_5ea6f5fa80184fe58ef86a536ec0f8f0",
            "value": " 1.21k/1.21k [00:00<00:00, 41.1kB/s]"
          }
        },
        "48de3465dc194fff9903fd3813aae91a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "29558f7cc7574024879d274548ac4cd7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ce5531904574465d84d65365b0fc2951": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "10cc03c556cf4fb791697277b3deef35": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2113acabfb014e7ab55d099d28845914": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "504b320daae543b78b8777cebbe65dea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5ea6f5fa80184fe58ef86a536ec0f8f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b1d7a5cf900b48408e515baa4c66a1cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6846a2acd95b45a3a0e2cb79f552f0c0",
              "IPY_MODEL_fce901d34cc34feeb92854999e98c0f9",
              "IPY_MODEL_5a89ce40643247fda326742531912a01"
            ],
            "layout": "IPY_MODEL_bcc7a0cd035e485680b41e7c4a78b8f8"
          }
        },
        "6846a2acd95b45a3a0e2cb79f552f0c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bd0a578fefb44fb4b2662d59fd2ff12e",
            "placeholder": "​",
            "style": "IPY_MODEL_6585bd6115c047fd881c0bfd323142f0",
            "value": "Downloading (…)"pytorch_model.bin";: 100%"
          }
        },
        "fce901d34cc34feeb92854999e98c0f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dbc7f7aa90174ff68b5cc829a6fd8690",
            "max": 891691430,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ca3a8e4611c6422380351b947882876a",
            "value": 891691430
          }
        },
        "5a89ce40643247fda326742531912a01": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2470365762844b62a09dc6fa818c4a09",
            "placeholder": "​",
            "style": "IPY_MODEL_3f2489ce0ae941a1a720c60a3052ee70",
            "value": " 892M/892M [00:03<00:00, 274MB/s]"
          }
        },
        "bcc7a0cd035e485680b41e7c4a78b8f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bd0a578fefb44fb4b2662d59fd2ff12e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6585bd6115c047fd881c0bfd323142f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dbc7f7aa90174ff68b5cc829a6fd8690": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ca3a8e4611c6422380351b947882876a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2470365762844b62a09dc6fa818c4a09": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3f2489ce0ae941a1a720c60a3052ee70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0795a8385c68409fb5539b9ea6756a47": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_05dfc6dc9f78483da34b2c6513315e7d",
              "IPY_MODEL_5cfe28a638cb42fc914dc81eb02a46f4",
              "IPY_MODEL_d061dcb2f3e840ec9ba6a6ec4d972619"
            ],
            "layout": "IPY_MODEL_df418dee3efd4da8aa57ca0044190b2e"
          }
        },
        "05dfc6dc9f78483da34b2c6513315e7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9d3d394c756d4eabb0f3fd66ba8ef05a",
            "placeholder": "​",
            "style": "IPY_MODEL_00612595fa42467a83aa6e4b55343339",
            "value": "Downloading (…)neration_config.json: 100%"
          }
        },
        "5cfe28a638cb42fc914dc81eb02a46f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_33521be9887b4c368915b4f8f2438440",
            "max": 147,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_990a862f07894fa9b9f08d3bb7e082ca",
            "value": 147
          }
        },
        "d061dcb2f3e840ec9ba6a6ec4d972619": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1b793ae9c46740bdbbec5e617a899683",
            "placeholder": "​",
            "style": "IPY_MODEL_cbfde7f5f0204417abdced523c5621e9",
            "value": " 147/147 [00:00<00:00, 5.72kB/s]"
          }
        },
        "df418dee3efd4da8aa57ca0044190b2e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9d3d394c756d4eabb0f3fd66ba8ef05a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "00612595fa42467a83aa6e4b55343339": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "33521be9887b4c368915b4f8f2438440": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "990a862f07894fa9b9f08d3bb7e082ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1b793ae9c46740bdbbec5e617a899683": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cbfde7f5f0204417abdced523c5621e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Instalacja pakietów"
      ],
      "metadata": {
        "id": "ZXsOR6oJOJbd"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8l0hzptKNiZS",
        "outputId": "49caa437-96c0-41bc-fc92-1d6c6847e100"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.26.1-py3-none-any.whl (6.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.3/6.3 MB\u001b[0m \u001b[31m45.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting datasets\n",
            "  Downloading datasets-2.9.0-py3-none-any.whl (462 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m462.8/462.8 KB\u001b[0m \u001b[31m28.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.8/dist-packages (1.13.1+cu116)\n",
            "Collecting sentencepiece\n",
            "  Downloading sentencepiece-0.1.97-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m40.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.8/dist-packages (from transformers) (4.64.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from transformers) (3.9.0)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m59.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from transformers) (2.25.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (2022.6.2)\n",
            "Collecting huggingface-hub<1.0,>=0.11.0\n",
            "  Downloading huggingface_hub-0.12.0-py3-none-any.whl (190 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m190.3/190.3 KB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from transformers) (23.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from transformers) (6.0)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (213 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m213.0/213.0 KB\u001b[0m \u001b[31m1.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.8/dist-packages (from datasets) (2023.1.0)\n",
            "Requirement already satisfied: dill<0.3.7 in /usr/local/lib/python3.8/dist-packages (from datasets) (0.3.6)\n",
            "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.8/dist-packages (from datasets) (9.0.0)\n",
            "Collecting multiprocess\n",
            "  Downloading multiprocess-0.70.14-py38-none-any.whl (132 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.0/132.0 KB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: aiohttp in /usr/local/lib/python3.8/dist-packages (from datasets) (3.8.3)\n",
            "Collecting responses<0.19\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.8/dist-packages (from datasets) (1.3.5)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.8/dist-packages (from torch) (4.4.0)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (2.1.1)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.8.2)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (22.2.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (6.0.4)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.3.3)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (4.0.2)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (4.0.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (1.24.3)\n",
            "Collecting urllib3<1.27,>=1.21.1\n",
            "  Downloading urllib3-1.26.14-py2.py3-none-any.whl (140 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m140.6/140.6 KB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets) (2022.7.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.8/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: tokenizers, sentencepiece, xxhash, urllib3, multiprocess, responses, huggingface-hub, transformers, datasets\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 1.24.3\n",
            "    Uninstalling urllib3-1.24.3:\n",
            "      Successfully uninstalled urllib3-1.24.3\n",
            "Successfully installed datasets-2.9.0 huggingface-hub-0.12.0 multiprocess-0.70.14 responses-0.18.0 sentencepiece-0.1.97 tokenizers-0.13.2 transformers-4.26.1 urllib3-1.26.14 xxhash-3.2.0\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers datasets torch sentencepiece"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Załadowanie datasetu"
      ],
      "metadata": {
        "id": "dhN0rmb5Oi3d"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_dataset"
      ],
      "metadata": {
        "id": "tnaDkwZ2Pbnn"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = load_dataset(\"sms_spam\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 231,
          "referenced_widgets": [
            "137fa30b14f34f57a0beb8a6c6e60cf5",
            "755a0a2d6a154feab203d06a6daf9848",
            "c0c63e8ee5e84757a66dc953b11116d7",
            "cf211c4bc94c47e2a9b752d4a9890271",
            "070471ec9b8d4523be8e8779c87e7d9e",
            "c22bd9c427b34a7dad89b1293da20e65",
            "39348508617043c8a3cbec645ef49331",
            "d1f02d9191c6458a9fc42f5ebb2d5961",
            "11b4792c447c4b56acbdc5b3f7427c54",
            "633950f724a8490b9425bb2fa2e4e84a",
            "fe588dfbdc0f4fabaf54a1c3e02c0be3",
            "47463520a286401ba3d5d29fce07ede5",
            "f26179b81ddd477588764f8262560e2f",
            "5d6c567d1f2a430c9632cf063572c8de",
            "25f45b49e5f643ee994fbb5998807676",
            "6030ad10c42c4993ac7d6610c8f0d77f",
            "945360806d6445418fd779166c114994",
            "d24b647f4a1542e5a4ebfa59bcf2bb2b",
            "1bc411a48f1c4f849351440d4e9c646f",
            "dd1c430002dd4fd6a08cb5b576bf0290",
            "2d551e5c6a4548ae82c34d827f919d18",
            "02cb3347a1be49bf8808e95a45a34801",
            "0926a24353a94b82a8e405ec72bef775",
            "7e44b8cbc06f4d3bb5f073fad6a0b151",
            "bcca70a24e7d4a5b9b2a98cebc5c5eda",
            "0d26ce82d9f1424dbe8d08ce7b7e5b0b",
            "9bdccfd08b814f4fb1f53c5a8a0bded0",
            "3977c23fd48348b1b3855c3cc0db6827",
            "2573a6dce3d94ca79cadee9a793d9e91",
            "f01fad907cf742f6aae936051f36579d",
            "86661cdb45b44daeaf1581ca1b8d4cc6",
            "04f96b4447a547b8ba8ff13b9bbedc04",
            "dbdeaaff21924caca89dc32b633f80da",
            "320ba3e1e74541288c307eedbd5e2754",
            "e9675016075e4ec89c891033452ec11c",
            "305a6be9c48c4381983307f584c5c6c8",
            "cbfc39d75c494de284e2b8a3c95e6057",
            "6bc6798a84944c9e8dad7391d1baa997",
            "86c211b9dac34fd0ad51d81385d59fd8",
            "21df98aa3cdc403f91acd1c21f4b4e9d",
            "eeb508fdb9064c09b8152b800fe61214",
            "ce0d0132b7904f1794037e81880a563a",
            "113ab2100a894a16b74040611824c91d",
            "834964317e404dd5a9a28c63a58ce6c9",
            "a950c7427a174b22abdd17fb7710ece7",
            "b72789fe47914768ba2808926ad54c86",
            "327a1bc8f6c644d6b3168f708a2c6876",
            "5f8f6e108647479eb92fb44ec1916d0d",
            "7fa6a321dd3a4e198451da879be60a9b",
            "b179785454d8400fb495f810ae2aede2",
            "2b16bccddc0e4be78d83cc012c2224e9",
            "6d601c96a62c46ff96cbc4fd8e0fbbb6",
            "498224e7a7b54c56ba45261a1b39c1c3",
            "e9319122f48e43a5b15ef55823347507",
            "7ea7ab8cba484142b5a3d8019e9c9c84",
            "fddd4ee4bc054b0f90ed88018fc3e3a0",
            "55b5df3163a34561a4a5ba27efada434",
            "2823c1041e914dd8887498410baaab43",
            "fba57e030f624bbab5b51b65d7d36722",
            "86fa9ce6cd9b4ec9af6d24d943aac75b",
            "97a0725bbb044103911e1e29bb07360e",
            "1cf99cee3c374e30abd95c03e9696bbe",
            "d17a7d096d964a41a7ee183f5028c037",
            "c6a51ab4f16649c0899bbfd14fdc9cb8",
            "876c522f96a64809a82d316f4afa1bae",
            "1c38d46a00224aa791cad25fdd4d33e2"
          ]
        },
        "id": "cCiAuRqrOkvV",
        "outputId": "b8dfca85-8b7a-4321-da77-ec7eea1843e9"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/3.21k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "137fa30b14f34f57a0beb8a6c6e60cf5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading metadata:   0%|          | 0.00/1.69k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "47463520a286401ba3d5d29fce07ede5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading readme:   0%|          | 0.00/4.87k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0926a24353a94b82a8e405ec72bef775"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading and preparing dataset sms_spam/plain_text to /root/.cache/huggingface/datasets/sms_spam/plain_text/1.0.0/53f051d3b5f62d99d61792c91acefe4f1577ad3e4c216fb0ad39e30b9f20019c...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/203k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "320ba3e1e74541288c307eedbd5e2754"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split:   0%|          | 0/5574 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a950c7427a174b22abdd17fb7710ece7"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset sms_spam downloaded and prepared to /root/.cache/huggingface/datasets/sms_spam/plain_text/1.0.0/53f051d3b5f62d99d61792c91acefe4f1577ad3e4c216fb0ad39e30b9f20019c. Subsequent calls will reuse this data.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fddd4ee4bc054b0f90ed88018fc3e3a0"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset['train'][0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JKFHPko3OnAV",
        "outputId": "2048bc4f-4d5f-45e4-e5c9-0be61d9d7349"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'sms': 'Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...\\n',\n",
              " 'label': 0}"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Modyfikacja datasetu - klasyfikacja"
      ],
      "metadata": {
        "id": "l140vJrgYxPr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "parsed_dataset = []\n",
        "\n",
        "for row in dataset['train']:\n",
        "  text = \"binary classification: \" + row['sms'].replace(\"\\n\", \"\")\n",
        "  new_row = {}\n",
        "  new_row['sms'] = text\n",
        "  if row['label'] == 0:\n",
        "    new_row['label'] = \"0\"\n",
        "  else:\n",
        "    new_row['label'] = \"1\"\n",
        "  parsed_dataset.append(new_row)\n",
        "\n",
        "parsed_dataset[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1boUF-YiY3_y",
        "outputId": "fed6fa9c-8699-4727-ae1b-37475f831b61"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'sms': 'binary classification: Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...',\n",
              " 'label': '0'}"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Tokenizer T5"
      ],
      "metadata": {
        "id": "O-J-jBDxPJcn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import T5Tokenizer"
      ],
      "metadata": {
        "id": "P23AYPX1PZ6g"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = T5Tokenizer.from_pretrained('t5-base')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203,
          "referenced_widgets": [
            "6e2b903343ad49c89339a38a1c626619",
            "b1e96f5c00d048c69c1c0fadeb31dcd9",
            "ec994f6daafb4ef8a08371f2394918dd",
            "699d9f4479854372ada35ab38fe80352",
            "cb36ca390eac4c76ad8cfbcbdb5b6950",
            "5ffe2a0b8e3342a9b764fbbdf1395f1c",
            "ea92e7a968d4479e842c368ece4b60c1",
            "95a4496f03414cf8a8d7cf5e6cc3f37b",
            "7b1d15df592048fe8a8c043e7a8461ad",
            "cc73442788a3462a8d5d53c9c799df7a",
            "ee75218b2c4047fdb265df7a54feea78",
            "e720c7e5ef0849918eb6e7123673c95e",
            "8b2ff14cab9941388b547140d06e1dd5",
            "0eb67bf20ecf4cc0b5f3bd0589440e6b",
            "4e976d7959e640f4b098d9a02320f228",
            "48de3465dc194fff9903fd3813aae91a",
            "29558f7cc7574024879d274548ac4cd7",
            "ce5531904574465d84d65365b0fc2951",
            "10cc03c556cf4fb791697277b3deef35",
            "2113acabfb014e7ab55d099d28845914",
            "504b320daae543b78b8777cebbe65dea",
            "5ea6f5fa80184fe58ef86a536ec0f8f0"
          ]
        },
        "id": "q5Jz0E_oPMBr",
        "outputId": "bbe8a564-fee5-42ef-da4d-04ae4af111db"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)ve/main/spiece.model:   0%|          | 0.00/792k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6e2b903343ad49c89339a38a1c626619"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/1.21k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e720c7e5ef0849918eb6e7123673c95e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/transformers/models/t5/tokenization_t5.py:163: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5.\n",
            "For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`.\n",
            "- Be aware that you SHOULD NOT rely on t5-base automatically truncating your input to 512 when padding/encoding.\n",
            "- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding.\n",
            "- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value.\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sms = parsed_dataset[0]['sms']\n",
        "print('Original: ', sms)\n",
        "print('Tokenized: ', tokenizer.tokenize(sms))\n",
        "print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sms)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dfxJQpoePsvI",
        "outputId": "fa44a9cd-aff1-4b64-957e-52595dad7472"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original:  binary classification: Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...\n",
            "Tokenized:  ['▁binary', '▁classification', ':', '▁Go', '▁until', '▁jur', 'ong', '▁point', ',', '▁crazy', '.', '.', '▁Available', '▁only', '▁in', '▁bug', 'is', '▁', 'n', '▁great', '▁world', '▁la', '▁', 'e', '▁buffet', '...', '▁Cine', '▁there', '▁got', '▁', 'a', 'more', '▁wa', 't', '...']\n",
            "Token IDs:  [14865, 13774, 10, 1263, 552, 10081, 2444, 500, 6, 6139, 5, 5, 8144, 163, 16, 8143, 159, 3, 29, 248, 296, 50, 3, 15, 15385, 233, 17270, 132, 530, 3, 9, 3706, 8036, 17, 233]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Check maximum lenght of a sentence"
      ],
      "metadata": {
        "id": "UpluhM8cU5Ir"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "max_len = 0\n",
        "\n",
        "for sentence in parsed_dataset:\n",
        "    input_ids = tokenizer.encode(sentence['sms'], add_special_tokens=True)\n",
        "    max_len = max(max_len, len(input_ids))\n",
        "\n",
        "print('Max sentence length: ', max_len)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7uNUkixPU85O",
        "outputId": "2ec78c60-f5ae-4201-c8e5-30208c94efab"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Max sentence length:  341\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "max_label_len = 0\n",
        "\n",
        "for sentence in parsed_dataset:\n",
        "    input_ids = tokenizer.encode(sentence['label'], add_special_tokens=True)\n",
        "    max_label_len = max(max_label_len, len(input_ids))\n",
        "\n",
        "print('Max sentence length: ', max_label_len)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lj0issBznZfK",
        "outputId": "2fb86f95-c0a0-45ea-a36d-a6b174f32aac"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Max sentence length:  3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Pre train tokenization"
      ],
      "metadata": {
        "id": "nfw62HdgSERb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch"
      ],
      "metadata": {
        "id": "KTXYalS1VLqH"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "input_ids = []\n",
        "target_ids = []\n",
        "attention_masks = []\n",
        "\n",
        "for sentence in parsed_dataset:\n",
        "    encoded_dict = tokenizer.encode_plus(\n",
        "                        sentence['sms'],\n",
        "                        add_special_tokens = True,\n",
        "                        max_length = 341,\n",
        "                        padding = 'max_length',\n",
        "                        truncation=True,\n",
        "                        return_attention_mask = True,\n",
        "                        return_tensors = 'pt',\n",
        "                   )\n",
        "    \n",
        "    encoded_target_dict = tokenizer.encode_plus(\n",
        "                        sentence['label'],\n",
        "                        add_special_tokens = True,\n",
        "                        max_length = 3,\n",
        "                        padding = 'max_length',\n",
        "                        truncation=True,\n",
        "                        return_attention_mask = True,\n",
        "                        return_tensors = 'pt',\n",
        "                   )\n",
        "      \n",
        "    input_ids.append(encoded_dict['input_ids'])\n",
        "    target_ids.append(encoded_target_dict['input_ids'])\n",
        "    attention_masks.append(encoded_dict['attention_mask'])\n",
        "\n",
        "input_ids = torch.cat(input_ids, dim=0)\n",
        "target_ids = torch.cat(target_ids, dim=0)\n",
        "attention_masks = torch.cat(attention_masks, dim=0)\n",
        "\n",
        "print('Original: ', parsed_dataset[0])\n",
        "print('Token IDs:', input_ids[0])\n",
        "print('Label token IDs:', target_ids[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z28QYfLnSGxR",
        "outputId": "e90e2369-25d1-4fc1-b7fd-b805eaf1f5de"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original:  {'sms': 'binary classification: Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...', 'label': '0'}\n",
            "Token IDs: tensor([14865, 13774,    10,  1263,   552, 10081,  2444,   500,     6,  6139,\n",
            "            5,     5,  8144,   163,    16,  8143,   159,     3,    29,   248,\n",
            "          296,    50,     3,    15, 15385,   233, 17270,   132,   530,     3,\n",
            "            9,  3706,  8036,    17,   233,     1,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0])\n",
            "Label token IDs: tensor([  3, 632,   1])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Split dataset"
      ],
      "metadata": {
        "id": "qD_t0y0KVVSy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.utils.data import TensorDataset, random_split"
      ],
      "metadata": {
        "id": "vN_SatRIVa4c"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = TensorDataset(input_ids, attention_masks, target_ids)\n",
        "\n",
        "test_size = 1000\n",
        "dataset_len = len(dataset)\n",
        "train_size = int(0.9 * (dataset_len-test_size))\n",
        "val_size = (dataset_len-test_size) - train_size\n",
        "\n",
        "test_dataset, train_dataset, val_dataset = random_split(dataset, [test_size, train_size, val_size])\n",
        "\n",
        "print('{:>5,} test samples'.format(test_size))\n",
        "print('{:>5,} training samples'.format(train_size))\n",
        "print('{:>5,} validation samples'.format(val_size))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Mm6vc6lLVW3l",
        "outputId": "af8a7007-791f-426c-c277-1c77a1fd9d78"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1,000 test samples\n",
            "4,116 training samples\n",
            "  458 validation samples\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Create train and validation loaders"
      ],
      "metadata": {
        "id": "bmgQOP4EVfA1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler"
      ],
      "metadata": {
        "id": "CxnQ3cmIVlNh"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "batch_size = 16\n",
        "\n",
        "train_dataloader = DataLoader(\n",
        "            train_dataset,\n",
        "            sampler = RandomSampler(train_dataset),\n",
        "            batch_size = batch_size\n",
        "        )\n",
        "\n",
        "validation_dataloader = DataLoader(\n",
        "            val_dataset,\n",
        "            sampler = SequentialSampler(val_dataset),\n",
        "            batch_size = batch_size\n",
        "        )"
      ],
      "metadata": {
        "id": "0hcpO_onVjEC"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Device check"
      ],
      "metadata": {
        "id": "efwhqLyyVu9z"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "if torch.cuda.is_available():     \n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ANBCfNGnVwVk",
        "outputId": "02086b95-30b8-4be0-aa4b-ac0041b4b007"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla T4\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Load T5 model"
      ],
      "metadata": {
        "id": "okTx_ynMV0rH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import T5ForConditionalGeneration"
      ],
      "metadata": {
        "id": "Eu-7Eed8WgN0"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = T5ForConditionalGeneration.from_pretrained('t5-base')\n",
        "\n",
        "model.cuda()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "b1d7a5cf900b48408e515baa4c66a1cd",
            "6846a2acd95b45a3a0e2cb79f552f0c0",
            "fce901d34cc34feeb92854999e98c0f9",
            "5a89ce40643247fda326742531912a01",
            "bcc7a0cd035e485680b41e7c4a78b8f8",
            "bd0a578fefb44fb4b2662d59fd2ff12e",
            "6585bd6115c047fd881c0bfd323142f0",
            "dbc7f7aa90174ff68b5cc829a6fd8690",
            "ca3a8e4611c6422380351b947882876a",
            "2470365762844b62a09dc6fa818c4a09",
            "3f2489ce0ae941a1a720c60a3052ee70",
            "0795a8385c68409fb5539b9ea6756a47",
            "05dfc6dc9f78483da34b2c6513315e7d",
            "5cfe28a638cb42fc914dc81eb02a46f4",
            "d061dcb2f3e840ec9ba6a6ec4d972619",
            "df418dee3efd4da8aa57ca0044190b2e",
            "9d3d394c756d4eabb0f3fd66ba8ef05a",
            "00612595fa42467a83aa6e4b55343339",
            "33521be9887b4c368915b4f8f2438440",
            "990a862f07894fa9b9f08d3bb7e082ca",
            "1b793ae9c46740bdbbec5e617a899683",
            "cbfde7f5f0204417abdced523c5621e9"
          ]
        },
        "id": "JKv9O8kfV2zZ",
        "outputId": "b4b823d6-f7dc-4b78-a12b-4a2bae4e463f"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)\"pytorch_model.bin\";:   0%|          | 0.00/892M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b1d7a5cf900b48408e515baa4c66a1cd"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)neration_config.json:   0%|          | 0.00/147 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0795a8385c68409fb5539b9ea6756a47"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "T5ForConditionalGeneration(\n",
              "  (shared): Embedding(32128, 768)\n",
              "  (encoder): T5Stack(\n",
              "    (embed_tokens): Embedding(32128, 768)\n",
              "    (block): ModuleList(\n",
              "      (0): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (relative_attention_bias): Embedding(32, 12)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (1): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (2): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (3): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (4): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (5): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (6): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (7): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (8): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (9): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (10): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (11): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (final_layer_norm): T5LayerNorm()\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "  )\n",
              "  (decoder): T5Stack(\n",
              "    (embed_tokens): Embedding(32128, 768)\n",
              "    (block): ModuleList(\n",
              "      (0): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (relative_attention_bias): Embedding(32, 12)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (1): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (2): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (3): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (4): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (5): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (6): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (7): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (8): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (9): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (10): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (11): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseActDense(\n",
              "              (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
              "              (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): ReLU()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (final_layer_norm): T5LayerNorm()\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "  )\n",
              "  (lm_head): Linear(in_features=768, out_features=32128, bias=False)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Helper functions"
      ],
      "metadata": {
        "id": "F_SDAwxoawDy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import datetime\n",
        "import numpy as np"
      ],
      "metadata": {
        "id": "s-q6_F38bLVA"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def calculate_accuracy(preds, target):\n",
        "  results_ok = 0.0\n",
        "  results_false = 0.0\n",
        "\n",
        "  for idx, pred in enumerate(preds):\n",
        "    if pred == target[idx]:\n",
        "      results_ok += 1.0\n",
        "    else:\n",
        "      results_false += 1.0\n",
        "\n",
        "  return results_ok / (results_ok + results_false)\n",
        "\n",
        "def format_time(elapsed):\n",
        "    '''\n",
        "    Takes a time in seconds and returns a string hh:mm:ss\n",
        "    '''\n",
        "    elapsed_rounded = int(round((elapsed)))\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))"
      ],
      "metadata": {
        "id": "FzUi8908ax61"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Init training"
      ],
      "metadata": {
        "id": "ucChBa-9bXJy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import get_linear_schedule_with_warmup"
      ],
      "metadata": {
        "id": "c9e7rbGwbdEp"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "optimizer = torch.optim.AdamW(model.parameters(),\n",
        "                  lr = 3e-4,\n",
        "                  eps = 1e-8\n",
        "                )\n",
        "\n",
        "epochs = 4\n",
        "\n",
        "total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
        "                                            num_warmup_steps = 0,\n",
        "                                            num_training_steps = total_steps)"
      ],
      "metadata": {
        "id": "A7XUF4PNbYy8"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Training"
      ],
      "metadata": {
        "id": "DAzQWODja0A3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "import time"
      ],
      "metadata": {
        "id": "Hoa7NlU0bI7G"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# This training code is based on the `run_glue.py` script here:\n",
        "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
        "\n",
        "seed_val = 42\n",
        "\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "training_stats = []\n",
        "total_t0 = time.time()\n",
        "\n",
        "for epoch_i in range(0, epochs):\n",
        "    \n",
        "    # ========================================\n",
        "    #               Training\n",
        "    # ========================================\n",
        "\n",
        "    print(\"\")\n",
        "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "    print('Training...')\n",
        "\n",
        "    t0 = time.time()\n",
        "    total_train_loss = 0\n",
        "    total_train_acc = 0\n",
        "\n",
        "    model.train()\n",
        "\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "        if step % 40 == 0 and not step == 0:\n",
        "            elapsed = format_time(time.time() - t0)\n",
        "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
        "\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "\n",
        "        y = batch[2].to(device)\n",
        "        y_ids = y[:, :-1].contiguous()\n",
        "        lm_labels = y[:, 1:].clone().detach()\n",
        "        lm_labels[y[:, 1:] == tokenizer.pad_token_id] = -100\n",
        "\n",
        "        model.zero_grad()        \n",
        "\n",
        "        outputs = model(\n",
        "            input_ids=b_input_ids,\n",
        "            attention_mask=b_input_mask,\n",
        "            decoder_input_ids=y_ids,\n",
        "            labels=lm_labels\n",
        "        )\n",
        "\n",
        "        generated_ids = model.generate(\n",
        "            input_ids = b_input_ids,\n",
        "            attention_mask = b_input_mask, \n",
        "            max_length=2, \n",
        "            num_beams=2,\n",
        "            repetition_penalty=2.5, \n",
        "            length_penalty=1.0, \n",
        "            early_stopping=True\n",
        "        )\n",
        "\n",
        "        preds = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=True) for g in generated_ids]\n",
        "        target = [tokenizer.decode(t, skip_special_tokens=True, clean_up_tokenization_spaces=True) for t in y]\n",
        "        total_train_acc += calculate_accuracy(preds, target)   \n",
        "\n",
        "        loss = outputs['loss']\n",
        "        total_train_loss += loss.item()\n",
        "\n",
        "        loss.backward()\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "        optimizer.step()\n",
        "        scheduler.step()\n",
        "\n",
        "    avg_train_loss = total_train_loss / len(train_dataloader)    \n",
        "    avg_train_acc = total_train_acc / len(train_dataloader)          \n",
        "    training_time = format_time(time.time() - t0)\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "    print(\"  Average training acc: {0:.2f}\".format(avg_train_acc))\n",
        "    print(\"  Training epcoh took: {:}\".format(training_time))\n",
        "        \n",
        "    # ========================================\n",
        "    #               Validation\n",
        "    # ========================================\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"Running Validation...\")\n",
        "\n",
        "    t0 = time.time()\n",
        "    model.eval()\n",
        "\n",
        "    total_eval_loss = 0\n",
        "    total_eval_accuracy = 0\n",
        "\n",
        "    for batch in validation_dataloader:\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "\n",
        "        y = batch[2].to(device)\n",
        "        y_ids = y[:, :-1].contiguous()\n",
        "        lm_labels = y[:, 1:].clone().detach()\n",
        "        lm_labels[y[:, 1:] == tokenizer.pad_token_id] = -100\n",
        "        \n",
        "        with torch.no_grad():        \n",
        "\n",
        "          outputs = model(\n",
        "            input_ids=b_input_ids,\n",
        "            attention_mask=b_input_mask,\n",
        "            decoder_input_ids=y_ids,\n",
        "            labels=lm_labels\n",
        "          )\n",
        "\n",
        "          loss = outputs['loss']\n",
        "          total_eval_loss += loss.item()\n",
        "\n",
        "          generated_ids = model.generate(\n",
        "            input_ids = b_input_ids,\n",
        "            attention_mask = b_input_mask, \n",
        "            max_length=2, \n",
        "            num_beams=2,\n",
        "            repetition_penalty=2.5, \n",
        "            length_penalty=1.0, \n",
        "            early_stopping=True\n",
        "          )\n",
        "\n",
        "          preds = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=True) for g in generated_ids]\n",
        "          target = [tokenizer.decode(t, skip_special_tokens=True, clean_up_tokenization_spaces=True) for t in y]\n",
        "          total_eval_accuracy += calculate_accuracy(preds, target)        \n",
        "\n",
        "    avg_val_loss = total_eval_loss / len(validation_dataloader)\n",
        "\n",
        "    avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)\n",
        "    print(\"  Accuracy: {0:.2f}\".format(avg_val_accuracy))\n",
        "    \n",
        "    validation_time = format_time(time.time() - t0)\n",
        "    print(\"  Validation took: {:}\".format(validation_time))\n",
        "    print(\"  Validation Loss: {0:.2f}\".format(avg_val_loss))\n",
        "\n",
        "    training_stats.append(\n",
        "        {\n",
        "            'epoch': epoch_i + 1,\n",
        "            'Training Loss': avg_train_loss,\n",
        "            'Training Accur.': avg_train_acc,\n",
        "            'Valid. Loss': avg_val_loss,\n",
        "            'Valid. Accur.': avg_val_accuracy,\n",
        "            'Training Time': training_time,\n",
        "            'Validation Time': validation_time\n",
        "        }\n",
        "    )\n",
        "\n",
        "print(\"\")\n",
        "print(\"Training complete!\")\n",
        "\n",
        "print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xsHxfslka1u5",
        "outputId": "60bea81f-a963-4599-ca22-b1992c14a3e5"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "======== Epoch 1 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    258.    Elapsed: 0:01:06.\n",
            "  Batch    80  of    258.    Elapsed: 0:02:13.\n",
            "  Batch   120  of    258.    Elapsed: 0:03:22.\n",
            "  Batch   160  of    258.    Elapsed: 0:04:32.\n",
            "  Batch   200  of    258.    Elapsed: 0:05:42.\n",
            "  Batch   240  of    258.    Elapsed: 0:06:52.\n",
            "\n",
            "  Average training loss: 0.09\n",
            "  Average training acc: 0.42\n",
            "  Training epcoh took: 0:07:22\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.68\n",
            "  Validation took: 0:00:25\n",
            "  Validation Loss: 0.00\n",
            "\n",
            "======== Epoch 2 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    258.    Elapsed: 0:01:10.\n",
            "  Batch    80  of    258.    Elapsed: 0:02:19.\n",
            "  Batch   120  of    258.    Elapsed: 0:03:29.\n",
            "  Batch   160  of    258.    Elapsed: 0:04:39.\n",
            "  Batch   200  of    258.    Elapsed: 0:05:48.\n",
            "  Batch   240  of    258.    Elapsed: 0:06:58.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Average training acc: 0.49\n",
            "  Training epcoh took: 0:07:28\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.72\n",
            "  Validation took: 0:00:25\n",
            "  Validation Loss: 0.00\n",
            "\n",
            "======== Epoch 3 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    258.    Elapsed: 0:01:10.\n",
            "  Batch    80  of    258.    Elapsed: 0:02:19.\n",
            "  Batch   120  of    258.    Elapsed: 0:03:29.\n",
            "  Batch   160  of    258.    Elapsed: 0:04:39.\n",
            "  Batch   200  of    258.    Elapsed: 0:05:49.\n",
            "  Batch   240  of    258.    Elapsed: 0:06:58.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Average training acc: 0.50\n",
            "  Training epcoh took: 0:07:29\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.72\n",
            "  Validation took: 0:00:25\n",
            "  Validation Loss: 0.00\n",
            "\n",
            "======== Epoch 4 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    258.    Elapsed: 0:01:10.\n",
            "  Batch    80  of    258.    Elapsed: 0:02:19.\n",
            "  Batch   120  of    258.    Elapsed: 0:03:29.\n",
            "  Batch   160  of    258.    Elapsed: 0:04:39.\n",
            "  Batch   200  of    258.    Elapsed: 0:05:49.\n",
            "  Batch   240  of    258.    Elapsed: 0:06:58.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Average training acc: 0.50\n",
            "  Training epcoh took: 0:07:29\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.72\n",
            "  Validation took: 0:00:25\n",
            "  Validation Loss: 0.00\n",
            "\n",
            "Training complete!\n",
            "Total training took 0:31:29 (h:mm:ss)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Train summary"
      ],
      "metadata": {
        "id": "xIpFPoRb91Or"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "pd.set_option('precision', 2)\n",
        "df_stats = pd.DataFrame(data=training_stats)\n",
        "\n",
        "df_stats = df_stats.set_index('epoch')\n",
        "df_stats"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "GjYqBrrO93Oh",
        "outputId": "d5742682-1cb4-4910-ab30-9424671b31e4"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       Training Loss  Training Accur.  Valid. Loss  Valid. Accur.  \\\n",
              "epoch                                                               \n",
              "1           8.67e-02             0.42     1.46e-08           0.68   \n",
              "2           2.02e-06             0.49     2.65e-10           0.72   \n",
              "3           1.50e-06             0.50     0.00e+00           0.72   \n",
              "4           1.10e-06             0.50     0.00e+00           0.72   \n",
              "\n",
              "      Training Time Validation Time  \n",
              "epoch                                \n",
              "1           0:07:22         0:00:25  \n",
              "2           0:07:28         0:00:25  \n",
              "3           0:07:29         0:00:25  \n",
              "4           0:07:29         0:00:25  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4b30f186-a8f8-4cc3-ab45-b65be984cbf4\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Training Accur.</th>\n",
              "      <th>Valid. Loss</th>\n",
              "      <th>Valid. Accur.</th>\n",
              "      <th>Training Time</th>\n",
              "      <th>Validation Time</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>epoch</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>8.67e-02</td>\n",
              "      <td>0.42</td>\n",
              "      <td>1.46e-08</td>\n",
              "      <td>0.68</td>\n",
              "      <td>0:07:22</td>\n",
              "      <td>0:00:25</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2.02e-06</td>\n",
              "      <td>0.49</td>\n",
              "      <td>2.65e-10</td>\n",
              "      <td>0.72</td>\n",
              "      <td>0:07:28</td>\n",
              "      <td>0:00:25</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.50e-06</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.00e+00</td>\n",
              "      <td>0.72</td>\n",
              "      <td>0:07:29</td>\n",
              "      <td>0:00:25</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1.10e-06</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.00e+00</td>\n",
              "      <td>0.72</td>\n",
              "      <td>0:07:29</td>\n",
              "      <td>0:00:25</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4b30f186-a8f8-4cc3-ab45-b65be984cbf4')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4b30f186-a8f8-4cc3-ab45-b65be984cbf4 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4b30f186-a8f8-4cc3-ab45-b65be984cbf4');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "import seaborn as sns\n",
        "\n",
        "sns.set(style='darkgrid')\n",
        "\n",
        "sns.set(font_scale=1.5)\n",
        "plt.rcParams[\"figure.figsize\"] = (12,6)\n",
        "\n",
        "plt.plot(df_stats['Training Loss'], 'b-o', label=\"Training\")\n",
        "plt.plot(df_stats['Valid. Loss'], 'g-o', label=\"Validation\")\n",
        "\n",
        "plt.title(\"Training & Validation Loss\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.legend()\n",
        "plt.xticks([1, 2, 3, 4])\n",
        "\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 427
        },
        "id": "Xk3gzkeU96v3",
        "outputId": "11211824-04f4-43b2-984f-56f9965d38bc"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvAAAAGaCAYAAABpIXfbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd1gU1/4/8PcuywLSpQhRRMTQFFAsWIgFC4jYsV+xlxij6eo1PTfmGzXRaKK5iordKIIVK5bExBIrIRQVLCCIiIKAtGX394c/9mZd0AUXZoH363nuk8uZOWc+uzDJe2fPnBEpFAoFiIiIiIioThALXQAREREREWmOAZ6IiIiIqA5hgCciIiIiqkMY4ImIiIiI6hAGeCIiIiKiOoQBnoiIiIioDmGAJ6IGLy0tDa6urli5cmW1x5g/fz5cXV21WFX9Vdn77erqivnz52s0xsqVK+Hq6oq0tDSt1xcZGQlXV1ecP39e62MTEWmDROgCiIieV5UgHBMTg2bNmtVgNXXP06dP8fPPPyM6OhoPHjxA48aN0b59e8yaNQvOzs4ajTFnzhwcOXIEe/bsgbu7e4X7KBQK9O7dG0+ePMGZM2dgaGiozZdRo86fP48LFy5gwoQJMDMzE7ocNWlpaejduzfGjRuHTz/9VOhyiEjHMMATkc5ZvHixys+XLl3CL7/8glGjRqF9+/Yq2xo3bvzKx2vatCliY2Ohp6dX7TG++uorfPHFF69cizZ8/PHHOHjwIIKDg9GpUydkZWXhxIkTuHbtmsYBPiQkBEeOHMHu3bvx8ccfV7jPuXPncO/ePYwaNUor4T02NhZice18MXzhwgX8+OOPGDp0qFqAHzx4MAYMGAB9ff1aqYWIqKoY4IlI5wwePFjl57KyMvzyyy9o27at2rbn5efnw8TEpErHE4lEMDAwqHKd/6QrYa+wsBCHDx+Gn58fvvvuO2X77NmzUVJSovE4fn5+sLe3x/79+/HRRx9BKpWq7RMZGQngWdjXhlf9HWiLnp7eK32YIyKqaZwDT0R1lr+/P8aPH4/4+HhMmTIF7du3x6BBgwA8C/LLli3DiBEj4OvrizZt2qBv375YunQpCgsLVcapaE72P9tOnjyJ4cOHw9PTE35+fvj2228hk8lUxqhoDnx5W15eHj777DN06dIFnp6eGD16NK5du6b2eh4/fowFCxbA19cX7dq1Q2hoKOLj4zF+/Hj4+/tr9J6IRCKIRKIKP1BUFMIrIxaLMXToUOTk5ODEiRNq2/Pz83H06FG4uLjAy8urSu93ZSqaAy+Xy/Hf//4X/v7+8PT0RHBwMPbt21dh/+TkZHz++ecYMGAA2rVrB29vbwwbNgy7du1S2W/+/Pn48ccfAQC9e/eGq6uryu+/sjnwjx49whdffIEePXqgTZs26NGjB7744gs8fvxYZb/y/mfPnsW6devQp08ftGnTBgEBAYiKitLovaiKxMREvPXWW/D19YWnpyeCgoKwdu1alJWVqeyXkZGBBQsWoFevXmjTpg26dOmC0aNHq9Qkl8sRHh6OgQMHol27dvDx8UFAQAD+/e9/o7S0VOu1E1H18Ao8EdVp6enpmDBhAgIDA9GvXz88ffoUAJCZmYmIiAj069cPwcHBkEgkuHDhAsLCwpCQkIB169ZpNP7p06exbds2jB49GsOHD0dMTAzWr18Pc3NzzJw5U6MxpkyZgsaNG+Ott95CTk4ONmzYgOnTpyMmJkb5bUFJSQkmTZqEhIQEDBs2DJ6enkhKSsKkSZNgbm6u8fthaGiIIUOGYPfu3Thw4ACCg4M17vu8YcOGYfXq1YiMjERgYKDKtoMHD6KoqAjDhw8HoL33+3nffPMNNm3ahI4dO2LixInIzs7Gl19+CQcHB7V9L1y4gIsXL6Jnz55o1qyZ8tuIjz/+GI8ePcKMGTMAAKNGjUJ+fj6OHTuGBQsWwNLSEsCL773Iy8vDmDFjcOfOHQwfPhweHh5ISEjA9u3bce7cOezatUvtm59ly5ahqKgIo0aNglQqxfbt2zF//nw0b95cbSpYdf31118YP348JBIJxo0bB2tra5w8eRJLly5FYmKi8lsYmUyGSZMmITMzE2PHjkWLFi2Qn5+PpKQkXLx4EUOHDgUArF69GitWrECvXr0wevRo6OnpIS0tDSdOnEBJSYnOfNNE1OApiIh03O7duxUuLi6K3bt3q7T36tVL4eLioti5c6dan+LiYkVJSYla+7JlyxQuLi6Ka9euKdtSU1MVLi4uihUrVqi1eXt7K1JTU5XtcrlcMWDAAEW3bt1Uxp03b57CxcWlwrbPPvtMpT06Olrh4uKi2L59u7Jty5YtChcXF8WqVatU9i1v79Wrl9prqUheXp5i2rRpijZt2ig8PDwUBw8e1KhfZUJDQxXu7u6KzMxMlfaRI0cqWrdurcjOzlYoFK/+fisUCoWLi4ti3rx5yp+Tk5MVrq6uitDQUIVMJlO2x8XFKVxdXRUuLi4qv5uCggK145eVlSn+9a9/KXx8fFTqW7FihVr/cuV/b+fOnVO2ff/99woXFxfFli1bVPYt//0sW7ZMrf/gwYMVxcXFyvb79+8rWrdurXj33XfVjvm88vfoiy++eOF+o0aNUri7uysSEhKUbXK5XDFnzhyFi4uL4o8//lAoFApFQkKCwsXFRbFmzZoXjjdkyBBF//79X1ofEQmLU2iIqE6zsLDAsGHD1NqlUqnyaqFMJkNubi4ePXqErl27AkCFU1gq0rt3b5VVbkQiEXx9fZGVlYWCggKNxpg4caLKz507dwYA3LlzR9l28uRJ6OnpITQ0VGXfESNGwNTUVKPjyOVyzJ07F4mJiTh06BC6d++ODz74APv371fZ75NPPkHr1q01mhMfEhKCsrIy7NmzR9mWnJyMq1evwt/fX3kTsbbe73+KiYmBQqHApEmTVOakt27dGt26dVPbv1GjRsr/X1xcjMePHyMnJwfdunVDfn4+UlJSqlxDuWPHjqFx48YYNWqUSvuoUaPQuHFjHD9+XK3P2LFjVaYtNWnSBE5OTrh9+3a16/in7OxsXLlyBf7+/nBzc1O2i0QivPnmm8q6ASj/hs6fP4/s7OxKxzQxMUFmZiYuXryolRqJqGZwCg0R1WkODg6V3nC4detW7NixAzdv3oRcLlfZlpubq/H4z7OwsAAA5OTkwNjYuMpjlE/ZyMnJUbalpaXB1tZWbTypVIpmzZrhyZMnLz1OTEwMzpw5gyVLlqBZs2b44YcfMHv2bHz00UeQyWTKaRJJSUnw9PTUaE58v379YGZmhsjISEyfPh0AsHv3bgBQTp8pp433+59SU1MBAC1btlTb5uzsjDNnzqi0FRQU4Mcff8ShQ4eQkZGh1keT97AyaWlpaNOmDSQS1f9sSiQStGjRAvHx8Wp9KvvbuXfvXrXreL4mAGjVqpXatpYtW0IsFivfw6ZNm2LmzJlYs2YN/Pz84O7ujs6dOyMwMBBeXl7Kfu+99x7eeustjBs3Dra2tujUqRN69uyJgICAKt1DQUQ1iwGeiOo0IyOjCts3bNiA//u//4Ofnx9CQ0Nha2sLfX19ZGZmYv78+VAoFBqN/6LVSF51DE37a6r8psuOHTsCeBb+f/zxR7z55ptYsGABZDIZ3NzccO3aNXz99dcajWlgYIDg4GBs27YNly9fhre3N/bt2wc7Ozu88cYbyv209X6/ivfffx+nTp3CyJEj0bFjR1hYWEBPTw+nT59GeHi42oeKmlZbS2Jq6t1330VISAhOnTqFixcvIiIiAuvWrcPUqVPx4YcfAgDatWuHY8eO4cyZMzh//jzOnz+PAwcOYPXq1di2bZvywysRCYsBnojqpb1796Jp06ZYu3atSpD69ddfBayqck2bNsXZs2dRUFCgchW+tLQUaWlpGj1sqPx13rt3D/b29gCehfhVq1Zh5syZ+OSTT9C0aVO4uLhgyJAhGtcWEhKCbdu2ITIyErm5ucjKysLMmTNV3teaeL/Lr2CnpKSgefPmKtuSk5NVfn7y5AlOnTqFwYMH48svv1TZ9scff6iNL
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Create test loader"
      ],
      "metadata": {
        "id": "UJlKxl0r-W-m"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "prediction_dataloader = DataLoader(\n",
        "            test_dataset,\n",
        "            sampler = SequentialSampler(test_dataset),\n",
        "            batch_size = batch_size\n",
        "        )"
      ],
      "metadata": {
        "id": "eQGsEEDh-YxG"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Evaluate on test dataset"
      ],
      "metadata": {
        "id": "gHSDNWvA-aq9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print('Predicting labels for {:,} test sentences...'.format(len(test_dataset)))\n",
        "\n",
        "model.eval()\n",
        "predictions , true_labels = [], []\n",
        "\n",
        "for batch in prediction_dataloader:\n",
        "\n",
        "    b_input_ids = batch[0].to(device)\n",
        "    b_input_mask = batch[1].to(device)\n",
        "    y = batch[2].to(device)\n",
        "    \n",
        "    with torch.no_grad():        \n",
        "\n",
        "      generated_ids = model.generate(\n",
        "        input_ids = b_input_ids,\n",
        "        attention_mask = b_input_mask, \n",
        "        max_length=2, \n",
        "        num_beams=2,\n",
        "        repetition_penalty=2.5, \n",
        "        length_penalty=1.0, \n",
        "        early_stopping=True\n",
        "      )\n",
        "        \n",
        "    preds = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=True) for g in generated_ids]\n",
        "    target = [tokenizer.decode(t, skip_special_tokens=True, clean_up_tokenization_spaces=True) for t in y]\n",
        "\n",
        "    predictions.append(preds)\n",
        "    true_labels.append(target)\n",
        "\n",
        "print('    DONE.')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OPcQkHnJ-c9A",
        "outputId": "ba586e95-e91f-4424-aee0-67493db5a99a"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Predicting labels for 1,000 test sentences...\n",
            "    DONE.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "results_ok = 0\n",
        "results_false = 0\n",
        "for idx, true_labels_batch in enumerate(true_labels):\n",
        "  for bidx, true_label in enumerate(true_labels_batch):\n",
        "    if true_label == predictions[idx][bidx]:\n",
        "      results_ok += 1\n",
        "    else:\n",
        "      results_false += 1\n",
        "\n",
        "print(\"Correct predictions: {}, incorrect results: {}, accuracy: {}\".format(results_ok, results_false, float(results_ok) / (results_ok + results_false)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ifz56jYW-zBN",
        "outputId": "005fd31f-28de-422a-9ccf-d1a15568c706"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Correct predictions: 738, incorrect results: 262, accuracy: 0.738\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Sample prediction: {}, expected: {}\".format(predictions[2][0], true_labels[2][0]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1LqVo4wW-2g-",
        "outputId": "084746aa-4677-4c54-b2ac-9d9af4a2488c"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sample prediction: 0, expected: 0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# MCC Score"
      ],
      "metadata": {
        "id": "dLYc9WXz_B1o"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import matthews_corrcoef\n",
        "\n",
        "matthews_set = []\n",
        "print('Calculating Matthews Corr. Coef. for each batch...')\n",
        "\n",
        "for i in range(len(true_labels)):\n",
        "  matthews = matthews_corrcoef(true_labels[i], predictions[i])                \n",
        "  matthews_set.append(matthews)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hPEPpXXX_DXR",
        "outputId": "eaac6737-fc5b-4aee-c7e3-3c2cffedb073"
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Calculating Matthews Corr. Coef. for each batch...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ax = sns.barplot(x=list(range(len(matthews_set))), y=matthews_set, ci=None)\n",
        "\n",
        "plt.title('MCC Score per Batch')\n",
        "plt.ylabel('MCC Score (-1 to +1)')\n",
        "plt.xlabel('Batch #')\n",
        "\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 427
        },
        "id": "qjtAYcme_EyM",
        "outputId": "697cb818-7af9-4237-e9c4-1712222ec85b"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvgAAAGaCAYAAAB69jcvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde1yUdd7/8fcAM4wcBA9gpuK6KlIqnsrSdF0PKZVn8VRKZqmb2e7az0K3u+69vdssYzddtU1LTdHKAyAeNnNt791aszTzliw0dc00Np1CEFAYhPn94Q0bwVzM6Mwg4+v5ePh4yPf7ub7X55q5Bj5cfK/vZXI4HA4BAAAA8AsBdZ0AAAAAAM+hwAcAAAD8CAU+AAAA4Eco8AEAAAA/QoEPAAAA+BEKfAAAAMCPUOADAFCPTZ48WQMGDKjrNABcR4LqOgEAqAsff/yxkpKSJEkPPPCAnn322Wox33//vfr166fS0lL17NlTqamp1WI+++wzrV+/Xvv375fNZlNAQIBatmypXr16acKECWrbtm2V+EuXLmnDhg3atWuXjh8/rqKiIkVERKhjx4665557NHz4cAUFGX9rLigoUGpqqt5991198803KisrU6NGjRQXF6f+/ftr7Nix1/DK4McGDBigb775pvJrk8mkJk2aqE2bNpo4caLuu+++qx579+7dys7O1uOPP+6JVAFAEgU+gBtccHCwtm/frrlz58pisVTpy8zMlMPhcFpwL126VEuXLlWjRo00dOhQtWvXTuXl5Tp+/LjeeecdrV+/Xvv27VNYWJgk6dSpU5o+fbq++uor9e7dW9OnT1ejRo30/fffa+/evZo3b56OHz+up556ymm+hYWFSkxM1OnTpzVkyBCNGTNGZrNZp0+f1qeffqq1a9dS4HvBTTfdpCeeeEKSVF5errNnzyojI0NPPPGEbDabpkyZclXj7t69WxkZGRT4ADyKAh/ADe3uu+/W9u3btXv3bt17771V+tLT0/Wzn/1MH330UbXtNm/erCVLluiOO+7QsmXLFB4eXqX/ySef1NKlSyu/Li4u1owZM3TmzBktWbJEgwcPrhI/ffp0ZWVl6bPPPjPMd+PGjfrqq6/0m9/8Rg8++GC1fpvNVusxe0NhYWHlLzL1icPh0MWLFxUaGmoYFx4erhEjRlRpGz9+vPr27av09PSrLvABwBuYgw/ghnbrrbeqQ4cOSk9Pr9KelZWlY8eOacyYMdW2sdvtWrRokUJCQrRo0aJqxb0kWa1WzZkzp7Lo3bRpk06ePKmHHnqoWnFfIT4+Xg888IBhvl999ZUkqVevXjX2R0VFVWs7deqU5s2bp5/97Gfq1KmT+vTpo0cffVSHDx+uErd7925NmDBBXbt2Vbdu3TRhwgTt3r272ngDBgzQ5MmT9cUXX+jhhx9Wjx49NHz48Co5Pvnkk+rTp486deqkAQMG6MUXX9TFixcNj+3H43/++edKSkpSt27d1LNnTyUnJ+v777+vFm+32/Xqq6/qvvvuU+fOnXXbbbfpF7/4hb744osqcR9//HHle71+/Xrde++96ty5s1atWuVSXj8WEREhi8Uis9lcpT0rK0tz587VkCFD1KVLl8rX8i9/+UuVuMmTJysjI0OS1KFDh8p/PzwXbTabnnvuOQ0cOFCdOnVSr1699NBDD2nPnj3V8jl79qyeeOIJ3X777erSpYsefvhhnTx58qqODUD9xhV8ADe8MWPG6IUXXtDZs2fVrFkzSVeu0Ddp0kQ///nPq8V/+umnstlsGjFihBo3buzSPt59911JV676XouYmBhJV/66MGfOnFrn63/22WeaMmWKLl++rMTERLVv3175+fnat2+fDh48qE6dOkmS1q9fr/nz5+unP/2pZs6cKUnKyMjQY489pvnz51fLOycnRw8++KASEhI0ePDgyuL98OHDevDBB9WwYUONHz9ezZo105EjR5SamqqDBw8qNTW1WkFck2+//VZTpkzR4MGDNWTIEH3xxRdKS0vT4cOHtXnzZjVo0ECSVFpaqocfflgHDx7UiBEj9MADD6iwsFAbN27UxIkTtW7dOnXu3LnK2GvWrFFeXp7Gjh2rqKgo3XTTTbXmU1ZWptzcXElXpujYbDatXbtWRUVFmjBhQpXYv/zlL/rnP/+phIQEtWjRQnl5ecrIyNCsWbOUkpKiYcOGSZJ+8YtfqLy8XJ988okWLlxYuX337t0lSWfOnNHEiRP1/fffa8SIEerUqZMuXbqkQ4cO6cMPP9Rdd91Vuc3Fixc1adIkdenSRbNnz9aZM2e0du1azZw5U9u3b1dgYGCtxwjAjzgA4Ab00UcfOWJjYx2vv/66Izc319GxY0fHn/70J4fD4XBcunTJ0aNHD8cLL7zgcDgcjq5duzomTZpUue3atWsdsbGxjlWrVrm8v549ezq6d+9+zXnn5eU5+vXr54iNjXX06tXL8fjjjzuWL1/u2L9/v6OsrKxKbHl5ueO+++5zdOrUyZGdnV1trIr4vLw8R9euXR2DBg1yFBQUVPYXFBQ4Bg4c6OjatasjPz+/sr1///6O2NhYx8aNG6uNOWzYMMeQIUOqjONwOBy7du1yxMbGOtLS0mo9xorxV69eXaV99erVjtjYWMfy5curtb3//vtVYgsKChz9+vWr8r5VvOe3336747vvvqs1jx/n8+N/nTt3drz99tvV4ouKiqq1Xbx40TF48GDHPffcU6U9OTnZERsbW+N+H3nkkRqPzeFwVHmvJ02a5IiNjXWsWLGiSsxrr73mdHsA/o0pOgBueI0aNdKAAQMqp0vs2rVLBQUFNU7Pka7MN5fk1pzzwsLCWud5uyIiIkLp6emaNm2awsPD9e677+r3v/+9HnjgAQ0aNEj/+Mc/KmOzs7N17NgxjR49WnFxcdXGCgi48iNgz549unjxoiZPnlzlmMLCwjR58mRdvHhRH374YZVtIyMjNXr06CptR48e1dGjRzV06FDZ7Xbl5uZW/uvRo4dCQkJqnFpSk7CwMN1///1V2u6//36FhYVVmeqydetW/fSnP1XHjh2r7M9ut6t37946cOCAiouLq4wzYsQINWnSxKU8KrRo0UKrV6/W6tWrtWrVKr3wwgvq0qWLfvvb3yotLa1KbEhISOX/L126pPPnz+vSpUu68847deLEicrzx0heXp4++OAD9e3bV3379q3WX/He/fDrilWhKtx5552SrkzRAnBjYYoOAOjKNJ3p06frk08+UVpamuLj49WuXbsaYyuK4KKiIpfHDwsLcyveSOPGjTVnzhzNmTNH58+f1//+7//qnXfe0datWzVr1ixlZmaqdevWlfP1b731VsPxzpw5I0lq3759tb6KttOnT1dpb9WqVbVpHydOnJAkLVmyREuWLKlxX999913tB/h/4/94VSOLxaJWrVpVyeXEiRMqLi52ek+CJJ0/f17Nmzev/PonP/mJSzn8UEhIiHr37l2lbdiwYRo1apSee+45DRgwQI0aNZJ0ZXnVRYsW6b333qvxnoELFy7U+svh119/LYfDUet7VyE6OlrBwcFV2iIjIyVd+WUBwI2FAh8AJPXp00fNmjXTsmXL9PHHH+u3v/2t09iKovfHN3Eaad++vfbv36/Tp0+rVatW15pupUaNGql///7q37+/mjdvrldffVU7duyonEfvLRVz4GsyderUGq86S1LDhg09mofD4VBsbKzmzZvnNObH90kY5e6OoKAg3XnnnVq7dq2ysrLUr18/ORwOTZ06VSdOnFBSUpI6deqk8PBwBQYGKi0tTdu3b1d5eblH9v9DRnPsHQ6Hx/cH4PpGgQ8AulIgjRw5UsuXL5fVatXQoUOdxnbv3l1RUVHavXu3zp8/X3nl1sjgwYO1f/9+bdq0qXI9dU/r0qWLpCurqUhSmzZtJF2ZqmOk4heOY8eOVbsSfvz48SoxRlq3bi3pynSRH1/tdtfp06dlt9urXMW32+06ffq0fvrTn1bZ5/nz53XnnXdWm7biC5cvX5b077/mHD16VEeOHNFjjz2mX/7yl1ViN23aVG17k8lU4
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "flat_predictions = np.concatenate(predictions, axis=0)\n",
        "flat_true_labels = np.concatenate(true_labels, axis=0)\n",
        "\n",
        "mcc = matthews_corrcoef(flat_true_labels, flat_predictions)\n",
        "print('Total MCC: %.3f' % mcc)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rkonN244_HPz",
        "outputId": "83396102-49b8-4351-ea83-92231720881e"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total MCC: 0.190\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Save model"
      ],
      "metadata": {
        "id": "GPhCp068_Iwq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "\n",
        "drive.mount('/content/gdrive/', force_remount=True)\n",
        "\n",
        "output_dir = '/content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/T5_Model'\n",
        "print(\"Saving model to %s\" % output_dir)\n",
        "\n",
        "model_to_save = model.module if hasattr(model, 'module') else model \n",
        "model_to_save.save_pretrained(output_dir)\n",
        "tokenizer.save_pretrained(output_dir)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "avafCMoS_KDF",
        "outputId": "baf501e6-b6c2-4394-db6c-4a971328d464"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive/\n",
            "Saving model to /content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/T5_Model\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('/content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/T5_Model/tokenizer_config.json',\n",
              " '/content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/T5_Model/special_tokens_map.json',\n",
              " '/content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/T5_Model/spiece.model',\n",
              " '/content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/T5_Model/added_tokens.json')"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Bibliografia\n",
        "- https://github.com/Shivanandroy/T5-Finetuning-PyTorch/blob/main/notebook/T5_Fine_tuning_with_PyTorch.ipynb\n",
        "- https://mccormickml.com/2019/07/22/BERT-fine-tuning/#a1-saving--loading-fine-tuned-model\n",
        "- https://huggingface.co/docs/transformers/model_doc/t5#training"
      ],
      "metadata": {
        "id": "wHzm2_nDA6i-"
      }
    }
  ]
}