Przetwarzanie_tekstu/projekt/GPT2_sms_spam.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "54a468d755ee4d398517315461829a70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7d9cfb12850a48c1a9cfde1118adc2ea",
              "IPY_MODEL_2534269fcfad4bc78214a4a606842072",
              "IPY_MODEL_b1e3f817928c42e9b69db75289a9d30a"
            ],
            "layout": "IPY_MODEL_91fd0a3f735a4d35a040f7bba80d3e24"
          }
        },
        "7d9cfb12850a48c1a9cfde1118adc2ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0ea13a9834954df78af18a82ac52593a",
            "placeholder": "​",
            "style": "IPY_MODEL_65da66d6460c4973a5cf76d626105013",
            "value": "Downloading (…)"pytorch_model.bin";: 100%"
          }
        },
        "2534269fcfad4bc78214a4a606842072": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1fc49833bbb74b358896cddf45f76efe",
            "max": 548118077,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2b5f3b65686a4794a54a96f48c03902c",
            "value": 548118077
          }
        },
        "b1e3f817928c42e9b69db75289a9d30a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f009b8b012b24afab7b485ebb31129a5",
            "placeholder": "​",
            "style": "IPY_MODEL_2efae8e22a294bc5a29abedeb136f909",
            "value": " 548M/548M [00:07<00:00, 81.8MB/s]"
          }
        },
        "91fd0a3f735a4d35a040f7bba80d3e24": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0ea13a9834954df78af18a82ac52593a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "65da66d6460c4973a5cf76d626105013": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1fc49833bbb74b358896cddf45f76efe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2b5f3b65686a4794a54a96f48c03902c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f009b8b012b24afab7b485ebb31129a5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2efae8e22a294bc5a29abedeb136f909": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "194e75a6683a4baa940e4fb1847bc7fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1b9cd839b234406181c51d207c33b712",
              "IPY_MODEL_dbcb3d10968942c3940ee041477c687e",
              "IPY_MODEL_367738533a794d33bfa343dc02df020a"
            ],
            "layout": "IPY_MODEL_598df57b89ca41758fbacd746f754b85"
          }
        },
        "1b9cd839b234406181c51d207c33b712": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0b433734ec4645558257242dbd3828f7",
            "placeholder": "​",
            "style": "IPY_MODEL_da116b9e20ec4241a8d1182740389832",
            "value": "Downloading builder script: 100%"
          }
        },
        "dbcb3d10968942c3940ee041477c687e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_52854fcfb8df4809b1adc0656dec510b",
            "max": 3208,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_57387a577b704b3cb0de7b31cab074a2",
            "value": 3208
          }
        },
        "367738533a794d33bfa343dc02df020a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a8a77976623d4659a05be0d8d30178ce",
            "placeholder": "​",
            "style": "IPY_MODEL_8df2f295498f4f96a59e776b251c3d9b",
            "value": " 3.21k/3.21k [00:00<00:00, 61.1kB/s]"
          }
        },
        "598df57b89ca41758fbacd746f754b85": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0b433734ec4645558257242dbd3828f7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "da116b9e20ec4241a8d1182740389832": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "52854fcfb8df4809b1adc0656dec510b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "57387a577b704b3cb0de7b31cab074a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a8a77976623d4659a05be0d8d30178ce": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8df2f295498f4f96a59e776b251c3d9b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "39cf5197378d4a4eb3963a4d16709519": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2cc92481dd8647879c607fd5037fb753",
              "IPY_MODEL_33fc2173fb574d228deeb33fbdc740fd",
              "IPY_MODEL_417e7fa0c4c3439581e3e3b1680f707b"
            ],
            "layout": "IPY_MODEL_872738dd74ca43978f8dc2eca4431de8"
          }
        },
        "2cc92481dd8647879c607fd5037fb753": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_920fee0f7a974eff9635906a5c86eb57",
            "placeholder": "​",
            "style": "IPY_MODEL_1590700b137049c6ad2bffb565bed672",
            "value": "Downloading metadata: 100%"
          }
        },
        "33fc2173fb574d228deeb33fbdc740fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5fe5b2ce9dd7497da4b8f1161f63136a",
            "max": 1687,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_94000bfc78984dddb528f7dc934b16fe",
            "value": 1687
          }
        },
        "417e7fa0c4c3439581e3e3b1680f707b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7cdfb805a82941c8973be3a2e3110574",
            "placeholder": "​",
            "style": "IPY_MODEL_ece2f63ea72f4ae582f1bc603d990e55",
            "value": " 1.69k/1.69k [00:00<00:00, 23.9kB/s]"
          }
        },
        "872738dd74ca43978f8dc2eca4431de8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "920fee0f7a974eff9635906a5c86eb57": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1590700b137049c6ad2bffb565bed672": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5fe5b2ce9dd7497da4b8f1161f63136a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "94000bfc78984dddb528f7dc934b16fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7cdfb805a82941c8973be3a2e3110574": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ece2f63ea72f4ae582f1bc603d990e55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "420a040cfaac4999b3ed5995c196a986": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ee3ef2c5f5784df9971022becf85740e",
              "IPY_MODEL_177662eada8947ab9bab8f466fbb8c88",
              "IPY_MODEL_c97707ed00604467acc65d8d1046bc6c"
            ],
            "layout": "IPY_MODEL_9ad63bf9f1894f1b86f6e1b637da03a3"
          }
        },
        "ee3ef2c5f5784df9971022becf85740e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_94e9ce3053fc4fd48928c7cd77710417",
            "placeholder": "​",
            "style": "IPY_MODEL_43afdbb9c8d64550b6ea2a391880f00e",
            "value": "Downloading readme: 100%"
          }
        },
        "177662eada8947ab9bab8f466fbb8c88": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3f7c35ae19c243bfb83456348920dc6c",
            "max": 4872,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9a855ba7ac2f49c5a1ed40ac9afc5ba3",
            "value": 4872
          }
        },
        "c97707ed00604467acc65d8d1046bc6c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_630210f8862a41f489002002edb8ea3e",
            "placeholder": "​",
            "style": "IPY_MODEL_b571da4200e644ecaeccb307b957674b",
            "value": " 4.87k/4.87k [00:00<00:00, 176kB/s]"
          }
        },
        "9ad63bf9f1894f1b86f6e1b637da03a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "94e9ce3053fc4fd48928c7cd77710417": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "43afdbb9c8d64550b6ea2a391880f00e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3f7c35ae19c243bfb83456348920dc6c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9a855ba7ac2f49c5a1ed40ac9afc5ba3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "630210f8862a41f489002002edb8ea3e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b571da4200e644ecaeccb307b957674b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7476faf522e443a9bcc1e7d76c113a88": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_659fc059abec4a5daaeb7aa2acae23f1",
              "IPY_MODEL_bd3f40adb27b4efda88f3a7d326154a1",
              "IPY_MODEL_aa8c057bbef54ef59b73ce5f17fc403b"
            ],
            "layout": "IPY_MODEL_d89672824faf40dab51a6b45c2fb5e18"
          }
        },
        "659fc059abec4a5daaeb7aa2acae23f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_561c9bb455d14175b6a8edf32df60cdf",
            "placeholder": "​",
            "style": "IPY_MODEL_47fc24ea0d734dc481012b005bd405d5",
            "value": "Downloading data: 100%"
          }
        },
        "bd3f40adb27b4efda88f3a7d326154a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_10ccab132c704c3098f2ab1fa2e3d0ff",
            "max": 203415,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_46b94a4ee39d43b9819d59a347d33ebb",
            "value": 203415
          }
        },
        "aa8c057bbef54ef59b73ce5f17fc403b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_32719dd06a5447e08831a65175249e80",
            "placeholder": "​",
            "style": "IPY_MODEL_56f0fb07c82f44ddb50f6973aa746ca9",
            "value": " 203k/203k [00:00<00:00, 358kB/s]"
          }
        },
        "d89672824faf40dab51a6b45c2fb5e18": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "561c9bb455d14175b6a8edf32df60cdf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "47fc24ea0d734dc481012b005bd405d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "10ccab132c704c3098f2ab1fa2e3d0ff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "46b94a4ee39d43b9819d59a347d33ebb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "32719dd06a5447e08831a65175249e80": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "56f0fb07c82f44ddb50f6973aa746ca9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f8917169c81e4ab6a5e88c0b432745db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_02671ac0aec64ac4b794d2699f488414",
              "IPY_MODEL_569a6c31b9424427bfc7ac305b3c784e",
              "IPY_MODEL_17bd589dd3b841dab6424ea4c48a5bf7"
            ],
            "layout": "IPY_MODEL_8fd18310c9114e349bdc9d01369b3a38"
          }
        },
        "02671ac0aec64ac4b794d2699f488414": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4630d7fe9a604d2c948d279de7e72d8d",
            "placeholder": "​",
            "style": "IPY_MODEL_7450fd339ff34e9598a8f8ef1ecd5cf4",
            "value": "Generating train split: 100%"
          }
        },
        "569a6c31b9424427bfc7ac305b3c784e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c4157f5519384e50894898ce7cc32496",
            "max": 5574,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e0693a18159a43daa79bee8b1b5936a7",
            "value": 5574
          }
        },
        "17bd589dd3b841dab6424ea4c48a5bf7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f3be03a5aed740ffaa1bdf3e938eb669",
            "placeholder": "​",
            "style": "IPY_MODEL_3d6860ab6f744e3fba6ad5e3b103d730",
            "value": " 5574/5574 [00:16<00:00, 12948.72 examples/s]"
          }
        },
        "8fd18310c9114e349bdc9d01369b3a38": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4630d7fe9a604d2c948d279de7e72d8d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7450fd339ff34e9598a8f8ef1ecd5cf4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c4157f5519384e50894898ce7cc32496": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e0693a18159a43daa79bee8b1b5936a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f3be03a5aed740ffaa1bdf3e938eb669": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3d6860ab6f744e3fba6ad5e3b103d730": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8d0715b6bb094b9fa14a7064e989ef87": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_bb560a238cd6461baf02ac14ffc37fb2",
              "IPY_MODEL_820efa622ea047eab4a05e70a0f0c579",
              "IPY_MODEL_37aa33596e58492388e8474ffa556a00"
            ],
            "layout": "IPY_MODEL_71f653d422dc4460a70d7559aa0464f5"
          }
        },
        "bb560a238cd6461baf02ac14ffc37fb2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_58c8f1a572a4452d8fcad0b3580cd985",
            "placeholder": "​",
            "style": "IPY_MODEL_0ac448bc3ac6477aa523780a4d792c61",
            "value": "100%"
          }
        },
        "820efa622ea047eab4a05e70a0f0c579": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ce228aa1861648b3a820e0f61a81ba65",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_eb8eeb4881d24f719153fd3a1c1d6fa2",
            "value": 1
          }
        },
        "37aa33596e58492388e8474ffa556a00": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_260018e32f0e4a709e52d4ddfeb39242",
            "placeholder": "​",
            "style": "IPY_MODEL_64bcb70b217f4ab9840e9b022b9499fb",
            "value": " 1/1 [00:00<00:00, 15.55it/s]"
          }
        },
        "71f653d422dc4460a70d7559aa0464f5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "58c8f1a572a4452d8fcad0b3580cd985": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0ac448bc3ac6477aa523780a4d792c61": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ce228aa1861648b3a820e0f61a81ba65": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eb8eeb4881d24f719153fd3a1c1d6fa2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "260018e32f0e4a709e52d4ddfeb39242": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "64bcb70b217f4ab9840e9b022b9499fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8fceb26a6d2747e899537d28e1a7f819": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1e1594b1d7ff495ebaf1055b7a5796b9",
              "IPY_MODEL_3d06186e5c35452c9f805ef332c8e963",
              "IPY_MODEL_f66e20dcbc964485929896e8e5d2929b"
            ],
            "layout": "IPY_MODEL_eb554373fa644b38a4cca04c9178a0a1"
          }
        },
        "1e1594b1d7ff495ebaf1055b7a5796b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_69a3b44309624139afa25663234dac82",
            "placeholder": "​",
            "style": "IPY_MODEL_bdbce0cabc9c4f6e8caef2ed48d6dd54",
            "value": "Downloading (…)olve/main/vocab.json: 100%"
          }
        },
        "3d06186e5c35452c9f805ef332c8e963": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_392d338529cb49a59b2afb1ebf0aff9a",
            "max": 1042301,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_90ab44c79e594a868fce9f16747f53d9",
            "value": 1042301
          }
        },
        "f66e20dcbc964485929896e8e5d2929b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_936f5b9e19354f44a4cb9c4477131405",
            "placeholder": "​",
            "style": "IPY_MODEL_1f34679a31554a6a91ae2db024361869",
            "value": " 1.04M/1.04M [00:00<00:00, 1.94MB/s]"
          }
        },
        "eb554373fa644b38a4cca04c9178a0a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "69a3b44309624139afa25663234dac82": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bdbce0cabc9c4f6e8caef2ed48d6dd54": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "392d338529cb49a59b2afb1ebf0aff9a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "90ab44c79e594a868fce9f16747f53d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "936f5b9e19354f44a4cb9c4477131405": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1f34679a31554a6a91ae2db024361869": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bd05251722ac43a0bf4dfe4958ea61a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_79ad467ff6f84d9f92a14b6ea41d7002",
              "IPY_MODEL_0ef61bcf73974c77bdd9a4114b944070",
              "IPY_MODEL_85aba1e0f89148e5abaa4247d9a62d15"
            ],
            "layout": "IPY_MODEL_713cd8fdb42c48d48142032a6948a3b7"
          }
        },
        "79ad467ff6f84d9f92a14b6ea41d7002": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_66ca513a92e04e9aafd22286fef64427",
            "placeholder": "​",
            "style": "IPY_MODEL_7debd2aa4b5140dbaff4528961b4714d",
            "value": "Downloading (…)olve/main/merges.txt: 100%"
          }
        },
        "0ef61bcf73974c77bdd9a4114b944070": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5032b90243bd4b67bf12ee696112c4d5",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_bdf13c209a2043388a5b1c648ccf4179",
            "value": 456318
          }
        },
        "85aba1e0f89148e5abaa4247d9a62d15": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e841fe2854bf4f6b9b7021ba91f5b002",
            "placeholder": "​",
            "style": "IPY_MODEL_d1d8838cc62e4684b2e0fdeb2e932829",
            "value": " 456k/456k [00:00<00:00, 1.05MB/s]"
          }
        },
        "713cd8fdb42c48d48142032a6948a3b7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "66ca513a92e04e9aafd22286fef64427": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7debd2aa4b5140dbaff4528961b4714d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5032b90243bd4b67bf12ee696112c4d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bdf13c209a2043388a5b1c648ccf4179": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e841fe2854bf4f6b9b7021ba91f5b002": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d1d8838cc62e4684b2e0fdeb2e932829": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b1b1058975a64c08bfe57e87b9392a5f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6f7eab6077ae4cdbbf66e90818c2e99d",
              "IPY_MODEL_e98d7df786314429a697f99160a8e724",
              "IPY_MODEL_c1ea782c1e484cee86d88b65acdf6646"
            ],
            "layout": "IPY_MODEL_d74bdb969f2d419197ff72fbf528f315"
          }
        },
        "6f7eab6077ae4cdbbf66e90818c2e99d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_81bf5d95f33a422591a2a7a693753909",
            "placeholder": "​",
            "style": "IPY_MODEL_eb50a4ad2ff2453dbc2010cd7d449464",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "e98d7df786314429a697f99160a8e724": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_42ceba12957b43108d420f9c04cd6bd1",
            "max": 665,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_83b8eb341a9d40b59a6a562221ef4241",
            "value": 665
          }
        },
        "c1ea782c1e484cee86d88b65acdf6646": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_21534596456144a1ab256f86eaaef343",
            "placeholder": "​",
            "style": "IPY_MODEL_efc6356942f2489d97f6704c7b0d1c94",
            "value": " 665/665 [00:00<00:00, 22.7kB/s]"
          }
        },
        "d74bdb969f2d419197ff72fbf528f315": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "81bf5d95f33a422591a2a7a693753909": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eb50a4ad2ff2453dbc2010cd7d449464": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "42ceba12957b43108d420f9c04cd6bd1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "83b8eb341a9d40b59a6a562221ef4241": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "21534596456144a1ab256f86eaaef343": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "efc6356942f2489d97f6704c7b0d1c94": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Instalacja pakietów"
      ],
      "metadata": {
        "id": "t2xXKpOpcZg_"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fdpN7ugfauLD",
        "outputId": "e82df883-5879-4df0-af6e-45df01222c9f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.26.1-py3-none-any.whl (6.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.3/6.3 MB\u001b[0m \u001b[31m51.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting datasets\n",
            "  Downloading datasets-2.9.0-py3-none-any.whl (462 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m462.8/462.8 KB\u001b[0m \u001b[31m24.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.8/dist-packages (1.13.1+cu116)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from transformers) (3.9.0)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m30.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from transformers) (23.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from transformers) (2.25.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.8/dist-packages (from transformers) (4.64.1)\n",
            "Collecting huggingface-hub<1.0,>=0.11.0\n",
            "  Downloading huggingface_hub-0.12.1-py3-none-any.whl (190 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m190.3/190.3 KB\u001b[0m \u001b[31m18.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multiprocess\n",
            "  Downloading multiprocess-0.70.14-py38-none-any.whl (132 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.0/132.0 KB\u001b[0m \u001b[31m10.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting xxhash\n",
            "  Downloading xxhash-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (213 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m213.0/213.0 KB\u001b[0m \u001b[31m12.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.8/dist-packages (from datasets) (2023.1.0)\n",
            "Requirement already satisfied: dill<0.3.7 in /usr/local/lib/python3.8/dist-packages (from datasets) (0.3.6)\n",
            "Collecting responses<0.19\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.8/dist-packages (from datasets) (9.0.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.8/dist-packages (from datasets) (3.8.4)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.8/dist-packages (from datasets) (1.3.5)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.8/dist-packages (from torch) (4.5.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (6.0.4)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (3.0.1)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.3.3)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.8.2)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (22.2.0)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (4.0.2)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (4.0.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2.10)\n",
            "Collecting urllib3<1.27,>=1.21.1\n",
            "  Downloading urllib3-1.26.14-py2.py3-none-any.whl (140 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m140.6/140.6 KB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets) (2022.7.1)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.8/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: tokenizers, xxhash, urllib3, multiprocess, responses, huggingface-hub, transformers, datasets\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 1.24.3\n",
            "    Uninstalling urllib3-1.24.3:\n",
            "      Successfully uninstalled urllib3-1.24.3\n",
            "Successfully installed datasets-2.9.0 huggingface-hub-0.12.1 multiprocess-0.70.14 responses-0.18.0 tokenizers-0.13.2 transformers-4.26.1 urllib3-1.26.14 xxhash-3.2.0\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers datasets torch"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Załadowanie pakietów"
      ],
      "metadata": {
        "id": "s8cfdy_6ldCn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_dataset\n",
        "from transformers import GPT2Tokenizer\n",
        "import torch\n",
        "from torch.utils.data import TensorDataset, random_split\n",
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "from transformers import GPT2ForSequenceClassification, GPT2Config\n",
        "from transformers import get_linear_schedule_with_warmup\n",
        "import numpy as np\n",
        "import time\n",
        "import datetime\n",
        "import random"
      ],
      "metadata": {
        "id": "yLS_x9DIlgSs"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Załadowanie datasetu\n",
        "sms_spam"
      ],
      "metadata": {
        "id": "fPwDyJd5cdaE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = load_dataset(\"sms_spam\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 263,
          "referenced_widgets": [
            "194e75a6683a4baa940e4fb1847bc7fc",
            "1b9cd839b234406181c51d207c33b712",
            "dbcb3d10968942c3940ee041477c687e",
            "367738533a794d33bfa343dc02df020a",
            "598df57b89ca41758fbacd746f754b85",
            "0b433734ec4645558257242dbd3828f7",
            "da116b9e20ec4241a8d1182740389832",
            "52854fcfb8df4809b1adc0656dec510b",
            "57387a577b704b3cb0de7b31cab074a2",
            "a8a77976623d4659a05be0d8d30178ce",
            "8df2f295498f4f96a59e776b251c3d9b",
            "39cf5197378d4a4eb3963a4d16709519",
            "2cc92481dd8647879c607fd5037fb753",
            "33fc2173fb574d228deeb33fbdc740fd",
            "417e7fa0c4c3439581e3e3b1680f707b",
            "872738dd74ca43978f8dc2eca4431de8",
            "920fee0f7a974eff9635906a5c86eb57",
            "1590700b137049c6ad2bffb565bed672",
            "5fe5b2ce9dd7497da4b8f1161f63136a",
            "94000bfc78984dddb528f7dc934b16fe",
            "7cdfb805a82941c8973be3a2e3110574",
            "ece2f63ea72f4ae582f1bc603d990e55",
            "420a040cfaac4999b3ed5995c196a986",
            "ee3ef2c5f5784df9971022becf85740e",
            "177662eada8947ab9bab8f466fbb8c88",
            "c97707ed00604467acc65d8d1046bc6c",
            "9ad63bf9f1894f1b86f6e1b637da03a3",
            "94e9ce3053fc4fd48928c7cd77710417",
            "43afdbb9c8d64550b6ea2a391880f00e",
            "3f7c35ae19c243bfb83456348920dc6c",
            "9a855ba7ac2f49c5a1ed40ac9afc5ba3",
            "630210f8862a41f489002002edb8ea3e",
            "b571da4200e644ecaeccb307b957674b",
            "7476faf522e443a9bcc1e7d76c113a88",
            "659fc059abec4a5daaeb7aa2acae23f1",
            "bd3f40adb27b4efda88f3a7d326154a1",
            "aa8c057bbef54ef59b73ce5f17fc403b",
            "d89672824faf40dab51a6b45c2fb5e18",
            "561c9bb455d14175b6a8edf32df60cdf",
            "47fc24ea0d734dc481012b005bd405d5",
            "10ccab132c704c3098f2ab1fa2e3d0ff",
            "46b94a4ee39d43b9819d59a347d33ebb",
            "32719dd06a5447e08831a65175249e80",
            "56f0fb07c82f44ddb50f6973aa746ca9",
            "f8917169c81e4ab6a5e88c0b432745db",
            "02671ac0aec64ac4b794d2699f488414",
            "569a6c31b9424427bfc7ac305b3c784e",
            "17bd589dd3b841dab6424ea4c48a5bf7",
            "8fd18310c9114e349bdc9d01369b3a38",
            "4630d7fe9a604d2c948d279de7e72d8d",
            "7450fd339ff34e9598a8f8ef1ecd5cf4",
            "c4157f5519384e50894898ce7cc32496",
            "e0693a18159a43daa79bee8b1b5936a7",
            "f3be03a5aed740ffaa1bdf3e938eb669",
            "3d6860ab6f744e3fba6ad5e3b103d730",
            "8d0715b6bb094b9fa14a7064e989ef87",
            "bb560a238cd6461baf02ac14ffc37fb2",
            "820efa622ea047eab4a05e70a0f0c579",
            "37aa33596e58492388e8474ffa556a00",
            "71f653d422dc4460a70d7559aa0464f5",
            "58c8f1a572a4452d8fcad0b3580cd985",
            "0ac448bc3ac6477aa523780a4d792c61",
            "ce228aa1861648b3a820e0f61a81ba65",
            "eb8eeb4881d24f719153fd3a1c1d6fa2",
            "260018e32f0e4a709e52d4ddfeb39242",
            "64bcb70b217f4ab9840e9b022b9499fb"
          ]
        },
        "id": "N1EWeM0KcYtO",
        "outputId": "f7a6ad94-6053-4e22-e197-844ba73b401e"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/3.21k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "194e75a6683a4baa940e4fb1847bc7fc"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading metadata:   0%|          | 0.00/1.69k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "39cf5197378d4a4eb3963a4d16709519"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading readme:   0%|          | 0.00/4.87k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "420a040cfaac4999b3ed5995c196a986"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading and preparing dataset sms_spam/plain_text to /root/.cache/huggingface/datasets/sms_spam/plain_text/1.0.0/53f051d3b5f62d99d61792c91acefe4f1577ad3e4c216fb0ad39e30b9f20019c...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/203k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7476faf522e443a9bcc1e7d76c113a88"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split:   0%|          | 0/5574 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f8917169c81e4ab6a5e88c0b432745db"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset sms_spam downloaded and prepared to /root/.cache/huggingface/datasets/sms_spam/plain_text/1.0.0/53f051d3b5f62d99d61792c91acefe4f1577ad3e4c216fb0ad39e30b9f20019c. Subsequent calls will reuse this data.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8d0715b6bb094b9fa14a7064e989ef87"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset['train'][0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Mf1QIM_dlp2x",
        "outputId": "3e9307f6-cbf5-498c-c1e5-48fd0c15957f"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'sms': 'Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...\\n',\n",
              " 'label': 0}"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Tokenizer GPT2"
      ],
      "metadata": {
        "id": "Qc7CIjSOchir"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
        "tokenizer.pad_token = tokenizer.eos_token\n"
      ],
      "metadata": {
        "id": "hmnlC_hubLmP",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "8fceb26a6d2747e899537d28e1a7f819",
            "1e1594b1d7ff495ebaf1055b7a5796b9",
            "3d06186e5c35452c9f805ef332c8e963",
            "f66e20dcbc964485929896e8e5d2929b",
            "eb554373fa644b38a4cca04c9178a0a1",
            "69a3b44309624139afa25663234dac82",
            "bdbce0cabc9c4f6e8caef2ed48d6dd54",
            "392d338529cb49a59b2afb1ebf0aff9a",
            "90ab44c79e594a868fce9f16747f53d9",
            "936f5b9e19354f44a4cb9c4477131405",
            "1f34679a31554a6a91ae2db024361869",
            "bd05251722ac43a0bf4dfe4958ea61a4",
            "79ad467ff6f84d9f92a14b6ea41d7002",
            "0ef61bcf73974c77bdd9a4114b944070",
            "85aba1e0f89148e5abaa4247d9a62d15",
            "713cd8fdb42c48d48142032a6948a3b7",
            "66ca513a92e04e9aafd22286fef64427",
            "7debd2aa4b5140dbaff4528961b4714d",
            "5032b90243bd4b67bf12ee696112c4d5",
            "bdf13c209a2043388a5b1c648ccf4179",
            "e841fe2854bf4f6b9b7021ba91f5b002",
            "d1d8838cc62e4684b2e0fdeb2e932829",
            "b1b1058975a64c08bfe57e87b9392a5f",
            "6f7eab6077ae4cdbbf66e90818c2e99d",
            "e98d7df786314429a697f99160a8e724",
            "c1ea782c1e484cee86d88b65acdf6646",
            "d74bdb969f2d419197ff72fbf528f315",
            "81bf5d95f33a422591a2a7a693753909",
            "eb50a4ad2ff2453dbc2010cd7d449464",
            "42ceba12957b43108d420f9c04cd6bd1",
            "83b8eb341a9d40b59a6a562221ef4241",
            "21534596456144a1ab256f86eaaef343",
            "efc6356942f2489d97f6704c7b0d1c94"
          ]
        },
        "outputId": "67151b75-dd52-405c-fdab-9cd67629e23f"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)olve/main/vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8fceb26a6d2747e899537d28e1a7f819"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)olve/main/merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bd05251722ac43a0bf4dfe4958ea61a4"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/665 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b1b1058975a64c08bfe57e87b9392a5f"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sms = dataset['train'][0]['sms']\n",
        "print('Original: ', sms)\n",
        "print('Tokenized: ', tokenizer.tokenize(sms))\n",
        "print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sms)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZxigrpcQdWCF",
        "outputId": "bc440b70-46ed-4ae0-9e6a-6d69a707778e"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original:  Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...\n",
            "\n",
            "Tokenized:  ['Go', 'Ġuntil', 'Ġjur', 'ong', 'Ġpoint', ',', 'Ġcrazy', '..', 'ĠAvailable', 'Ġonly', 'Ġin', 'Ġbug', 'is', 'Ġn', 'Ġgreat', 'Ġworld', 'Ġla', 'Ġe', 'Ġbuffet', '...', 'ĠC', 'ine', 'Ġthere', 'Ġgot', 'Ġam', 'ore', 'Ġwat', '...', 'Ċ']\n",
            "Token IDs:  [5247, 1566, 8174, 506, 966, 11, 7165, 492, 14898, 691, 287, 5434, 271, 299, 1049, 995, 8591, 304, 44703, 986, 327, 500, 612, 1392, 716, 382, 4383, 986, 198]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Check maximum length of a sentence"
      ],
      "metadata": {
        "id": "wVT0m8T7evoz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "max_len = 0\n",
        "\n",
        "for sentence in dataset['train']:\n",
        "    input_ids = tokenizer.encode(sentence['sms'], add_special_tokens=True)\n",
        "    max_len = max(max_len, len(input_ids))\n",
        "\n",
        "print('Max sentence length: ', max_len)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "cmUVPrQYez3J",
        "outputId": "c0829832-7a6d-4019-8982-c63d0022b42d"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Max sentence length:  258\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Special tokenization"
      ],
      "metadata": {
        "id": "2NfXDfYifX5S"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "input_ids = []\n",
        "attention_masks = []\n",
        "\n",
        "for sentence in dataset['train']:\n",
        "    encoded_dict = tokenizer.encode_plus(\n",
        "                        sentence['sms'],\n",
        "                        add_special_tokens = True,\n",
        "                        max_length = 260,\n",
        "                        padding = 'max_length',\n",
        "                        truncation=True,\n",
        "                        return_attention_mask = True,\n",
        "                        return_tensors = 'pt',\n",
        "                   )\n",
        "    \n",
        "    input_ids.append(encoded_dict['input_ids'])\n",
        "    attention_masks.append(encoded_dict['attention_mask'])\n",
        "\n",
        "input_ids = torch.cat(input_ids, dim=0)\n",
        "attention_masks = torch.cat(attention_masks, dim=0)\n",
        "labels = torch.tensor([sentence['label'] for sentence in dataset['train']])\n",
        "\n",
        "print('Original: ', dataset['train'][0])\n",
        "print('Token IDs:', input_ids[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4u03dIS1fbKU",
        "outputId": "fe0e0eab-fda3-4a72-a8b5-e2a48212fb78"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original:  {'sms': 'Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...\\n', 'label': 0}\n",
            "Token IDs: tensor([ 5247,  1566,  8174,   506,   966,    11,  7165,   492, 14898,   691,\n",
            "          287,  5434,   271,   299,  1049,   995,  8591,   304, 44703,   986,\n",
            "          327,   500,   612,  1392,   716,   382,  4383,   986,   198, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256,\n",
            "        50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Split dataset\n",
        "Class balance ratio should be similar to base dataset ratio."
      ],
      "metadata": {
        "id": "Z6cC0YjAhmw_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def check_class_balance(dataset):\n",
        "  spam_count = 0.0\n",
        "  not_spam_count = 0.0\n",
        "  for row in dataset:\n",
        "    if row[2].item() == 1:\n",
        "      spam_count += 1.0\n",
        "    else:\n",
        "      not_spam_count += 1.0\n",
        "  return spam_count / not_spam_count "
      ],
      "metadata": {
        "id": "83DeMQwJvuMA"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = TensorDataset(input_ids, attention_masks, labels)\n",
        "print(\"Spam to not spam messages ratio: {}\\n\".format(check_class_balance(dataset)))\n",
        "\n",
        "test_size = 1000\n",
        "dataset_len = len(dataset)\n",
        "train_size = int(0.9 * (dataset_len-test_size))\n",
        "val_size = (dataset_len-test_size) - train_size\n",
        "\n",
        "test_dataset, train_dataset, val_dataset = random_split(dataset, [test_size, train_size, val_size])\n",
        "\n",
        "print('{:>5,} test samples'.format(test_size))\n",
        "print(\"Ratio: {}\\n\".format(check_class_balance(test_dataset)))\n",
        "print('{:>5,} training samples'.format(train_size))\n",
        "print(\"Ratio: {}\\n\".format(check_class_balance(train_dataset)))\n",
        "print('{:>5,} validation samples'.format(val_size))\n",
        "print(\"Ratio: {}\\n\".format(check_class_balance(val_dataset)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vH3yXhA0hT3n",
        "outputId": "8905beef-086b-459b-aa91-99bf4c448017"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Spam to not spam messages ratio: 0.15475450590428838\n",
            "\n",
            "1,000 test samples\n",
            "Ratio: 0.15606936416184972\n",
            "\n",
            "4,116 training samples\n",
            "Ratio: 0.15294117647058825\n",
            "\n",
            "  458 validation samples\n",
            "Ratio: 0.1683673469387755\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Create train and validation loaders"
      ],
      "metadata": {
        "id": "z1hVsejihpO2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "batch_size = 8\n",
        "\n",
        "train_dataloader = DataLoader(\n",
        "            train_dataset,\n",
        "            sampler = RandomSampler(train_dataset),\n",
        "            batch_size = batch_size\n",
        "        )\n",
        "\n",
        "validation_dataloader = DataLoader(\n",
        "            val_dataset,\n",
        "            sampler = SequentialSampler(val_dataset),\n",
        "            batch_size = batch_size\n",
        "        )"
      ],
      "metadata": {
        "id": "k4pXght6hre3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Device check"
      ],
      "metadata": {
        "id": "MnErwHAbl_rF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "if torch.cuda.is_available():    \n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aUvyBFxzmBUy",
        "outputId": "04bc746e-0d7a-443f-dfd2-df757b49cc04"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla T4\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Load GPT2 model"
      ],
      "metadata": {
        "id": "o-YrojT-iIfY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = GPT2ForSequenceClassification.from_pretrained(\n",
        "    'gpt2',\n",
        "    num_labels = 2,\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84,
          "referenced_widgets": [
            "54a468d755ee4d398517315461829a70",
            "7d9cfb12850a48c1a9cfde1118adc2ea",
            "2534269fcfad4bc78214a4a606842072",
            "b1e3f817928c42e9b69db75289a9d30a",
            "91fd0a3f735a4d35a040f7bba80d3e24",
            "0ea13a9834954df78af18a82ac52593a",
            "65da66d6460c4973a5cf76d626105013",
            "1fc49833bbb74b358896cddf45f76efe",
            "2b5f3b65686a4794a54a96f48c03902c",
            "f009b8b012b24afab7b485ebb31129a5",
            "2efae8e22a294bc5a29abedeb136f909"
          ]
        },
        "id": "sIP3VGZmiK9s",
        "outputId": "ebac6f6b-b0c4-49a3-8da7-2c98fe1bbbf1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)\"pytorch_model.bin\";:   0%|          | 0.00/548M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "54a468d755ee4d398517315461829a70"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of GPT2ForSequenceClassification were not initialized from the model checkpoint at gpt2 and are newly initialized: ['score.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.resize_token_embeddings(len(tokenizer))\n",
        "model.config.pad_token_id = model.config.eos_token_id\n",
        "\n",
        "model.cuda()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LhpRR5YEeU1S",
        "outputId": "c1e1d515-12fe-41bf-94b5-c61ad83afbc2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GPT2ForSequenceClassification(\n",
              "  (transformer): GPT2Model(\n",
              "    (wte): Embedding(50257, 768)\n",
              "    (wpe): Embedding(1024, 768)\n",
              "    (drop): Dropout(p=0.1, inplace=False)\n",
              "    (h): ModuleList(\n",
              "      (0): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (act): NewGELUActivation()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (1): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (act): NewGELUActivation()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (2): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (act): NewGELUActivation()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (3): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (act): NewGELUActivation()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (4): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (act): NewGELUActivation()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (5): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (act): NewGELUActivation()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (6): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (act): NewGELUActivation()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (7): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (act): NewGELUActivation()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (8): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (act): NewGELUActivation()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (9): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (act): NewGELUActivation()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (10): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (act): NewGELUActivation()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (11): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (act): NewGELUActivation()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "  )\n",
              "  (score): Linear(in_features=768, out_features=2, bias=False)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Init training parameters"
      ],
      "metadata": {
        "id": "NZDC4iiQizdX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "optimizer = torch.optim.AdamW(model.parameters(),\n",
        "                  lr = 2e-5,\n",
        "                  eps = 1e-8\n",
        "                )\n",
        "\n",
        "epochs = 4\n",
        "total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
        "                                            num_warmup_steps = 0,\n",
        "                                            num_training_steps = total_steps)"
      ],
      "metadata": {
        "id": "_uffUPNEi3S5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Helper functions"
      ],
      "metadata": {
        "id": "bnAwgfZekeYD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def flat_accuracy(preds, labels):\n",
        "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
        "\n",
        "def format_time(elapsed):\n",
        "    '''\n",
        "    Takes a time in seconds and returns a string hh:mm:ss\n",
        "    '''\n",
        "    elapsed_rounded = int(round((elapsed)))\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))"
      ],
      "metadata": {
        "id": "Z3XSZuFmkgVr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Training"
      ],
      "metadata": {
        "id": "L-ZeLPfbkqy9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# This training code is based on the `run_glue.py` script here:\n",
        "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
        "\n",
        "seed_val = 42\n",
        "\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "training_stats = []\n",
        "total_t0 = time.time()\n",
        "\n",
        "for epoch_i in range(0, epochs):\n",
        "    \n",
        "    # ========================================\n",
        "    #               Training\n",
        "    # ========================================\n",
        "\n",
        "    print(\"\")\n",
        "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "    print('Training...')\n",
        "\n",
        "    t0 = time.time()\n",
        "    total_train_loss = 0\n",
        "\n",
        "    model.train()\n",
        "\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "        if step % 40 == 0 and not step == 0:\n",
        "            elapsed = format_time(time.time() - t0)\n",
        "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
        "\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "\n",
        "        model.zero_grad()        \n",
        "\n",
        "        outputs = model(b_input_ids, \n",
        "                             token_type_ids=None, \n",
        "                             attention_mask=b_input_mask, \n",
        "                             labels=b_labels)\n",
        "\n",
        "        loss = outputs['loss']\n",
        "        total_train_loss += loss.item()\n",
        "\n",
        "        loss.backward()\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "        optimizer.step()\n",
        "        scheduler.step()\n",
        "\n",
        "    avg_train_loss = total_train_loss / len(train_dataloader)            \n",
        "    training_time = format_time(time.time() - t0)\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "    print(\"  Training epcoh took: {:}\".format(training_time))\n",
        "        \n",
        "    # ========================================\n",
        "    #               Validation\n",
        "    # ========================================\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"Running Validation...\")\n",
        "\n",
        "    t0 = time.time()\n",
        "    model.eval()\n",
        "\n",
        "    total_eval_accuracy = 0\n",
        "    total_eval_loss = 0\n",
        "    nb_eval_steps = 0\n",
        "\n",
        "    for batch in validation_dataloader:\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "        \n",
        "        with torch.no_grad():        \n",
        "            outputs = model(b_input_ids, \n",
        "                                   token_type_ids=None, \n",
        "                                   attention_mask=b_input_mask,\n",
        "                                   labels=b_labels)\n",
        "            loss = outputs['loss']\n",
        "            logits = outputs['logits']\n",
        "            \n",
        "        total_eval_loss += loss.item()\n",
        "\n",
        "        logits = logits.detach().cpu().numpy()\n",
        "        label_ids = b_labels.to('cpu').numpy()\n",
        "\n",
        "        total_eval_accuracy += flat_accuracy(logits, label_ids)\n",
        "        \n",
        "    avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)\n",
        "    print(\"  Accuracy: {0:.2f}\".format(avg_val_accuracy))\n",
        "\n",
        "    avg_val_loss = total_eval_loss / len(validation_dataloader)\n",
        "    validation_time = format_time(time.time() - t0)\n",
        "    \n",
        "    print(\"  Validation Loss: {0:.2f}\".format(avg_val_loss))\n",
        "    print(\"  Validation took: {:}\".format(validation_time))\n",
        "\n",
        "    training_stats.append(\n",
        "        {\n",
        "            'epoch': epoch_i + 1,\n",
        "            'Training Loss': avg_train_loss,\n",
        "            'Valid. Loss': avg_val_loss,\n",
        "            'Valid. Accur.': avg_val_accuracy,\n",
        "            'Training Time': training_time,\n",
        "            'Validation Time': validation_time\n",
        "        }\n",
        "    )\n",
        "\n",
        "print(\"\")\n",
        "print(\"Training complete!\")\n",
        "\n",
        "print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hnq-2iztdYie",
        "outputId": "f2cf6703-9ab8-4dbf-e5d4-22f899a28776"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "======== Epoch 1 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    515.    Elapsed: 0:00:19.\n",
            "  Batch    80  of    515.    Elapsed: 0:00:35.\n",
            "  Batch   120  of    515.    Elapsed: 0:00:52.\n",
            "  Batch   160  of    515.    Elapsed: 0:01:08.\n",
            "  Batch   200  of    515.    Elapsed: 0:01:25.\n",
            "  Batch   240  of    515.    Elapsed: 0:01:42.\n",
            "  Batch   280  of    515.    Elapsed: 0:01:58.\n",
            "  Batch   320  of    515.    Elapsed: 0:02:15.\n",
            "  Batch   360  of    515.    Elapsed: 0:02:32.\n",
            "  Batch   400  of    515.    Elapsed: 0:02:49.\n",
            "  Batch   440  of    515.    Elapsed: 0:03:06.\n",
            "  Batch   480  of    515.    Elapsed: 0:03:24.\n",
            "\n",
            "  Average training loss: 0.14\n",
            "  Training epcoh took: 0:03:38\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.97\n",
            "  Validation Loss: 0.21\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 2 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    515.    Elapsed: 0:00:17.\n",
            "  Batch    80  of    515.    Elapsed: 0:00:35.\n",
            "  Batch   120  of    515.    Elapsed: 0:00:52.\n",
            "  Batch   160  of    515.    Elapsed: 0:01:09.\n",
            "  Batch   200  of    515.    Elapsed: 0:01:27.\n",
            "  Batch   240  of    515.    Elapsed: 0:01:44.\n",
            "  Batch   280  of    515.    Elapsed: 0:02:01.\n",
            "  Batch   320  of    515.    Elapsed: 0:02:19.\n",
            "  Batch   360  of    515.    Elapsed: 0:02:36.\n",
            "  Batch   400  of    515.    Elapsed: 0:02:54.\n",
            "  Batch   440  of    515.    Elapsed: 0:03:11.\n",
            "  Batch   480  of    515.    Elapsed: 0:03:29.\n",
            "\n",
            "  Average training loss: 0.04\n",
            "  Training epcoh took: 0:03:44\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.97\n",
            "  Validation Loss: 0.19\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 3 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    515.    Elapsed: 0:00:17.\n",
            "  Batch    80  of    515.    Elapsed: 0:00:35.\n",
            "  Batch   120  of    515.    Elapsed: 0:00:52.\n",
            "  Batch   160  of    515.    Elapsed: 0:01:10.\n",
            "  Batch   200  of    515.    Elapsed: 0:01:27.\n",
            "  Batch   240  of    515.    Elapsed: 0:01:45.\n",
            "  Batch   280  of    515.    Elapsed: 0:02:02.\n",
            "  Batch   320  of    515.    Elapsed: 0:02:20.\n",
            "  Batch   360  of    515.    Elapsed: 0:02:37.\n",
            "  Batch   400  of    515.    Elapsed: 0:02:55.\n",
            "  Batch   440  of    515.    Elapsed: 0:03:12.\n",
            "  Batch   480  of    515.    Elapsed: 0:03:30.\n",
            "\n",
            "  Average training loss: 0.03\n",
            "  Training epcoh took: 0:03:45\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.97\n",
            "  Validation Loss: 0.16\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "======== Epoch 4 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    515.    Elapsed: 0:00:17.\n",
            "  Batch    80  of    515.    Elapsed: 0:00:35.\n",
            "  Batch   120  of    515.    Elapsed: 0:00:52.\n",
            "  Batch   160  of    515.    Elapsed: 0:01:10.\n",
            "  Batch   200  of    515.    Elapsed: 0:01:27.\n",
            "  Batch   240  of    515.    Elapsed: 0:01:45.\n",
            "  Batch   280  of    515.    Elapsed: 0:02:02.\n",
            "  Batch   320  of    515.    Elapsed: 0:02:20.\n",
            "  Batch   360  of    515.    Elapsed: 0:02:37.\n",
            "  Batch   400  of    515.    Elapsed: 0:02:55.\n",
            "  Batch   440  of    515.    Elapsed: 0:03:12.\n",
            "  Batch   480  of    515.    Elapsed: 0:03:30.\n",
            "\n",
            "  Average training loss: 0.01\n",
            "  Training epcoh took: 0:03:45\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.98\n",
            "  Validation Loss: 0.11\n",
            "  Validation took: 0:00:08\n",
            "\n",
            "Training complete!\n",
            "Total training took 0:15:24 (h:mm:ss)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Train summary"
      ],
      "metadata": {
        "id": "z3nngo5DgZe4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "pd.set_option('precision', 2)\n",
        "df_stats = pd.DataFrame(data=training_stats)\n",
        "\n",
        "df_stats = df_stats.set_index('epoch')\n",
        "df_stats"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "qVSGSZ5-gbnV",
        "outputId": "b6e5d689-6748-4e0d-a43d-0484de05129d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       Training Loss  Valid. Loss  Valid. Accur. Training Time Validation Time\n",
              "epoch                                                                         \n",
              "1               0.14         0.21           0.97       0:03:38         0:00:08\n",
              "2               0.04         0.19           0.97       0:03:44         0:00:08\n",
              "3               0.03         0.16           0.97       0:03:45         0:00:08\n",
              "4               0.01         0.11           0.98       0:03:45         0:00:08"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6c6be55d-42e7-4ea4-9c6a-7e650f43571b\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Valid. Loss</th>\n",
              "      <th>Valid. Accur.</th>\n",
              "      <th>Training Time</th>\n",
              "      <th>Validation Time</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>epoch</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.14</td>\n",
              "      <td>0.21</td>\n",
              "      <td>0.97</td>\n",
              "      <td>0:03:38</td>\n",
              "      <td>0:00:08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.04</td>\n",
              "      <td>0.19</td>\n",
              "      <td>0.97</td>\n",
              "      <td>0:03:44</td>\n",
              "      <td>0:00:08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.03</td>\n",
              "      <td>0.16</td>\n",
              "      <td>0.97</td>\n",
              "      <td>0:03:45</td>\n",
              "      <td>0:00:08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.01</td>\n",
              "      <td>0.11</td>\n",
              "      <td>0.98</td>\n",
              "      <td>0:03:45</td>\n",
              "      <td>0:00:08</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6c6be55d-42e7-4ea4-9c6a-7e650f43571b')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6c6be55d-42e7-4ea4-9c6a-7e650f43571b button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6c6be55d-42e7-4ea4-9c6a-7e650f43571b');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "import seaborn as sns\n",
        "\n",
        "sns.set(style='darkgrid')\n",
        "\n",
        "sns.set(font_scale=1.5)\n",
        "plt.rcParams[\"figure.figsize\"] = (12,6)\n",
        "\n",
        "plt.plot(df_stats['Training Loss'], 'b-o', label=\"Training\")\n",
        "plt.plot(df_stats['Valid. Loss'], 'g-o', label=\"Validation\")\n",
        "\n",
        "plt.title(\"Training & Validation Loss\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.legend()\n",
        "plt.xticks([1, 2, 3, 4])\n",
        "\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 427
        },
        "id": "qhDALEePhHp1",
        "outputId": "ddcab2bb-da52-4647-8d04-167993f6c98f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvoAAAGaCAYAAAB+A+cSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeVzUdeI/8NfcM8BwDzd4oIAHoiIqaoc3KqUWZuVKaVq2Wf1s29Ktdqtdq6+5mWa5m1m2ppkHeF+lZnkkoiYegImlco8g9zUwn98fwMjHAQUEZ4DX8/HoQbw/13sGPvL6vOd9SARBEEBERERERO2K1NIVICIiIiKilsegT0RERETUDjHoExERERG1Qwz6RERERETtEIM+EREREVE7xKBPRERERNQOMegTETVBamoqAgMD8cknnzT7HPPnz0dgYGAL1qr9auj9DgwMxPz58xt1jk8++QSBgYFITU1t8frFxMQgMDAQx48fb/FzExHdLbmlK0BEdDeaEpj3798PHx+fVqxN21NSUoL//Oc/2LVrF7Kzs+Hs7IzQ0FD8+c9/hr+/f6PO8dJLL2Hv3r3YsmULevToUe8+giBg5MiRKCgowOHDh6FWq1vyZbSq48ePIy4uDk899RTs7e0tXR0zqampGDlyJKZNm4a///3vlq4OEVkRBn0iatMWLVok+v7kyZP47rvvMHXqVISGhoq2OTs73/X1vL29kZCQAJlM1uxz/POf/8Q777xz13VpCW+++SZ27tyJyMhIDBw4EHq9HgcOHMCZM2caHfSjoqKwd+9ebN68GW+++Wa9+/zyyy9IS0vD1KlTWyTkJyQkQCq9Nx9Kx8XFYfny5Zg8ebJZ0J84cSImTJgAhUJxT+pCRNQUDPpE1KZNnDhR9H1VVRW+++479O3b12zbrYqKimBnZ9ek60kkEqhUqibXsy5rCYWlpaXYs2cPhg0bhn//+9+m8rlz56KioqLR5xk2bBg8PT2xfft2vPbaa1AqlWb7xMTEAKh+KGgJd/szaCkymeyuHvqIiFoT++gTUYcwYsQITJ8+HRcuXMAzzzyD0NBQPPzwwwCqA/+SJUswZcoUDBo0CL1798bo0aOxePFilJaWis5TX5/xumUHDx7Eo48+iuDgYAwbNgz/93//h8rKStE56uujX1tWWFiIf/zjHwgPD0dwcDAef/xxnDlzxuz13LhxAwsWLMCgQYPQr18/REdH48KFC5g+fTpGjBjRqPdEIpFAIpHU++BRX1hviFQqxeTJk5GXl4cDBw6YbS8qKsK+ffsQEBCAPn36NOn9bkh9ffSNRiP++9//YsSIEQgODkZkZCS2bdtW7/EpKSl4++23MWHCBPTr1w8hISF45JFHsHHjRtF+8+fPx/LlywEAI0eORGBgoOjn31Af/dzcXLzzzjt44IEH0Lt3bzzwwAN45513cOPGDdF+tccfO3YMq1atwqhRo9C7d2+MHTsWsbGxjXovmiIpKQkvvPACBg0ahODgYIwfPx4rV65EVVWVaL+MjAwsWLAAw4cPR+/evREeHo7HH39cVCej0YjVq1fjoYceQr9+/dC/f3+MHTsWf/vb32AwGFq87kTUdGzRJ6IOIz09HU899RQiIiIwZswYlJSUAACysrKwadMmjBkzBpGRkZDL5YiLi8MXX3yBxMRErFq1qlHnP3ToENatW4fHH38cjz76KPbv348vv/wSDg4OmDNnTqPO8cwzz8DZ2RkvvPAC8vLy8NVXX+HZZ5/F/v37TZ8+VFRUYMaMGUhMTMQjjzyC4OBgJCcnY8aMGXBwcGj0+6FWqzFp0iRs3rwZO3bsQGRkZKOPvdUjjzyCFStWICYmBhEREaJtO3fuRFlZGR599FEALfd+3+r999/H//73P4SFheHpp59GTk4O3n33Xfj6+prtGxcXh/j4eDz44IPw8fExfbrx5ptvIjc3F8899xwAYOrUqSgqKsL333+PBQsWwMnJCcDtx4YUFhbiiSeewJUrV/Doo4+iZ8+eSExMxLfffotffvkFGzduNPskacmSJSgrK8PUqVOhVCrx7bffYv78+fDz8zPrgtZcZ8+exfTp0yGXyzFt2jS4urri4MGDWLx4MZKSkkyf6lRWVmLGjBnIysrCk08+ic6dO6OoqAjJycmIj4/H5MmTAQArVqzAsmXLMHz4cDz++OOQyWRITU3FgQMHUFFRYTWfXBF1aAIRUTuyefNmISAgQNi8ebOofPjw4UJAQICwYcMGs2PKy8uFiooKs/IlS5YIAQEBwpkzZ0xl165dEwICAoRly5aZlYWEhAjXrl0zlRuNRmHChAnC0KFDRed9/fXXhYCAgHrL/vGPf4jKd+3aJQQEBAjffvutqeybb74RAgIChM8++0y0b2358OHDzV5LfQoLC4XZs2cLvXv3Fnr27Cns3LmzUcc1JDo6WujRo4eQlZUlKn/ssceEXr16CTk5OYIg3P37LQiCEBAQILz++uum71NSUoTAwEAhOjpaqKysNJWfO3dOCAwMFAICAkQ/m+LiYrPrV1VVCX/605+E/v37i+q3bNkys+Nr1f6+/fLLL6ayjz76SAgICBC++eYb0b61P58lS5aYHT9x4kShvLzcVJ6ZmSn06tVLmDdvntk1b1X7Hr3zzju33W/q1KlCjx49hMTERFOZ0WgUXnrpJSEgIEA4evSoIAiCkJiYKAQEBAiff/75bc83adIkYdy4cXesHxFZDrvuEFGH4ejoiEceecSsXKlUmlofKysrkZ+fj9zcXAwZMgQA6u06U5+RI0eKZvWRSCQYNGgQ9Ho9iouLG3WOp59+WvT94MGDAQBXrlwxlR08eBAymQzR0dGifadMmQKtVtuo6xiNRrz88stISkrC7t27cf/99+PVV1/F9u3bRfu99dZb6NWrV6P67EdFRaGqqgpbtmwxlaWkpODXX3/FiBEjTIOhW+r9rmv//v0QBAEzZswQ9Znv1asXhg4dara/jY2N6f/Ly8tx48YN5OXlYejQoSgqKsLly5ebXIda33//PZydnTF16lRR+dSpU+Hs7IwffvjB7Jgnn3xS1F3K3d0dXbp0wR9//NHsetSVk5OD06dPY8SIEQgKCjKVSyQSPP/886Z6AzD9Dh0/fhw5OTkNntPOzg5ZWVmIj49vkToSUctj1x0i6jB8fX0bHDi5du1arF+/HpcuXYLRaBRty8/Pb/T5b+Xo6AgAyMvLg62tbZPPUdtVJC8vz1SWmpoKNzc3s/MplUr4+PigoKDgjtfZv38/Dh8+jA8//BA+Pj5YunQp5s6di9deew2VlZWm7hnJyckIDg5uVJ/9MWPGwN7eHjExMXj22WcBAJs3bwYAU7edWi3xftd17do1AEDXrl3Ntvn7++Pw4cOisuLiYixfvhy7d+9GRkaG2TGNeQ8bkpqait69e0MuF/+Jlcvl6Ny5My5cuGB2TEO/O2lpac2ux611AoBu3bqZbevatSukUqnpPfT29sacOXPw+eefY9iwYejRowcGDx6MiIgI9OnTx3TcK6+8ghdeeAHTpk2Dm5sbBg4ciAcffBBjx45t0hgPImo9DPpE1GFoNJp6y7/66it88MEHGDZsGKKjo+Hm5gaFQoGsrCzMnz8fgiA06vy3m33lbs/R2OMbq3bwaFhYGIDqh4Tly5fj+eefx4IFC1BZWYmgoCCcOXMGCxcubNQ5VSoVIiMjsW7dOpw6dQohISHYtm0bPDw8cN9995n2a6n3+2785S9/wY8//ojHHnsMYWFhcHR0hEwmw6FDh7B69Wqzh4/Wdq+mCm2sefPmISoqCj/++CPi4+OxadMmrFq1CrNmzcJf//pXAEC/fv3w/fff4/Dhwzh+/DiOHz+OHTt2YMWKFVi3bp3pIZeILIdBn4g6vK1bt8Lb2xsrV64UBa6ffvrJgrVqmLe3N44dO4bi4mJRq77BYEBqamqjFnWqfZ1paWnw9PQEUB32P/vsM8yZMwdvvfUWvL29ERAQgEmTJjW6blFRUVi3bh1iYmKQn58PvV6POXPmiN7X1ni/a1vEL1++DD8/P9G2lJQU0fcFB
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Create test loader"
      ],
      "metadata": {
        "id": "7gwWvjFwhJen"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "prediction_dataloader = DataLoader(\n",
        "            test_dataset,\n",
        "            sampler = SequentialSampler(test_dataset),\n",
        "            batch_size = batch_size\n",
        "        )"
      ],
      "metadata": {
        "id": "du6qCdHyhMms"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Evaluate on test dataset"
      ],
      "metadata": {
        "id": "n9E84sH2hOt7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print('Predicting labels for {:,} test sentences...'.format(len(test_dataset)))\n",
        "\n",
        "model.eval()\n",
        "predictions , true_labels = [], []\n",
        "\n",
        "for batch in prediction_dataloader:\n",
        "  batch = tuple(t.to(device) for t in batch)\n",
        "  \n",
        "  b_input_ids, b_input_mask, b_labels = batch\n",
        "  \n",
        "  with torch.no_grad():\n",
        "      outputs = model(b_input_ids, token_type_ids=None, \n",
        "                      attention_mask=b_input_mask)\n",
        "\n",
        "  logits = outputs['logits']\n",
        "\n",
        "  logits = logits.detach().cpu().numpy()\n",
        "  label_ids = b_labels.to('cpu').numpy()\n",
        "\n",
        "  predictions.append(logits)\n",
        "  true_labels.append(label_ids)\n",
        "\n",
        "print('    DONE.')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s3nFSXgbhRs1",
        "outputId": "39a16e42-8d7e-4e31-95f1-e29118ce62f3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Predicting labels for 1,000 test sentences...\n",
            "    DONE.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "results_ok = 0\n",
        "results_false = 0\n",
        "for idx, true_labels_batch in enumerate(true_labels):\n",
        "  predictions_i = np.argmax(predictions[idx], axis=1).flatten()\n",
        "  for bidx, true_label in enumerate(true_labels_batch):\n",
        "    if true_label == predictions_i[bidx]:\n",
        "      results_ok += 1\n",
        "    else:\n",
        "      results_false += 1\n",
        "\n",
        "print(\"Correct predictions: {}, incorrect results: {}, accuracy: {}\".format(results_ok, results_false, float(results_ok) / (results_ok + results_false)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eNMYIt7RhWYM",
        "outputId": "7257f066-6539-4e42-d0ae-e6c5609f1812"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Correct predictions: 990, incorrect results: 10, accuracy: 0.99\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# MCC Score"
      ],
      "metadata": {
        "id": "SwHJwpqKhZ51"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import matthews_corrcoef\n",
        "\n",
        "matthews_set = []\n",
        "print('Calculating Matthews Corr. Coef. for each batch...')\n",
        "\n",
        "for i in range(len(true_labels)):\n",
        "  pred_labels_i = np.argmax(predictions[i], axis=1).flatten()\n",
        "  \n",
        "  matthews = matthews_corrcoef(true_labels[i], pred_labels_i)                \n",
        "  matthews_set.append(matthews)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oqfHjUXThb2J",
        "outputId": "2bbfcaeb-5ea8-498e-a5a0-8f2fac83feea"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Calculating Matthews Corr. Coef. for each batch...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ax = sns.barplot(x=list(range(len(matthews_set))), y=matthews_set, ci=None)\n",
        "\n",
        "plt.title('MCC Score per Batch')\n",
        "plt.ylabel('MCC Score (-1 to +1)')\n",
        "plt.xlabel('Batch #')\n",
        "\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 427
        },
        "id": "JJoRzvr0hePf",
        "outputId": "ebc78102-65e6-4847-d3c0-d3825870dc78"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvIAAAGaCAYAAABt1KfmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdfZzM9f7/8efstb1g0S6SlbCu12WKSISUa9ZV2FCIdEo/neX0rXNOp5OSSsdFIcRSrnbXhuMi1ekCITmtagmJ1R5MsdhlLbvz+2MzzM7FztidXR897reb2znz+rw/7/drZnbG0/Sez5osFotFAAAAAAzFp6wbAAAAAOA5gjwAAABgQAR5AAAAwIAI8gAAAIABEeQBAAAAAyLIAwAAAAZEkAcA4AYxfPhwderUqazbAGAQfmXdAAAU144dOxQXFydJGjp0qF544QW7Mb/99ps6dOigS5cuqXXr1kpISLAbs3fvXi1btky7du2S2WyWj4+PbrvtNrVp00aDBw9W7dq1bcZfuHBBK1as0ObNm3Xw4EFlZ2erQoUKatSokR588EH16tVLfn6u32bPnTunhIQEbdq0Sb/88ovy8vJUsWJF1a9fXx07dtSAAQOK8cigsE6dOumXX36x3jaZTKpcubJq1aqlIUOGqHv37tc995YtW5SWlqYnn3yyJFoFgCIR5AHcNAIDA7Vu3TpNnjxZAQEBNsdSUlJksVicButZs2Zp1qxZqlixonr06KE6deooPz9fBw8e1IYNG7Rs2TLt3LlToaGhkqQjR45ozJgx+vnnn9W2bVuNGTNGFStW1G+//abt27drypQpOnjwoP785z877TcrK0uxsbFKT0/XAw88oP79+8vf31/p6en65ptvtGTJEoK8F1StWlXPPPOMJCk/P18nTpxQcnKynnnmGZnNZo0YMeK65t2yZYuSk5MJ8gBKDUEewE2jS5cuWrdunbZs2aKHHnrI5lhSUpLuvfdeffXVV3bnrV69WjNnztRdd92l2bNnKywszOb4s88+q1mzZllv5+TkaOzYsTp27Jhmzpyprl272owfM2aMUlNTtXfvXpf9rly5Uj///LP+8pe/6JFHHrE7bjabi7zP3pCVlWX9B4uRWCwWnT9/XiEhIS7HhYWFqXfv3ja1QYMGqX379kpKSrruIA8ApY098gBuGg0bNlS9evWUlJRkU09NTdWBAwfUv39/u3Nyc3M1Y8YMBQcHa8aMGXYhXpKCgoI0adIka7hdtWqVDh8+rJEjR9qF+CtiYmI0dOhQl/3+/PPPkqQ2bdo4PB4REWFXO3LkiKZMmaJ7771XjRs3Vrt27TRu3Dh99913NuO2bNmiwYMHq1mzZmrevLkGDx6sLVu22M3XqVMnDR8+XD/88IMeffRRtWzZUr169bLp8dlnn1W7du3UuHFjderUSa+++qrOnz/v8r4Vnv/7779XXFycmjdvrtatWys+Pl6//fab3fjc3Fy988476t69u5o0aaJWrVrp8ccf1w8//GAzbseOHdbnetmyZXrooYfUpEkTLVy40K2+CqtQoYICAgLk7+9vU09NTdXkyZP1wAMPqGnTptbH8qOPPrIZN3z4cCUnJ0uS6tWrZ/1z7c+i2WzWSy+9pPvvv1+NGzdWmzZtNHLkSG3dutWunxMnTuiZZ57RnXfeqaZNm+rRRx/V4cOHr+u+Abh58Yk8gJtK//799corr+jEiROqUqWKpIJP3CtXrqz77rvPbvw333wjs9ms3r17q1KlSm6tsWnTJkkFn+IWR1RUlKSC/1owadKkIvfT7927VyNGjNDly5cVGxurunXr6syZM9q5c6f27Nmjxo0bS5KWLVumF198UXfccYfGjx8vSUpOTtYTTzyhF1980a7vjIwMPfLII+rWrZu6du1qDenfffedHnnkEZUvX16DBg1SlSpVtG/fPiUkJGjPnj1KSEiwC76OHD9+XCNGjFDXrl31wAMP6IcfflBiYqK+++47rV69WuXKlZMkXbp0SY8++qj27Nmj3r17a+jQocrKytLKlSs1ZMgQLV26VE2aNLGZe/HixcrMzNSAAQMUERGhqlWrFtlPXl6eTp06Jalga43ZbNaSJUuUnZ2twYMH24z96KOP9NNPP6lbt26qXr26MjMzlZycrAkTJmj69Onq2bOnJOnxxx9Xfn6+vv76a02bNs16fosWLSRJx44d05AhQ/Tbb7+pd+/eaty4sS5cuKBvv/1W27Zt0z333GM95/z58xo2bJiaNm2qiRMn6tixY1qyZInGjx+vdevWydfXt8j7COAPwgIABvfVV19ZoqOjLe+++67l1KlTlkaNGlnefvtti8VisVy4cMHSsmVLyyuvvGKxWCyWZs2aWYYNG2Y9d8mSJZbo6GjLwoUL3V6vdevWlhYtWhS778zMTEuHDh0s0dHRljZt2liefPJJy9y5cy27du2y5OXl2YzNz8+3dO/e3dK4cWNLWlqa3VxXxmdmZlqaNWtm6dy5s+XcuXPW4+fOnbPcf//9lmbNmlnOnDljrXfs2NESHR1tWblypd2cPXv2tDzwwAM281gsFsvmzZst0dHRlsTExCLv45X5Fy1aZFNftGiRJTo62jJ37ly72ueff24z9ty5c5YOHTrYPG9XnvM777zT8uuvvxbZR+F+Cv9p0qSJZfny5Xbjs7Oz7Wrnz5+3dO3a1fLggw/a1OPj4y3R0dEO133ssccc3jeLxWLzXA8bNswSHR1tmTdvns2Y+fPnOz0fwB8XW2sA3FQqVqyoTp06Wbc5bN68WefOnXO4rUYq2A8uyaM94VlZWUXuw3ZHhQoVlJSUpNGjRyssLEybNm3S66+/rqFDh6pz58768ssvrWPT0tJ04MAB9evXT/Xr17eby8en4O1869atOn/+vIYPH25zn0JDQzV8+HCdP39e27Ztszk3PDxc/fr1s6nt379f+/fvV48ePZSbm6tTp05Z/7Rs2VLBwcEOt4Q4Ehoaqocfftim9vDDDys0NNRmi8qHH36oO+64Q40aNbJZLzc3V23bttXu3buVk5NjM0/v3r1VuXJlt/q4onr16lq0aJEWLVqkhQsX6pVXXlHTpk31t7/9TYmJiTZjg4ODrf//woULOn36tC5cuKC7775bhw4dsv78uJKZmakvvvhC7du3V/v27e2OX3nurr195SpMV9x9992SCrZWAcAVbK0BcNPp37+/xowZo6+//lqJiYmKiYlRnTp1HI69Enazs7Pdnj80NNSj8a5UqlRJkyZN0qRJk3T69Gn997//1YYNG/Thhx9qwoQJSklJUc2aNa376Rs2bOhyvmPHjkmS6tata3fsSi09Pd2mXqNGDbvtGocOHZIkzZw5UzNnznS41q+//lr0Hfx9/sJXEQoICFCNGjVsejl06JBycnKcfmdAkk6fPq1q1apZb99+++1u9XCt4OBgtW3b1qbWs2dP9e3bVy+99JI6deqkihUrSiq4bOmMGTP08ccfO9zTf/bs2SL/EXj06FFZLJYin7srIiMjFRgYaFMLDw+XVPCPAgC4giAP4KbTrl07ValSRbNnz9aOHTv0t7/9zenYK+G28JcpXalbt6527dql9PR01ahRo7jtWlWsWFEdO3ZUx44dVa1aNb3zzjtav369dZ+7t1zZo+7IqFGjHH6KLEnly5cv0T4sFouio6M1ZcoUp2MKf4/BVe+e8PPz0913360lS5YoNTVVHTp0kMVi0ahRo3To0CHFxcWpcePGCgsLk6+vrxITE7Vu3Trl5+eXyPrXcrUH3mKxlPh6AIyLIA/gpuPr66s+ffpo7ty5CgoKUo8ePZyObdGihSIiIrRlyxadPn3a+kmsK127dtWuXbu0atUq6/XIS1rTpk0lFVy9RJJq1aolqWCLjStX/mFx4MABu0+2Dx48aDPGlZo1a0oq2OZR+NNrT6Wnpys3N9fmU/nc3Fylp6frjjvusFnz9OnTuvvuu+22m5SGy5cvS7r6X2f279+vffv26YknntCf/vQnm7GrVq2yO99kMjmcNyoqSiaTqcjnDgA8xR55ADelwYMHa
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "flat_predictions = np.concatenate(predictions, axis=0)\n",
        "flat_predictions = np.argmax(flat_predictions, axis=1).flatten()\n",
        "\n",
        "flat_true_labels = np.concatenate(true_labels, axis=0)\n",
        "\n",
        "mcc = matthews_corrcoef(flat_true_labels, flat_predictions)\n",
        "\n",
        "print('Total MCC: %.3f' % mcc)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8XER3sOFhfny",
        "outputId": "77ec6114-8ab3-4abd-c7b7-de95528a2bef"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total MCC: 0.960\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Save model"
      ],
      "metadata": {
        "id": "ZTd3f1yKhhkP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "\n",
        "drive.mount('/content/gdrive/', force_remount=True)\n",
        "\n",
        "output_dir = '/content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/GPT2_Model'\n",
        "print(\"Saving model to %s\" % output_dir)\n",
        "\n",
        "model_to_save = model.module if hasattr(model, 'module') else model\n",
        "model_to_save.save_pretrained(output_dir)\n",
        "tokenizer.save_pretrained(output_dir)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gqSUWqCqhizx",
        "outputId": "76d1febd-031d-456a-b108-7b664b2b5729"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive/\n",
            "Saving model to /content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/GPT2_Model\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('/content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/GPT2_Model/tokenizer_config.json',\n",
              " '/content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/GPT2_Model/special_tokens_map.json',\n",
              " '/content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/GPT2_Model/vocab.json',\n",
              " '/content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/GPT2_Model/merges.txt',\n",
              " '/content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/GPT2_Model/added_tokens.json')"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Bibliografia\n",
        "- https://gmihaila.github.io/tutorial_notebooks/gpt2_finetune_classification/\n",
        "- https://mccormickml.com/2019/07/22/BERT-fine-tuning/#a1-saving--loading-fine-tuned-model"
      ],
      "metadata": {
        "id": "Er-thm7dkbIW"
      }
    }
  ]
}