Przetwarzanie_tekstu/projekt/BERT_sms_spam.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "371154699498422189229c97ccbfa508": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_51de31720ffc40ba8067dd2c2033851b",
              "IPY_MODEL_8c1eaf11c9db4c09aa6936a201615412",
              "IPY_MODEL_57ac64a5a8534de889c977d558fb81b8"
            ],
            "layout": "IPY_MODEL_7fb1c882a0524cbd8a71ab42cf54d02a"
          }
        },
        "51de31720ffc40ba8067dd2c2033851b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_981ae04516214af6995cd9f846f5f45a",
            "placeholder": "​",
            "style": "IPY_MODEL_815a8f36e137412aa54aa012adb7306d",
            "value": "Downloading (…)"pytorch_model.bin";: 100%"
          }
        },
        "8c1eaf11c9db4c09aa6936a201615412": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_83723a5f3104486193880e58b7e9228c",
            "max": 440473133,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9947d9cd0a124c26b132c75e3bcafd2b",
            "value": 440473133
          }
        },
        "57ac64a5a8534de889c977d558fb81b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a323a1744edb4e84b7f2b80530abc097",
            "placeholder": "​",
            "style": "IPY_MODEL_665af58d5b5349f1a85dd4493407020f",
            "value": " 440M/440M [00:02<00:00, 145MB/s]"
          }
        },
        "7fb1c882a0524cbd8a71ab42cf54d02a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "981ae04516214af6995cd9f846f5f45a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "815a8f36e137412aa54aa012adb7306d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "83723a5f3104486193880e58b7e9228c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9947d9cd0a124c26b132c75e3bcafd2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a323a1744edb4e84b7f2b80530abc097": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "665af58d5b5349f1a85dd4493407020f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1877adc7cb1a4525b016c4b623b0e26c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4f6fc3da4b70433db4e51fd5f9d36aac",
              "IPY_MODEL_b01f51fbdeb64732acf4c379b9d98872",
              "IPY_MODEL_a9c6b1eeba6c43e69660ef104f1b162a"
            ],
            "layout": "IPY_MODEL_aed474ff4bed4e3a85632768c6269341"
          }
        },
        "4f6fc3da4b70433db4e51fd5f9d36aac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_591dc81957d844429bd99dc17803b22d",
            "placeholder": "​",
            "style": "IPY_MODEL_bd9ecb19f63d422392467475e8a18d84",
            "value": "Downloading builder script: 100%"
          }
        },
        "b01f51fbdeb64732acf4c379b9d98872": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_99d7eae0a8a54d47a9bb8a7f29d6fdcd",
            "max": 3208,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f88729c7de1c4138bf47254d06577652",
            "value": 3208
          }
        },
        "a9c6b1eeba6c43e69660ef104f1b162a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6370e81644df40cea0207b1551a6221b",
            "placeholder": "​",
            "style": "IPY_MODEL_c5ad3afe82e04e50bbd66e8e8e84ebd5",
            "value": " 3.21k/3.21k [00:00<00:00, 40.6kB/s]"
          }
        },
        "aed474ff4bed4e3a85632768c6269341": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "591dc81957d844429bd99dc17803b22d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bd9ecb19f63d422392467475e8a18d84": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "99d7eae0a8a54d47a9bb8a7f29d6fdcd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f88729c7de1c4138bf47254d06577652": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6370e81644df40cea0207b1551a6221b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c5ad3afe82e04e50bbd66e8e8e84ebd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cdeea076963c4466805f2db30f8dadae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cf01a082efe7471eabf0958cf9ad8fe7",
              "IPY_MODEL_97147e507dbb4f17a43c674ed459f363",
              "IPY_MODEL_49c4cea660db4cd0a23591ff718ca506"
            ],
            "layout": "IPY_MODEL_93047e8d99924f8cac15aaf46953e20a"
          }
        },
        "cf01a082efe7471eabf0958cf9ad8fe7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_40849aa7220b42b899a09d787afc0f7c",
            "placeholder": "​",
            "style": "IPY_MODEL_3d74c20c83c3474dbe85d61b1b5232f0",
            "value": "Downloading metadata: 100%"
          }
        },
        "97147e507dbb4f17a43c674ed459f363": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9690c5ff99bb46cdb72b639461f3983f",
            "max": 1687,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a90d0945cb13456889b97368eaba4851",
            "value": 1687
          }
        },
        "49c4cea660db4cd0a23591ff718ca506": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1ee19fd8ea7747e4814096d3c24db483",
            "placeholder": "​",
            "style": "IPY_MODEL_16a3d55eccad4f2c844ce98bddb49b1b",
            "value": " 1.69k/1.69k [00:00<00:00, 18.5kB/s]"
          }
        },
        "93047e8d99924f8cac15aaf46953e20a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "40849aa7220b42b899a09d787afc0f7c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3d74c20c83c3474dbe85d61b1b5232f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9690c5ff99bb46cdb72b639461f3983f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a90d0945cb13456889b97368eaba4851": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1ee19fd8ea7747e4814096d3c24db483": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "16a3d55eccad4f2c844ce98bddb49b1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "67f9ea537f544a67bd791cd7afb931ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f3916037b6ba454a88b28175e42afb95",
              "IPY_MODEL_b95bf142874f4259b598970ab04969d0",
              "IPY_MODEL_b61d99fe6cfa4557849c02259ac53185"
            ],
            "layout": "IPY_MODEL_600e74e43b71416284652475cc72e86a"
          }
        },
        "f3916037b6ba454a88b28175e42afb95": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e83f9fbe6f2345868ff5f52b1e718684",
            "placeholder": "​",
            "style": "IPY_MODEL_7985caa4f03e43aabd664b812329704b",
            "value": "Downloading readme: 100%"
          }
        },
        "b95bf142874f4259b598970ab04969d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_76342fc1b67d442ea1d5e7b14d3b6c2b",
            "max": 4872,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_64d332e261c24520af234d9014052789",
            "value": 4872
          }
        },
        "b61d99fe6cfa4557849c02259ac53185": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_871154eda162439fb31323272ef7189e",
            "placeholder": "​",
            "style": "IPY_MODEL_276d93f3c4404d8b8ad29292a58cbe78",
            "value": " 4.87k/4.87k [00:00<00:00, 101kB/s]"
          }
        },
        "600e74e43b71416284652475cc72e86a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e83f9fbe6f2345868ff5f52b1e718684": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7985caa4f03e43aabd664b812329704b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "76342fc1b67d442ea1d5e7b14d3b6c2b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "64d332e261c24520af234d9014052789": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "871154eda162439fb31323272ef7189e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "276d93f3c4404d8b8ad29292a58cbe78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ed5b0b0708de49f5ba826e4e72e51e0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1786fefd8f7045fe94288c47aefcb1ef",
              "IPY_MODEL_bb729621dd3045caa18a648dbdae5c42",
              "IPY_MODEL_e31faab315654a86a831ae9561a4ebcb"
            ],
            "layout": "IPY_MODEL_f635fbb6aa304b4abfd04ca248fd63da"
          }
        },
        "1786fefd8f7045fe94288c47aefcb1ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_99e2b2ca8ba5476d92ac2c8c740ac747",
            "placeholder": "​",
            "style": "IPY_MODEL_9e24e28b5490432f982195e3fcec6e9f",
            "value": "Downloading data: 100%"
          }
        },
        "bb729621dd3045caa18a648dbdae5c42": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4782fddda552489eaf98ef72ee2cb33b",
            "max": 203415,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8844bf52874b4e2e8c89e4074d046c0c",
            "value": 203415
          }
        },
        "e31faab315654a86a831ae9561a4ebcb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_da71ed4edff047f2b0077716cdb34559",
            "placeholder": "​",
            "style": "IPY_MODEL_e48116245d9249c6a44783022ac07e7c",
            "value": " 203k/203k [00:00<00:00, 8.67kB/s]"
          }
        },
        "f635fbb6aa304b4abfd04ca248fd63da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "99e2b2ca8ba5476d92ac2c8c740ac747": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9e24e28b5490432f982195e3fcec6e9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4782fddda552489eaf98ef72ee2cb33b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8844bf52874b4e2e8c89e4074d046c0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "da71ed4edff047f2b0077716cdb34559": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e48116245d9249c6a44783022ac07e7c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c9fbec63c320439ca6885333433e74f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cee19025ed0b4ac1a58d0fc330797cd5",
              "IPY_MODEL_c93158c8990b482ead12b98d05e59340",
              "IPY_MODEL_dcdc267085844396b9007043e950b802"
            ],
            "layout": "IPY_MODEL_216e0797bc3149579085f302d06d071a"
          }
        },
        "cee19025ed0b4ac1a58d0fc330797cd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7818aa6b2a084f2da4360013566d0829",
            "placeholder": "​",
            "style": "IPY_MODEL_8955e603436a49668108af16167216ba",
            "value": "Generating train split: 100%"
          }
        },
        "c93158c8990b482ead12b98d05e59340": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_31a2dc6a5482499e89ea453f6d2d8b99",
            "max": 5574,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_12e95793269647518e10cee66da26495",
            "value": 5574
          }
        },
        "dcdc267085844396b9007043e950b802": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b21104130e9541bf8f3227f1bd0a82a8",
            "placeholder": "​",
            "style": "IPY_MODEL_a8d77fd8c4f64e5a9e00e6406f68fe21",
            "value": " 5574/5574 [00:17<00:00, 6668.87 examples/s]"
          }
        },
        "216e0797bc3149579085f302d06d071a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7818aa6b2a084f2da4360013566d0829": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8955e603436a49668108af16167216ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "31a2dc6a5482499e89ea453f6d2d8b99": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "12e95793269647518e10cee66da26495": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b21104130e9541bf8f3227f1bd0a82a8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a8d77fd8c4f64e5a9e00e6406f68fe21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "87d89b350a3a4999a6a0041e43a4e3f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cf47e047462c4bc5ba703d95ffe12e97",
              "IPY_MODEL_9013b8c8201d497b86b17bf7bb7a1ed2",
              "IPY_MODEL_66e4c8f41a5b49d5b415b51ffd30e27c"
            ],
            "layout": "IPY_MODEL_d580b92217d240c68dc5427fbfbdce17"
          }
        },
        "cf47e047462c4bc5ba703d95ffe12e97": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d62f4230216449388902b9fb729ec97d",
            "placeholder": "​",
            "style": "IPY_MODEL_80dd56148030406dba141f3447d4f548",
            "value": "100%"
          }
        },
        "9013b8c8201d497b86b17bf7bb7a1ed2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c869a3fb7575433bbeb60d6a48166bb9",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_120c856683be4d20ab86722f7cb898d9",
            "value": 1
          }
        },
        "66e4c8f41a5b49d5b415b51ffd30e27c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_52cd03195e5845988a042c54c834dfc1",
            "placeholder": "​",
            "style": "IPY_MODEL_6a0e53c32e934b75aa9b877fcf630371",
            "value": " 1/1 [00:00<00:00, 23.67it/s]"
          }
        },
        "d580b92217d240c68dc5427fbfbdce17": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d62f4230216449388902b9fb729ec97d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "80dd56148030406dba141f3447d4f548": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c869a3fb7575433bbeb60d6a48166bb9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "120c856683be4d20ab86722f7cb898d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "52cd03195e5845988a042c54c834dfc1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6a0e53c32e934b75aa9b877fcf630371": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "99bf66d8aec6450d9ff5fd26d500494c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_890923e10c5d4e64affa391a71ce7282",
              "IPY_MODEL_ff4c790f333b42ed96bf437662131b4c",
              "IPY_MODEL_a5373b2df3d348dabb2277e56774793f"
            ],
            "layout": "IPY_MODEL_ea84d4faa4714f609474394fe01803e1"
          }
        },
        "890923e10c5d4e64affa391a71ce7282": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f87538bb318a4682a1efca4f29c88995",
            "placeholder": "​",
            "style": "IPY_MODEL_0ad66419e2304c6daa9812cf5d0b0d05",
            "value": "Downloading (…)solve/main/vocab.txt: 100%"
          }
        },
        "ff4c790f333b42ed96bf437662131b4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c9bafe9fdc2a44a9b5bf0cc69dad81ce",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_314d4cb8e2bc4f2fa54f63742f974806",
            "value": 231508
          }
        },
        "a5373b2df3d348dabb2277e56774793f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_346386cf68cc4fa3aad1ea30613480f4",
            "placeholder": "​",
            "style": "IPY_MODEL_c1f18d2eceaf4ea289e520fd603851b8",
            "value": " 232k/232k [00:00<00:00, 931kB/s]"
          }
        },
        "ea84d4faa4714f609474394fe01803e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f87538bb318a4682a1efca4f29c88995": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0ad66419e2304c6daa9812cf5d0b0d05": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c9bafe9fdc2a44a9b5bf0cc69dad81ce": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "314d4cb8e2bc4f2fa54f63742f974806": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "346386cf68cc4fa3aad1ea30613480f4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c1f18d2eceaf4ea289e520fd603851b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ad09da5aaf0c40488d4bc6e542e135c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7f2c407700df4b78b47025722b791731",
              "IPY_MODEL_50462493f96845cda39002a9199690c1",
              "IPY_MODEL_682fa8fe6782431d9541974cb1fddcb8"
            ],
            "layout": "IPY_MODEL_fc66f507ecef46a089c7ab5ddd4cc221"
          }
        },
        "7f2c407700df4b78b47025722b791731": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bd5d850ab6fc4bf7ba2547639af6787d",
            "placeholder": "​",
            "style": "IPY_MODEL_5d0aa841704d4b1d8041a486c9836010",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "50462493f96845cda39002a9199690c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f6cb7c5b730947699012901130fc3873",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9ef0d4e16597499f91b8c0ec000699c9",
            "value": 28
          }
        },
        "682fa8fe6782431d9541974cb1fddcb8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_914a002878364acda89a024bd56ae215",
            "placeholder": "​",
            "style": "IPY_MODEL_e3c5222bfc1c42669990caf5664c2181",
            "value": " 28.0/28.0 [00:00<00:00, 213B/s]"
          }
        },
        "fc66f507ecef46a089c7ab5ddd4cc221": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bd5d850ab6fc4bf7ba2547639af6787d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5d0aa841704d4b1d8041a486c9836010": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f6cb7c5b730947699012901130fc3873": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9ef0d4e16597499f91b8c0ec000699c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "914a002878364acda89a024bd56ae215": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e3c5222bfc1c42669990caf5664c2181": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6ba96781e3e2463580fa9f2453c6857a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1ffdb4bfc9c94c4f9d2fe8e367e93b9c",
              "IPY_MODEL_06e9256f06b046fe8566d5edae45c1ba",
              "IPY_MODEL_da726d2194cd4231afbf5922129eb8ec"
            ],
            "layout": "IPY_MODEL_b0b91057028745da92696106688bd770"
          }
        },
        "1ffdb4bfc9c94c4f9d2fe8e367e93b9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_20d0b0999c1c42209aea4a38993ecba9",
            "placeholder": "​",
            "style": "IPY_MODEL_d9a753e4053b4e9994a88dd41df5ad46",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "06e9256f06b046fe8566d5edae45c1ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d0327ba72cee4dddb6198875f0bb68bc",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6a7ddd28b45f4ad9a4ddf308c3d78f6e",
            "value": 570
          }
        },
        "da726d2194cd4231afbf5922129eb8ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_64405aeef9104a79a02d048cf3e0b1b3",
            "placeholder": "​",
            "style": "IPY_MODEL_0f524bc971724924bc4085abaf7d3781",
            "value": " 570/570 [00:00<00:00, 8.75kB/s]"
          }
        },
        "b0b91057028745da92696106688bd770": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "20d0b0999c1c42209aea4a38993ecba9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d9a753e4053b4e9994a88dd41df5ad46": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d0327ba72cee4dddb6198875f0bb68bc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6a7ddd28b45f4ad9a4ddf308c3d78f6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "64405aeef9104a79a02d048cf3e0b1b3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0f524bc971724924bc4085abaf7d3781": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Instalacja pakietów"
      ],
      "metadata": {
        "id": "t2xXKpOpcZg_"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fdpN7ugfauLD",
        "outputId": "2b2e545d-314d-4917-82d7-420e7d9f1c07"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.26.1-py3-none-any.whl (6.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.3/6.3 MB\u001b[0m \u001b[31m40.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting datasets\n",
            "  Downloading datasets-2.9.0-py3-none-any.whl (462 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m462.8/462.8 KB\u001b[0m \u001b[31m15.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.8/dist-packages (1.13.1+cu116)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from transformers) (23.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from transformers) (3.9.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.8/dist-packages (from transformers) (4.64.1)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m20.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (1.21.6)\n",
            "Collecting huggingface-hub<1.0,>=0.11.0\n",
            "  Downloading huggingface_hub-0.12.1-py3-none-any.whl (190 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m190.3/190.3 KB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from transformers) (2.25.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (2022.6.2)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (213 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m213.0/213.0 KB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting responses<0.19\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: dill<0.3.7 in /usr/local/lib/python3.8/dist-packages (from datasets) (0.3.6)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.8/dist-packages (from datasets) (3.8.4)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.8/dist-packages (from datasets) (1.3.5)\n",
            "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.8/dist-packages (from datasets) (9.0.0)\n",
            "Collecting multiprocess\n",
            "  Downloading multiprocess-0.70.14-py38-none-any.whl (132 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.0/132.0 KB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.8/dist-packages (from datasets) (2023.1.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.8/dist-packages (from torch) (4.5.0)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (4.0.2)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (22.2.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.8.2)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (6.0.4)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (3.0.1)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.3.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (4.0.0)\n",
            "Collecting urllib3<1.27,>=1.21.1\n",
            "  Downloading urllib3-1.26.14-py2.py3-none-any.whl (140 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m140.6/140.6 KB\u001b[0m \u001b[31m8.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets) (2022.7.1)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.8/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: tokenizers, xxhash, urllib3, multiprocess, responses, huggingface-hub, transformers, datasets\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 1.24.3\n",
            "    Uninstalling urllib3-1.24.3:\n",
            "      Successfully uninstalled urllib3-1.24.3\n",
            "Successfully installed datasets-2.9.0 huggingface-hub-0.12.1 multiprocess-0.70.14 responses-0.18.0 tokenizers-0.13.2 transformers-4.26.1 urllib3-1.26.14 xxhash-3.2.0\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers datasets torch"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Załadowanie pakietów"
      ],
      "metadata": {
        "id": "s8cfdy_6ldCn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_dataset\n",
        "from transformers import BertTokenizer\n",
        "import torch\n",
        "from torch.utils.data import TensorDataset, random_split\n",
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "from transformers import BertForSequenceClassification, BertConfig\n",
        "from transformers import get_linear_schedule_with_warmup\n",
        "import numpy as np\n",
        "import time\n",
        "import datetime\n",
        "import random"
      ],
      "metadata": {
        "id": "yLS_x9DIlgSs"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Załadowanie datasetu\n",
        "sms_spam"
      ],
      "metadata": {
        "id": "fPwDyJd5cdaE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = load_dataset(\"sms_spam\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 263,
          "referenced_widgets": [
            "1877adc7cb1a4525b016c4b623b0e26c",
            "4f6fc3da4b70433db4e51fd5f9d36aac",
            "b01f51fbdeb64732acf4c379b9d98872",
            "a9c6b1eeba6c43e69660ef104f1b162a",
            "aed474ff4bed4e3a85632768c6269341",
            "591dc81957d844429bd99dc17803b22d",
            "bd9ecb19f63d422392467475e8a18d84",
            "99d7eae0a8a54d47a9bb8a7f29d6fdcd",
            "f88729c7de1c4138bf47254d06577652",
            "6370e81644df40cea0207b1551a6221b",
            "c5ad3afe82e04e50bbd66e8e8e84ebd5",
            "cdeea076963c4466805f2db30f8dadae",
            "cf01a082efe7471eabf0958cf9ad8fe7",
            "97147e507dbb4f17a43c674ed459f363",
            "49c4cea660db4cd0a23591ff718ca506",
            "93047e8d99924f8cac15aaf46953e20a",
            "40849aa7220b42b899a09d787afc0f7c",
            "3d74c20c83c3474dbe85d61b1b5232f0",
            "9690c5ff99bb46cdb72b639461f3983f",
            "a90d0945cb13456889b97368eaba4851",
            "1ee19fd8ea7747e4814096d3c24db483",
            "16a3d55eccad4f2c844ce98bddb49b1b",
            "67f9ea537f544a67bd791cd7afb931ce",
            "f3916037b6ba454a88b28175e42afb95",
            "b95bf142874f4259b598970ab04969d0",
            "b61d99fe6cfa4557849c02259ac53185",
            "600e74e43b71416284652475cc72e86a",
            "e83f9fbe6f2345868ff5f52b1e718684",
            "7985caa4f03e43aabd664b812329704b",
            "76342fc1b67d442ea1d5e7b14d3b6c2b",
            "64d332e261c24520af234d9014052789",
            "871154eda162439fb31323272ef7189e",
            "276d93f3c4404d8b8ad29292a58cbe78",
            "ed5b0b0708de49f5ba826e4e72e51e0c",
            "1786fefd8f7045fe94288c47aefcb1ef",
            "bb729621dd3045caa18a648dbdae5c42",
            "e31faab315654a86a831ae9561a4ebcb",
            "f635fbb6aa304b4abfd04ca248fd63da",
            "99e2b2ca8ba5476d92ac2c8c740ac747",
            "9e24e28b5490432f982195e3fcec6e9f",
            "4782fddda552489eaf98ef72ee2cb33b",
            "8844bf52874b4e2e8c89e4074d046c0c",
            "da71ed4edff047f2b0077716cdb34559",
            "e48116245d9249c6a44783022ac07e7c",
            "c9fbec63c320439ca6885333433e74f6",
            "cee19025ed0b4ac1a58d0fc330797cd5",
            "c93158c8990b482ead12b98d05e59340",
            "dcdc267085844396b9007043e950b802",
            "216e0797bc3149579085f302d06d071a",
            "7818aa6b2a084f2da4360013566d0829",
            "8955e603436a49668108af16167216ba",
            "31a2dc6a5482499e89ea453f6d2d8b99",
            "12e95793269647518e10cee66da26495",
            "b21104130e9541bf8f3227f1bd0a82a8",
            "a8d77fd8c4f64e5a9e00e6406f68fe21",
            "87d89b350a3a4999a6a0041e43a4e3f8",
            "cf47e047462c4bc5ba703d95ffe12e97",
            "9013b8c8201d497b86b17bf7bb7a1ed2",
            "66e4c8f41a5b49d5b415b51ffd30e27c",
            "d580b92217d240c68dc5427fbfbdce17",
            "d62f4230216449388902b9fb729ec97d",
            "80dd56148030406dba141f3447d4f548",
            "c869a3fb7575433bbeb60d6a48166bb9",
            "120c856683be4d20ab86722f7cb898d9",
            "52cd03195e5845988a042c54c834dfc1",
            "6a0e53c32e934b75aa9b877fcf630371"
          ]
        },
        "id": "N1EWeM0KcYtO",
        "outputId": "77762bad-8bed-4d2e-bd81-b8acc5927f0a"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/3.21k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1877adc7cb1a4525b016c4b623b0e26c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading metadata:   0%|          | 0.00/1.69k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "cdeea076963c4466805f2db30f8dadae"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading readme:   0%|          | 0.00/4.87k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "67f9ea537f544a67bd791cd7afb931ce"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading and preparing dataset sms_spam/plain_text to /root/.cache/huggingface/datasets/sms_spam/plain_text/1.0.0/53f051d3b5f62d99d61792c91acefe4f1577ad3e4c216fb0ad39e30b9f20019c...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/203k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ed5b0b0708de49f5ba826e4e72e51e0c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split:   0%|          | 0/5574 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c9fbec63c320439ca6885333433e74f6"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset sms_spam downloaded and prepared to /root/.cache/huggingface/datasets/sms_spam/plain_text/1.0.0/53f051d3b5f62d99d61792c91acefe4f1577ad3e4c216fb0ad39e30b9f20019c. Subsequent calls will reuse this data.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "87d89b350a3a4999a6a0041e43a4e3f8"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset['train'][0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Mf1QIM_dlp2x",
        "outputId": "4faede3d-adc1-4d95-b7e7-8f75394c01dc"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'sms': 'Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...\\n',\n",
              " 'label': 0}"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Tokenizer BERT"
      ],
      "metadata": {
        "id": "Qc7CIjSOchir"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "99bf66d8aec6450d9ff5fd26d500494c",
            "890923e10c5d4e64affa391a71ce7282",
            "ff4c790f333b42ed96bf437662131b4c",
            "a5373b2df3d348dabb2277e56774793f",
            "ea84d4faa4714f609474394fe01803e1",
            "f87538bb318a4682a1efca4f29c88995",
            "0ad66419e2304c6daa9812cf5d0b0d05",
            "c9bafe9fdc2a44a9b5bf0cc69dad81ce",
            "314d4cb8e2bc4f2fa54f63742f974806",
            "346386cf68cc4fa3aad1ea30613480f4",
            "c1f18d2eceaf4ea289e520fd603851b8",
            "ad09da5aaf0c40488d4bc6e542e135c7",
            "7f2c407700df4b78b47025722b791731",
            "50462493f96845cda39002a9199690c1",
            "682fa8fe6782431d9541974cb1fddcb8",
            "fc66f507ecef46a089c7ab5ddd4cc221",
            "bd5d850ab6fc4bf7ba2547639af6787d",
            "5d0aa841704d4b1d8041a486c9836010",
            "f6cb7c5b730947699012901130fc3873",
            "9ef0d4e16597499f91b8c0ec000699c9",
            "914a002878364acda89a024bd56ae215",
            "e3c5222bfc1c42669990caf5664c2181",
            "6ba96781e3e2463580fa9f2453c6857a",
            "1ffdb4bfc9c94c4f9d2fe8e367e93b9c",
            "06e9256f06b046fe8566d5edae45c1ba",
            "da726d2194cd4231afbf5922129eb8ec",
            "b0b91057028745da92696106688bd770",
            "20d0b0999c1c42209aea4a38993ecba9",
            "d9a753e4053b4e9994a88dd41df5ad46",
            "d0327ba72cee4dddb6198875f0bb68bc",
            "6a7ddd28b45f4ad9a4ddf308c3d78f6e",
            "64405aeef9104a79a02d048cf3e0b1b3",
            "0f524bc971724924bc4085abaf7d3781"
          ]
        },
        "id": "hmnlC_hubLmP",
        "outputId": "2b20ce23-5568-4b0f-f4fe-eefa5fce266e"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "99bf66d8aec6450d9ff5fd26d500494c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ad09da5aaf0c40488d4bc6e542e135c7"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6ba96781e3e2463580fa9f2453c6857a"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sms = dataset['train'][0]['sms']\n",
        "print('Original: ', sms)\n",
        "print('Tokenized: ', tokenizer.tokenize(sms))\n",
        "print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sms)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZxigrpcQdWCF",
        "outputId": "60c33065-3487-4a2b-c9d1-b975d29d4b18"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original:  Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...\n",
            "\n",
            "Tokenized:  ['go', 'until', 'ju', '##rong', 'point', ',', 'crazy', '.', '.', 'available', 'only', 'in', 'bug', '##is', 'n', 'great', 'world', 'la', 'e', 'buffet', '.', '.', '.', 'ci', '##ne', 'there', 'got', 'amore', 'wat', '.', '.', '.']\n",
            "Token IDs:  [2175, 2127, 18414, 17583, 2391, 1010, 4689, 1012, 1012, 2800, 2069, 1999, 11829, 2483, 1050, 2307, 2088, 2474, 1041, 28305, 1012, 1012, 1012, 25022, 2638, 2045, 2288, 26297, 28194, 1012, 1012, 1012]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Check maximum length of a sentence"
      ],
      "metadata": {
        "id": "wVT0m8T7evoz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "max_len = 0\n",
        "\n",
        "for sentence in dataset['train']:\n",
        "    input_ids = tokenizer.encode(sentence['sms'], add_special_tokens=True)\n",
        "    max_len = max(max_len, len(input_ids))\n",
        "\n",
        "print('Max sentence length: ', max_len)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cmUVPrQYez3J",
        "outputId": "9e5f5ef6-4551-4cc8-de91-25991002ebae"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Max sentence length:  238\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Special tokenization"
      ],
      "metadata": {
        "id": "2NfXDfYifX5S"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "input_ids = []\n",
        "attention_masks = []\n",
        "\n",
        "for sentence in dataset['train']:\n",
        "    encoded_dict = tokenizer.encode_plus(\n",
        "                        sentence['sms'],\n",
        "                        add_special_tokens = True,\n",
        "                        max_length = 240,\n",
        "                        padding = 'max_length',\n",
        "                        truncation=True,\n",
        "                        return_attention_mask = True,\n",
        "                        return_tensors = 'pt',\n",
        "                   )\n",
        "    \n",
        "    input_ids.append(encoded_dict['input_ids'])\n",
        "    attention_masks.append(encoded_dict['attention_mask'])\n",
        "\n",
        "input_ids = torch.cat(input_ids, dim=0)\n",
        "attention_masks = torch.cat(attention_masks, dim=0)\n",
        "labels = torch.tensor([sentence['label'] for sentence in dataset['train']])\n",
        "\n",
        "print('Original: ', dataset['train'][0])\n",
        "print('Token IDs:', input_ids[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4u03dIS1fbKU",
        "outputId": "3faafb89-6c7e-4485-9fb3-cc6d2cb89e27"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original:  {'sms': 'Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...\\n', 'label': 0}\n",
            "Token IDs: tensor([  101,  2175,  2127, 18414, 17583,  2391,  1010,  4689,  1012,  1012,\n",
            "         2800,  2069,  1999, 11829,  2483,  1050,  2307,  2088,  2474,  1041,\n",
            "        28305,  1012,  1012,  1012, 25022,  2638,  2045,  2288, 26297, 28194,\n",
            "         1012,  1012,  1012,   102,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Split dataset\n",
        "Class balance ratio should be similar to base dataset ratio."
      ],
      "metadata": {
        "id": "Z6cC0YjAhmw_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def check_class_balance(dataset):\n",
        "  spam_count = 0.0\n",
        "  not_spam_count = 0.0\n",
        "  for row in dataset:\n",
        "    if row[2].item() == 1:\n",
        "      spam_count += 1.0\n",
        "    else:\n",
        "      not_spam_count += 1.0\n",
        "  return spam_count / not_spam_count "
      ],
      "metadata": {
        "id": "IhwCBeAjvGdm"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = TensorDataset(input_ids, attention_masks, labels)\n",
        "print(\"Spam to not spam messages ratio: {}\\n\".format(check_class_balance(dataset)))\n",
        "\n",
        "test_size = 1000\n",
        "dataset_len = len(dataset)\n",
        "train_size = int(0.9 * (dataset_len-test_size))\n",
        "val_size = (dataset_len-test_size) - train_size\n",
        "\n",
        "test_dataset, train_dataset, val_dataset = random_split(dataset, [test_size, train_size, val_size])\n",
        "\n",
        "print('{:>5,} test samples'.format(test_size))\n",
        "print(\"Ratio: {}\\n\".format(check_class_balance(test_dataset)))\n",
        "print('{:>5,} training samples'.format(train_size))\n",
        "print(\"Ratio: {}\\n\".format(check_class_balance(train_dataset)))\n",
        "print('{:>5,} validation samples'.format(val_size))\n",
        "print(\"Ratio: {}\\n\".format(check_class_balance(val_dataset)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vH3yXhA0hT3n",
        "outputId": "f03cb870-88ba-4adc-b85c-ece5a704060a"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Spam to not spam messages ratio: 0.15475450590428838\n",
            "\n",
            "1,000 test samples\n",
            "Ratio: 0.17370892018779344\n",
            "\n",
            "4,116 training samples\n",
            "Ratio: 0.149399609047752\n",
            "\n",
            "  458 validation samples\n",
            "Ratio: 0.16243654822335024\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Create train and validation loaders"
      ],
      "metadata": {
        "id": "z1hVsejihpO2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "batch_size = 32\n",
        "\n",
        "train_dataloader = DataLoader(\n",
        "            train_dataset,\n",
        "            sampler = RandomSampler(train_dataset),\n",
        "            batch_size = batch_size\n",
        "        )\n",
        "\n",
        "validation_dataloader = DataLoader(\n",
        "            val_dataset,\n",
        "            sampler = SequentialSampler(val_dataset),\n",
        "            batch_size = batch_size\n",
        "        )"
      ],
      "metadata": {
        "id": "k4pXght6hre3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Device check"
      ],
      "metadata": {
        "id": "MnErwHAbl_rF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "if torch.cuda.is_available():    \n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aUvyBFxzmBUy",
        "outputId": "830f843f-f1ab-47ee-def7-0dfa3943b264"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla T4\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Load BERT model"
      ],
      "metadata": {
        "id": "o-YrojT-iIfY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = BertForSequenceClassification.from_pretrained(\n",
        "    \"bert-base-uncased\",\n",
        "    num_labels = 2,\n",
        "    output_attentions = False,\n",
        "    output_hidden_states = False,\n",
        ")\n",
        "\n",
        "model.cuda()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "371154699498422189229c97ccbfa508",
            "51de31720ffc40ba8067dd2c2033851b",
            "8c1eaf11c9db4c09aa6936a201615412",
            "57ac64a5a8534de889c977d558fb81b8",
            "7fb1c882a0524cbd8a71ab42cf54d02a",
            "981ae04516214af6995cd9f846f5f45a",
            "815a8f36e137412aa54aa012adb7306d",
            "83723a5f3104486193880e58b7e9228c",
            "9947d9cd0a124c26b132c75e3bcafd2b",
            "a323a1744edb4e84b7f2b80530abc097",
            "665af58d5b5349f1a85dd4493407020f"
          ]
        },
        "id": "sIP3VGZmiK9s",
        "outputId": "2f4e0a13-f379-4033-cca7-92dfc0155cd5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)\"pytorch_model.bin\";:   0%|          | 0.00/440M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "371154699498422189229c97ccbfa508"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForSequenceClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Model architecture"
      ],
      "metadata": {
        "id": "l5WEUOO_igvM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "params = list(model.named_parameters())\n",
        "\n",
        "print('The BERT model has {:} different named parameters.\\n'.format(len(params)))\n",
        "\n",
        "print('==== Embedding Layer ====\\n')\n",
        "\n",
        "for p in params[0:5]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "\n",
        "print('\\n==== First Transformer ====\\n')\n",
        "\n",
        "for p in params[5:21]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "\n",
        "print('\\n==== Output Layer ====\\n')\n",
        "\n",
        "for p in params[-4:]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QRAQLbNuigcW",
        "outputId": "b01b3fc8-1c72-4529-bd88-cc019c097361"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The BERT model has 201 different named parameters.\n",
            "\n",
            "==== Embedding Layer ====\n",
            "\n",
            "bert.embeddings.word_embeddings.weight                  (30522, 768)\n",
            "bert.embeddings.position_embeddings.weight                (512, 768)\n",
            "bert.embeddings.token_type_embeddings.weight                (2, 768)\n",
            "bert.embeddings.LayerNorm.weight                              (768,)\n",
            "bert.embeddings.LayerNorm.bias                                (768,)\n",
            "\n",
            "==== First Transformer ====\n",
            "\n",
            "bert.encoder.layer.0.attention.self.query.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.query.bias                (768,)\n",
            "bert.encoder.layer.0.attention.self.key.weight            (768, 768)\n",
            "bert.encoder.layer.0.attention.self.key.bias                  (768,)\n",
            "bert.encoder.layer.0.attention.self.value.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.value.bias                (768,)\n",
            "bert.encoder.layer.0.attention.output.dense.weight        (768, 768)\n",
            "bert.encoder.layer.0.attention.output.dense.bias              (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.weight        (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.bias          (768,)\n",
            "bert.encoder.layer.0.intermediate.dense.weight           (3072, 768)\n",
            "bert.encoder.layer.0.intermediate.dense.bias                 (3072,)\n",
            "bert.encoder.layer.0.output.dense.weight                 (768, 3072)\n",
            "bert.encoder.layer.0.output.dense.bias                        (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.weight                  (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.bias                    (768,)\n",
            "\n",
            "==== Output Layer ====\n",
            "\n",
            "bert.pooler.dense.weight                                  (768, 768)\n",
            "bert.pooler.dense.bias                                        (768,)\n",
            "classifier.weight                                           (2, 768)\n",
            "classifier.bias                                                 (2,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Init training parameters"
      ],
      "metadata": {
        "id": "NZDC4iiQizdX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "optimizer = torch.optim.AdamW(model.parameters(),\n",
        "                  lr = 2e-5,\n",
        "                  eps = 1e-8\n",
        "                )\n",
        "\n",
        "epochs = 4\n",
        "\n",
        "total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
        "                                            num_warmup_steps = 0,\n",
        "                                            num_training_steps = total_steps)"
      ],
      "metadata": {
        "id": "_uffUPNEi3S5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Helper functions"
      ],
      "metadata": {
        "id": "bnAwgfZekeYD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def flat_accuracy(preds, labels):\n",
        "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
        "\n",
        "def format_time(elapsed):\n",
        "    '''\n",
        "    Takes a time in seconds and returns a string hh:mm:ss\n",
        "    '''\n",
        "    elapsed_rounded = int(round((elapsed)))\n",
        "    \n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))"
      ],
      "metadata": {
        "id": "Z3XSZuFmkgVr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Training"
      ],
      "metadata": {
        "id": "L-ZeLPfbkqy9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# This training code is based on the `run_glue.py` script here:\n",
        "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
        "\n",
        "seed_val = 42\n",
        "\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "training_stats = []\n",
        "total_t0 = time.time()\n",
        "\n",
        "for epoch_i in range(0, epochs):\n",
        "    \n",
        "    # ========================================\n",
        "    #               Training\n",
        "    # ========================================\n",
        "\n",
        "    print(\"\")\n",
        "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "    print('Training...')\n",
        "\n",
        "    t0 = time.time()\n",
        "    total_train_loss = 0\n",
        "\n",
        "    model.train()\n",
        "\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "        if step % 40 == 0 and not step == 0:\n",
        "            elapsed = format_time(time.time() - t0)\n",
        "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
        "\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "\n",
        "        model.zero_grad()        \n",
        "\n",
        "        outputs = model(b_input_ids, \n",
        "                             token_type_ids=None, \n",
        "                             attention_mask=b_input_mask, \n",
        "                             labels=b_labels)\n",
        "\n",
        "        loss = outputs['loss']\n",
        "        total_train_loss += loss.item()\n",
        "\n",
        "        loss.backward()\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "        optimizer.step()\n",
        "        scheduler.step()\n",
        "\n",
        "    avg_train_loss = total_train_loss / len(train_dataloader)            \n",
        "    training_time = format_time(time.time() - t0)\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "    print(\"  Training epcoh took: {:}\".format(training_time))\n",
        "        \n",
        "    # ========================================\n",
        "    #               Validation\n",
        "    # ========================================\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"Running Validation...\")\n",
        "\n",
        "    t0 = time.time()\n",
        "    model.eval()\n",
        "\n",
        "    total_eval_accuracy = 0\n",
        "    total_eval_loss = 0\n",
        "    nb_eval_steps = 0\n",
        "\n",
        "    for batch in validation_dataloader:\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "        \n",
        "        with torch.no_grad():        \n",
        "            outputs = model(b_input_ids, \n",
        "                                   token_type_ids=None, \n",
        "                                   attention_mask=b_input_mask,\n",
        "                                   labels=b_labels)\n",
        "            loss = outputs['loss']\n",
        "            logits = outputs['logits']\n",
        "            \n",
        "        total_eval_loss += loss.item()\n",
        "\n",
        "        logits = logits.detach().cpu().numpy()\n",
        "        label_ids = b_labels.to('cpu').numpy()\n",
        "\n",
        "        total_eval_accuracy += flat_accuracy(logits, label_ids)\n",
        "        \n",
        "\n",
        "    avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)\n",
        "    print(\"  Accuracy: {0:.2f}\".format(avg_val_accuracy))\n",
        "\n",
        "    avg_val_loss = total_eval_loss / len(validation_dataloader)\n",
        "    validation_time = format_time(time.time() - t0)\n",
        "    \n",
        "    print(\"  Validation Loss: {0:.2f}\".format(avg_val_loss))\n",
        "    print(\"  Validation took: {:}\".format(validation_time))\n",
        "\n",
        "    training_stats.append(\n",
        "        {\n",
        "            'epoch': epoch_i + 1,\n",
        "            'Training Loss': avg_train_loss,\n",
        "            'Valid. Loss': avg_val_loss,\n",
        "            'Valid. Accur.': avg_val_accuracy,\n",
        "            'Training Time': training_time,\n",
        "            'Validation Time': validation_time\n",
        "        }\n",
        "    )\n",
        "\n",
        "print(\"\")\n",
        "print(\"Training complete!\")\n",
        "\n",
        "print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QZ9H2EJNksT_",
        "outputId": "7c5d39fb-13d3-48c0-8d04-1dba8740bfcd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "======== Epoch 1 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    129.    Elapsed: 0:00:49.\n",
            "  Batch    80  of    129.    Elapsed: 0:01:34.\n",
            "  Batch   120  of    129.    Elapsed: 0:02:19.\n",
            "\n",
            "  Average training loss: 0.11\n",
            "  Training epcoh took: 0:02:29\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.99\n",
            "  Validation Loss: 0.07\n",
            "  Validation took: 0:00:06\n",
            "\n",
            "======== Epoch 2 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    129.    Elapsed: 0:00:46.\n",
            "  Batch    80  of    129.    Elapsed: 0:01:30.\n",
            "  Batch   120  of    129.    Elapsed: 0:02:15.\n",
            "\n",
            "  Average training loss: 0.02\n",
            "  Training epcoh took: 0:02:25\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.99\n",
            "  Validation Loss: 0.08\n",
            "  Validation took: 0:00:06\n",
            "\n",
            "======== Epoch 3 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    129.    Elapsed: 0:00:45.\n",
            "  Batch    80  of    129.    Elapsed: 0:01:30.\n",
            "  Batch   120  of    129.    Elapsed: 0:02:15.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Training epcoh took: 0:02:25\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.98\n",
            "  Validation Loss: 0.10\n",
            "  Validation took: 0:00:06\n",
            "\n",
            "======== Epoch 4 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    129.    Elapsed: 0:00:45.\n",
            "  Batch    80  of    129.    Elapsed: 0:01:30.\n",
            "  Batch   120  of    129.    Elapsed: 0:02:15.\n",
            "\n",
            "  Average training loss: 0.00\n",
            "  Training epcoh took: 0:02:25\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.99\n",
            "  Validation Loss: 0.09\n",
            "  Validation took: 0:00:06\n",
            "\n",
            "Training complete!\n",
            "Total training took 0:10:06 (h:mm:ss)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Train summary"
      ],
      "metadata": {
        "id": "eZ1fmJMjrRgc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "pd.set_option('precision', 2)\n",
        "df_stats = pd.DataFrame(data=training_stats)\n",
        "\n",
        "df_stats = df_stats.set_index('epoch')\n",
        "df_stats"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "w4ov2mClrLGW",
        "outputId": "ad5057e3-f0e5-44c0-8c5a-bf4d69c600ab"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       Training Loss  Valid. Loss  Valid. Accur. Training Time Validation Time\n",
              "epoch                                                                         \n",
              "1           1.07e-01         0.07           0.99       0:02:29         0:00:06\n",
              "2           1.89e-02         0.08           0.99       0:02:25         0:00:06\n",
              "3           4.73e-03         0.10           0.98       0:02:25         0:00:06\n",
              "4           1.93e-03         0.09           0.99       0:02:25         0:00:06"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ee4b2d83-3992-4ef7-9181-ffeb684a31a1\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Valid. Loss</th>\n",
              "      <th>Valid. Accur.</th>\n",
              "      <th>Training Time</th>\n",
              "      <th>Validation Time</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>epoch</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.07e-01</td>\n",
              "      <td>0.07</td>\n",
              "      <td>0.99</td>\n",
              "      <td>0:02:29</td>\n",
              "      <td>0:00:06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.89e-02</td>\n",
              "      <td>0.08</td>\n",
              "      <td>0.99</td>\n",
              "      <td>0:02:25</td>\n",
              "      <td>0:00:06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4.73e-03</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.98</td>\n",
              "      <td>0:02:25</td>\n",
              "      <td>0:00:06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1.93e-03</td>\n",
              "      <td>0.09</td>\n",
              "      <td>0.99</td>\n",
              "      <td>0:02:25</td>\n",
              "      <td>0:00:06</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ee4b2d83-3992-4ef7-9181-ffeb684a31a1')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-ee4b2d83-3992-4ef7-9181-ffeb684a31a1 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-ee4b2d83-3992-4ef7-9181-ffeb684a31a1');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "import seaborn as sns\n",
        "\n",
        "sns.set(style='darkgrid')\n",
        "\n",
        "sns.set(font_scale=1.5)\n",
        "plt.rcParams[\"figure.figsize\"] = (12,6)\n",
        "\n",
        "plt.plot(df_stats['Training Loss'], 'b-o', label=\"Training\")\n",
        "plt.plot(df_stats['Valid. Loss'], 'g-o', label=\"Validation\")\n",
        "\n",
        "plt.title(\"Training & Validation Loss\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.legend()\n",
        "plt.xticks([1, 2, 3, 4])\n",
        "\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 427
        },
        "id": "4Jg3MOeZrTf9",
        "outputId": "1782176d-be6e-43c8-8903-d1e698d3d700"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvAAAAGaCAYAAABpIXfbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd1hUZ/o+8HtmmBnK0DuIiihFmoCgqImxRIlix2hMRI3pxZTNbnQ12Y35JtlfNlWzZtckG42xRAVLTDRFTTaxATZE0SiWoLShd6ad3x/IhHFAQYFDuT/XlQs4c87MMxOO3PPO+z5HIgiCACIiIiIi6hKkYhdAREREREQtxwBPRERERNSFMMATEREREXUhDPBERERERF0IAzwRERERURfCAE9ERERE1IUwwBNRj3f16lUEBARg5cqVt30fixcvRkBAQBtW1X0193oHBARg8eLFLbqPlStXIiAgAFevXm3z+pKTkxEQEIAjR460+X0TEbUFC7ELICK6UWuC8N69e9GrV692rKbrqa6uxr///W98++23KCgogJOTE6KiovDUU0/Bz8+vRfexaNEifPfdd9i+fTuCgoKa3EcQBIwZMwbl5eX49ddfYWlp2ZZPo10dOXIEKSkpmDdvHuzs7MQux8zVq1cxZswYPPjgg3j11VfFLoeIOhkGeCLqdN5++22Tn48ePYqvvvoKs2bNQlRUlMltTk5Od/x43t7eSE9Ph0wmu+37eP311/Haa6/dcS1tYdmyZfjmm28QHx+PmJgYqNVq7Nu3DydPnmxxgE9ISMB3332HpKQkLFu2rMl9Dh8+jGvXrmHWrFltEt7T09MhlXbMB8MpKSn46KOPMG3aNLMAP2XKFEycOBFyubxDaiEiai0GeCLqdKZMmWLys16vx1dffYVBgwaZ3XajyspKqFSqVj2eRCKBUqlsdZ2NdZawV1NTgz179mDEiBF49913jdufeeYZaDSaFt/PiBEj4Onpia+//hp/+ctfoFAozPZJTk4GUB/228Kd/j9oKzKZ7I7ezBERtTfOgSeiLmv06NGYO3cuzpw5g4ULFyIqKgqTJ08GUB/k33//fcycORNDhgxBSEgI7r33XrzzzjuoqakxuZ+m5mQ33rZ//37MmDEDoaGhGDFiBP7f//t/0Ol0JvfR1Bz4hm0VFRX429/+htjYWISGhmL27Nk4efKk2fMpKSnBkiVLMGTIEERERCAxMRFnzpzB3LlzMXr06Ba9JhKJBBKJpMk3FE2F8OZIpVJMmzYNpaWl2Ldvn9ntlZWV+P777+Hv74+wsLBWvd7NaWoOvMFgwH/+8x+MHj0aoaGhiI+Px86dO5s8PisrC3//+98xceJEREREIDw8HNOnT8eWLVtM9lu8eDE++ugjAMCYMWMQEBBg8v+/uTnwxcXFeO211zBy5EiEhIRg5MiReO2111BSUmKyX8Pxhw4dwmeffYaxY8ciJCQE48ePx7Zt21r0WrTG2bNn8fTTT2PIkCEIDQ3FhAkT8Mknn0Cv15vsl5ubiyVLlmDUqFEICQlBbGwsZs+ebVKTwWDAmjVrMGnSJERERCAyMhLjx4/HX//6V2i12javnYhuD0fgiahLy8nJwbx58xAXF4dx48ahuroaAJCfn4+tW7di3LhxiI+Ph4WFBVJSUvDpp58iMzMTn332WYvu/+eff8aGDRswe/ZszJgxA3v37sV///tf2Nvb44knnmjRfSxcuBBOTk54+umnUVpais8//xyPPfYY9u7da/y0QKPRYMGCBcjMzMT06dMRGhqKc+fOYcGCBbC3t2/x62FpaYmpU6ciKSkJu3btQnx8fIuPvdH06dPx8ccfIzk5GXFxcSa3ffPNN6itrcWMGTMAtN3rfaO33noLX3zxBaKjozF//nwUFRVh+fLl8PHxMds3JSUFaWlpuOeee9CrVy/jpxHLli1DcXExHn/8cQDArFmzUFlZiR9++AFLliyBo6MjgJuvvaioqMADDzyAK1euYMaMGRg4cCAyMzOxceNGHD58GFu2bDH75Of9999HbW0tZs2aBYVCgY0bN2Lx4sXo3bu32VSw23Xq1CnMnTsXFhYWePDBB+Hi4oL9+/fjnXfewdmzZ42fwuh0OixYsAD5+fmYM2cO+vbti8rKSpw7dw5paWmYNm0aAODjjz/GihUrMGrUKMyePRsymQxXr17Fvn37oNFoOs0nTUQ9nkBE1MklJSUJ/v7+QlJSksn2UaNGCf7+/sLmzZvNjqmrqxM0Go3Z9vfff1/w9/cXTp48adyWnZ0t+Pv7CytWrDDbFh4eLmRnZxu3GwwGYeLEicLw4cNN7vfll18W/P39m9z2t7/9zWT7t99+K/j7+wsbN240bvvyyy8Ff39/YdWqVSb7NmwfNWqU2XNpSkVFhfDoo48KISEhwsCBA4VvvvmmRcc1JzExUQgKChLy8/NNtt9///1CcHCwUFRUJAjCnb/egiAI/v7+wssvv2z8OSsrSwgICBASExMFnU5n3J6RkSEEBAQI/v7+Jv9vqqqqzB5fr9cLDz30kBAZGWlS34oVK8yOb9Dw+3b48GHjtvfee0/w9/cXvvzyS5N9G/7/vP/++2bHT5kyRairqzNuz8vLE4KDg4UXXnjB7DFv1PAavfbaazfdb9asWUJQUJCQmZlp3GYwGIRFixYJ/v7+wsGDBwVBEITMzEzB399fWL169U3vb+rUqcJ99913y/qISFycQkNEXZqDgwOmT59utl2hUBhHC3U6HcrKylBcXIxhw4YBQJNTWJoyZswYky43EokEQ4YMgVqtRlVVVYvuY/78+SY/Dx06FABw5coV47b9+/dDJpMhMTHRZN+ZM2fC1ta2RY9jMBjw3HPP4ezZs9i9ezfuvvtuvPTSS/j6669N9nvllVcQHBzcojnxCQkJ0Ov12L59u3FbVlYWTpw4gdGjRxsXEbfV693Y3r17IQgCFixYYDInPTg4GMOHDzfb39ra2vh9XV0dSkpKUFpaiuHDh6OyshIXL15sdQ0NfvjhBzg5OWHWrFkm22fNmgUnJyf8+OOPZsfMmTPHZNqSu7s7fH19cfny5duuo7GioiIcP34co0ePRmBgoHG7RCLBk08+aawbgPF36MiRIygqKmr2PlUqFfLz85GWltYmNRJR++AUGiLq0nx8fJpdcLh+/Xps2rQJFy5cgMFgMLmtrKysxfd/IwcHBwBAaWkpbGxsWn0fDVM2SktLjduuXr0KNzc3s/tTKBTo1asXysvLb/k4e/fuxa+//op//vOf6NWrFz788EM888wz+Mtf/gKdTmecJnHu3DmEhoa2aE78uHHjYGdnh+TkZDz22GMAgKSkJAAwTp9p0Bavd2PZ2dkAgH79+pnd5ufnh19//dVkW1VVFT766CPs3r0bubm5Zse05DVsztWrVxESEgILC9M/mxYWFujbty/OnDljdkxzvzvXrl277TpurAkA+vfvb3Zbv379IJVKja+ht7c3nnjiCaxevRojRoxAUFAQhg4diri4OISFhRmPe/HFF/H000/jwQcfhJubG2JiYnDPPfdg/PjxrVpDQUTtiwGeiLo0KyurJrd//vnn+Mc//oERI0YgMTERbm5ukMvlyM/Px+LFiyEIQovu/2bdSO70Plp6fEs1LLqMjo4GUB/+P/roIzz55JNYsmQJdDodAgMDcfLkSbzxxhstuk+lUon4+Hhs2LABx44dQ3h4OHbu3AkPDw/cddddxv3a6vW+E3/605/w008/4f7770d0dDQcHBwgk8nw888/Y82aNWZvKtpbR7XEbKkXXngBCQkJ+Omnn5CWloatW7fis88+wyOPPII///nPAICIiAj88MMP+PXXX3HkyBEcOXIEu3btwscff4wNGzYY37wSkbgY4ImoW9qxYwe8vb3xySefmASp//3vfyJW1Txvb28cOnQIVVVVJqPwWq0WV69ebdHFhhqe57Vr1+Dp6QmgPsSvWrUKTzzxBF555RV4e3vD398fU6dObXFtCQkJ2LBhA5KTk1FWVga1Wo0nnnjC5HVtj9e7YQT74sWL6N27t8ltW
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Create test loader"
      ],
      "metadata": {
        "id": "kn87qhtR9MP4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "prediction_dataloader = DataLoader(\n",
        "            test_dataset,\n",
        "            sampler = SequentialSampler(test_dataset),\n",
        "            batch_size = batch_size\n",
        "        )"
      ],
      "metadata": {
        "id": "ENqiBZio9a7a"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Evaluate on test dataset"
      ],
      "metadata": {
        "id": "dAF9Iqol-kAP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print('Predicting labels for {:,} test sentences...'.format(len(test_dataset)))\n",
        "\n",
        "model.eval()\n",
        "predictions , true_labels = [], []\n",
        "\n",
        "for batch in prediction_dataloader:\n",
        "  batch = tuple(t.to(device) for t in batch)\n",
        "  \n",
        "  b_input_ids, b_input_mask, b_labels = batch\n",
        "  \n",
        "  with torch.no_grad():\n",
        "      outputs = model(b_input_ids, token_type_ids=None, \n",
        "                      attention_mask=b_input_mask)\n",
        "\n",
        "  logits = outputs['logits']\n",
        "\n",
        "  logits = logits.detach().cpu().numpy()\n",
        "  label_ids = b_labels.to('cpu').numpy()\n",
        "  \n",
        "  predictions.append(logits)\n",
        "  true_labels.append(label_ids)\n",
        "\n",
        "print('    DONE.')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XXCWIk8c9Oun",
        "outputId": "ad95fd9e-f132-4cef-d848-57d126687d8b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Predicting labels for 1,000 test sentences...\n",
            "    DONE.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "results_ok = 0\n",
        "results_false = 0\n",
        "for idx, true_labels_batch in enumerate(true_labels):\n",
        "  predictions_i = np.argmax(predictions[idx], axis=1).flatten()\n",
        "  for bidx, true_label in enumerate(true_labels_batch):\n",
        "    if true_label == predictions_i[bidx]:\n",
        "      results_ok += 1\n",
        "    else:\n",
        "      results_false += 1\n",
        "\n",
        "print(\"Correct predictions: {}, incorrect results: {}, accuracy: {}\".format(results_ok, results_false, float(results_ok) / (results_ok + results_false)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TVIFBFmFEwNv",
        "outputId": "8d0a7751-cb1b-40ed-a705-40944ec75309"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Correct predictions: 994, incorrect results: 6, accuracy: 0.994\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# MCC Score"
      ],
      "metadata": {
        "id": "6gTgKchs_OXY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import matthews_corrcoef\n",
        "\n",
        "matthews_set = []\n",
        "print('Calculating Matthews Corr. Coef. for each batch...')\n",
        "\n",
        "for i in range(len(true_labels)):\n",
        "  pred_labels_i = np.argmax(predictions[i], axis=1).flatten()\n",
        "\n",
        "  matthews = matthews_corrcoef(true_labels[i], pred_labels_i)                \n",
        "  matthews_set.append(matthews)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hOelDEdn_QDE",
        "outputId": "022d5656-09cb-4279-878e-13611ceb68fc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Calculating Matthews Corr. Coef. for each batch...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ax = sns.barplot(x=list(range(len(matthews_set))), y=matthews_set, ci=None)\n",
        "\n",
        "plt.title('MCC Score per Batch')\n",
        "plt.ylabel('MCC Score (-1 to +1)')\n",
        "plt.xlabel('Batch #')\n",
        "\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 427
        },
        "id": "cF34A88N_UgQ",
        "outputId": "439ba21c-94a3-4a7b-bbc8-c2b828dbc6ba"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuUAAAGaCAYAAACopj13AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde3yMZ/7/8ffkLAkSaaKKoIg4xbGUUqVo1JlQiji0qNJt9asbtr92d7vdUlR10RZtldAqkkixDqXdHpyLChWKOoRsmSIhiUgk8/vDV76NScYkmXGTvJ6Ph8fKdd/X5/5MJmbfuXvNNSaLxWIRAAAAAMO4GN0AAAAAUNYRygEAAACDEcoBAAAAgxHKAQAAAIMRygEAAACDEcoBAAAAgxHKAQC4SwwbNkydOnUyug0ABnAzugEAKKmdO3cqMjJSkjRkyBC9/vrrVudcuHBBHTp0UHZ2tlq1aqXo6Gircw4cOKBly5Zp9+7dMpvNcnFxUbVq1dSmTRsNGjRItWvXznf+1atX9cUXX2jTpk06duyY0tPTVbFiRTVs2FDdunVTr1695OZm+2X2ypUrio6O1saNG3X27Fnl5OTI399foaGh6tixowYMGFCC7wxu1alTJ509ezbva5PJpICAANWqVUuDBw9W9+7di1178+bNSkxM1AsvvOCIVgGUMYRyAKWGp6en1q5dq8mTJ8vDwyPfsfj4eFkslkJD8ty5czV37lz5+/urR48eqlOnjnJzc3Xs2DGtX79ey5Yt065du+Tr6ytJOnXqlMaMGaOTJ0+qbdu2GjNmjPz9/XXhwgVt375dU6ZM0bFjx/TnP/+50H7T0tIUERGhpKQkPfHEE+rfv7/c3d2VlJSkvXv3asmSJYRyJ7j//vv18ssvS5Jyc3N17tw5xcXF6eWXX5bZbNaIESOKVXfz5s2Ki4sjlAMoFkI5gFKjS5cuWrt2rTZv3qwnn3wy37HY2Fg9+uij2rFjh9W8VatWac6cOWrdurXmzZun8uXL5zv+yiuvaO7cuXlfZ2ZmauzYsTpz5ozmzJmjrl275jt/zJgxSkhI0IEDB2z2u2LFCp08eVJ/+ctfNHz4cKvjZrP5to/ZGdLS0vJ++biXWCwWZWRkyMfHx+Z55cuXV+/evfONPfXUU2rfvr1iY2OLHcoBoCRYUw6g1GjQoIHq1aun2NjYfOMJCQk6evSo+vfvbzUnKytLs2fPlre3t2bPnm0VyCXJy8tLkyZNyguqK1eu1IkTJzRy5EirQH5TWFiYhgwZYrPfkydPSpLatGlT4PHAwECrsVOnTmnKlCl69NFH1ahRI7Vr107jxo3TwYMH8523efNmDRo0SE2bNlWzZs00aNAgbd682apep06dNGzYMB06dEjPPPOMWrRooV69euXr8ZVXXlG7du3UqFEjderUSW+//bYyMjJsPrZb6//888+KjIxUs2bN1KpVK0VFRenChQtW52dlZenDDz9U9+7d1bhxY7Vs2VLPPfecDh06lO+8nTt35j3Xy5Yt05NPPqnGjRvrk08+sauvW1WsWFEeHh5yd3fPN56QkKDJkyfriSeeUJMmTfK+l1999VW+84YNG6a4uDhJUr169fL+/PFn0Ww2680339Tjjz+uRo0aqU2bNho5cqS2bt1q1c+5c+f08ssv66GHHlKTJk30zDPP6MSJE8V6bADuDdwpB1Cq9O/fX9OmTdO5c+dUuXJlSTfuhAcEBOixxx6zOn/v3r0ym83q3bu3KlWqZNc1Nm7cKOnG3dWSCA4OlnTjLv6kSZNuu/78wIEDGjFihK5fv66IiAjVrVtXqamp2rVrl/bt26dGjRpJkpYtW6Y33nhDDz74oJ5//nlJUlxcnMaPH6833njDqu/k5GQNHz5c4eHh6tq1a17gPnjwoIYPH64KFSroqaeeUuXKlXX48GFFR0dr3759io6OtgqxBfntt980YsQIde3aVU888YQOHTqkmJgYHTx4UKtWrVK5cuUkSdnZ2XrmmWe0b98+9e7dW0OGDFFaWppWrFihwYMHa+nSpWrcuHG+2osXL1ZKSooGDBigwMBA3X///bftJycnRxcvXpR0Y/mK2WzWkiVLlJ6erkGDBuU796uvvtKvv/6q8PBwVa1aVSkpKYqLi9OECRM0c+ZM9ezZU5L03HPPKTc3Vz/++KOmT5+eN7958+aSpDNnzmjw4MG6cOGCevfurUaNGunq1avav3+/tm3bpkceeSRvTkZGhoYOHaomTZpo4sSJOnPmjJYsWaLnn39ea9eulaur620fI4B7kAUA7nE7duywhISEWD766CPLxYsXLQ0bNrR88MEHFovFYrl69aqlRYsWlmnTplksFouladOmlqFDh+bNXbJkiSUkJMTyySef2H29Vq1aWZo3b17ivlNSUiwdOnSwhISEWNq0aWN54YUXLPPnz7fs3r3bkpOTk+/c3NxcS/fu3S2NGjWyJCYmWtW6eX5KSoqladOmls6dO1uuXLmSd/zKlSuWxx9/3NK0aVNLampq3njHjh0tISEhlhUrVljV7Nmzp+WJJ57IV8disVg2bdpkCQkJscTExNz2Md6sv2jRonzjixYtsoSEhFjmz59vNfbdd9/lO/fKlSuWDh065Hvebj7nDz30kOX333+/bR+39nPrn8aNG1uWL19udX56errVWEZGhqVr166Wbt265RuPioqyhISEFHjdZ599tsDHZrFY8j3XQ4cOtYSEhFgWLFiQ75yFCxcWOh9A6cDyFQClir+/vzp16pS3lGDTpk26cuVKgUtXpBvrpyUVaQ11Wlrabdct26NixYqKjY3V6NGjVb58eW3cuFHvvPOOhgwZos6dO+uHH37IOzcxMVFHjx5Vv379FBoaalXLxeXGy/nWrVuVkZGhYcOG5XtMvr6+GjZsmDIyMrRt27Z8c/38/NSvX798Y0eOHNGRI0fUo0cPZWVl6eLFi3l/WrRoIW9v7wKXXRTE19dXTz/9dL6xp59+Wr6+vvmWgXz55Zd68MEH1bBhw3zXy8rKUtu2bbVnzx5lZmbmq9O7d28FBATY1cdNVatW1aJFi7Ro0SJ98sknmjZtmpo0aaK//e1viomJyXeut7d33t+vXr2qS5cu6erVq3r44Yd1/PjxvJ8fW1JSUvT999+rffv2at++vdXxm8/dH7++uZvQTQ8//LCkG8uXAJROLF8BUOr0799fY8aM0Y8//qiYmBiFhYWpTp06BZ57M7imp6fbXd/X17dI59tSqVIlTZo0SZMmTdKlS5f0008/af369fryyy81YcIExcfHq0aNGnnrzxs0aGCz3pkzZyRJdevWtTp2cywpKSnfePXq1a2WRBw/flySNGfOHM2ZM6fAa/3++++3f4D/W//W3XA8PDxUvXr1fL0cP35cmZmZha6xl6RLly6pSpUqeV/XrFnTrh7+yNvbW23bts031rNnT/Xt21dvvvmmOnXqJH9/f0k3ttKcPXu2tmzZUuAa+MuXL9/2F7rTp0/LYrHc9rm7KSgoSJ6envnG/Pz8JN0I+ABKJ0I5gFKnXbt2qly5subNm6edO3fqb3/7W6Hn3gyqt76R0Ja6detq9+7dSkpKUvXq1Uvabh5/f3917NhRHTt2VJUqVfThhx9q3bp1eevCneXmmu6CjBo1qsC7u5JUoUIFh/ZhsVgUEhKiKVOmFHrOrev+bfVeFG5ubnr44Ye1ZMkSJSQkqEOHDrJYLBo1apSOHz+uyMhINWrUSOXLl5erq6tiYmK0du1a5ebmOuT6f2RrzbjFYnH49QDcHQjlAEodV1dX9enTR/Pnz5eXl5d69OhR6LnNmzdXYGCgNm/erEuXLuXdIbWla9eu2r17t1auXJm337WjNWnSRNKNXTgkqVatWpJuLGOx5eYvCUePHrW643zs2LF859hSo0YNSTeWUtx6V7mokpKSlJWVle9ueVZWlpKSkvTggw/mu+alS5f08MMPWy3puBOuX78u6f/+q8mRI0d0+PBhjR8/Xn/605/ynbty5Uqr+SaTqcC6wcHBMplMt33uAJRtrCkHUCoNGjRIEyZM0
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "flat_predictions = np.concatenate(predictions, axis=0)\n",
        "\n",
        "flat_predictions = np.argmax(flat_predictions, axis=1).flatten()\n",
        "flat_true_labels = np.concatenate(true_labels, axis=0)\n",
        "\n",
        "mcc = matthews_corrcoef(flat_true_labels, flat_predictions)\n",
        "\n",
        "print('Total MCC: %.3f' % mcc)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vx_SM19o_XqD",
        "outputId": "a37b67e7-4b48-4c69-cef1-21dbb0513829"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total MCC: 0.973\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Save model"
      ],
      "metadata": {
        "id": "ZE7xxmFJ-oBM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "\n",
        "drive.mount('/content/gdrive/', force_remount=True)\n",
        "\n",
        "output_dir = '/content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/BERT_Model'\n",
        "print(\"Saving model to %s\" % output_dir)\n",
        "\n",
        "model_to_save = model.module if hasattr(model, 'module') else model\n",
        "model_to_save.save_pretrained(output_dir)\n",
        "tokenizer.save_pretrained(output_dir)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fJAqcU8F-p2Z",
        "outputId": "329483d2-97dd-4f1e-ee10-b94869c8e5ac"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive/\n",
            "Saving model to /content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/BERT_Model\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('/content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/BERT_Model/tokenizer_config.json',\n",
              " '/content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/BERT_Model/special_tokens_map.json',\n",
              " '/content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/BERT_Model/vocab.txt',\n",
              " '/content/gdrive/My Drive/UAM/Przetwarzanie-tekstu/BERT_Model/added_tokens.json')"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Bibliografia\n",
        "https://mccormickml.com/2019/07/22/BERT-fine-tuning/#a1-saving--loading-fine-tuned-model"
      ],
      "metadata": {
        "id": "6pFz8n_aHca9"
      }
    }
  ]
}