{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"gpuType":"T4","authorship_tag":"ABX9TyPAY/aWg/HHUkd3lVWeN1bw"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU","widgets":{"application/vnd.jupyter.widget-state+json":{"8a4cd0fe025744e2a571185047c0b16d":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_84fdb66d0ab149cc9a77fa2f16346a15","IPY_MODEL_2b4e41745b7f43dc92c6ab707d273a34","IPY_MODEL_79cb1b3dfcab4f248237bb4176e0c8b0"],"layout":"IPY_MODEL_269b128cbed84afb8017856dc2885669"}},"84fdb66d0ab149cc9a77fa2f16346a15":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_4fe276bbe9bf449083438d902159eb3d","placeholder":"​","style":"IPY_MODEL_1437d395f89c4135a1190ef399e750e0","value":"Map: 100%"}},"2b4e41745b7f43dc92c6ab707d273a34":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_afc7a4c00fb34ee2b6e6aa37956c5b94","max":65749,"min":0,"orientation":"horizontal","style":"IPY_MODEL_f14e0a1b7606419f9318228bcac386c4","value":65749}},"79cb1b3dfcab4f248237bb4176e0c8b0":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_8051c5ef57c14e14a5c01559e042cbf7","placeholder":"​","style":"IPY_MODEL_71d4f17b40b4444aa0a6195b9d353f6a","value":" 65749/65749 [00:34<00:00, 1923.91 examples/s]"}},"269b128cbed84afb8017856dc2885669":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"4fe276bbe9bf449083438d902159eb3d":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1437d395f89c4135a1190ef399e750e0":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"afc7a4c00fb34ee2b6e6aa37956c5b94":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f14e0a1b7606419f9318228bcac386c4":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"8051c5ef57c14e14a5c01559e042cbf7":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"71d4f17b40b4444aa0a6195b9d353f6a":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"5975bb78688648f6929a860df3539dd1":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_45d488d39ce849c0a874628b4a48650e","IPY_MODEL_b543b695bf7347c4bd212d7572d3c8db","IPY_MODEL_5cc60e037e444a08a44d963f589710cd"],"layout":"IPY_MODEL_9da5c3e95a4546d693d9a10b03943092"}},"45d488d39ce849c0a874628b4a48650e":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_124e2c87eaf94821ad77e92f2c640f68","placeholder":"​","style":"IPY_MODEL_c9ca9d5a515e49cfb34d9e9314e52368","value":"Map: 100%"}},"b543b695bf7347c4bd212d7572d3c8db":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_90e003cf18594727a9574dfed98b46e8","max":1600,"min":0,"orientation":"horizontal","style":"IPY_MODEL_0f6dbc3278c7408cb2d7299d03d2eb44","value":1600}},"5cc60e037e444a08a44d963f589710cd":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_99d3750d05f94e2c8ed225fa4f938c1a","placeholder":"​","style":"IPY_MODEL_207dd5fa45d0404394d1d871e6007a20","value":" 1600/1600 [00:00<00:00, 2134.10 examples/s]"}},"9da5c3e95a4546d693d9a10b03943092":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"124e2c87eaf94821ad77e92f2c640f68":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c9ca9d5a515e49cfb34d9e9314e52368":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"90e003cf18594727a9574dfed98b46e8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0f6dbc3278c7408cb2d7299d03d2eb44":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"99d3750d05f94e2c8ed225fa4f938c1a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"207dd5fa45d0404394d1d871e6007a20":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"cells":[{"cell_type":"code","execution_count":1,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"40SPtVYzaWER","executionInfo":{"status":"ok","timestamp":1704975274733,"user_tz":-60,"elapsed":29303,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}},"outputId":"066c2e3e-b0c1-4448-ec21-dc9dcd4f0189"},"outputs":[{"output_type":"stream","name":"stdout","text":["\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m330.3/330.3 kB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m22.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.2/8.2 MB\u001b[0m \u001b[31m66.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m52.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h"]}],"source":["!pip install -q datasets transformers"]},{"cell_type":"code","source":["from datasets import load_dataset\n","import torch\n","from transformers import AutoTokenizer, GPT2ForSequenceClassification, Trainer, TrainingArguments, GPT2Tokenizer,GPT2Config\n","from sklearn.metrics import accuracy_score, precision_recall_fscore_support"],"metadata":{"id":"SC8kQjrzbPEC","executionInfo":{"status":"ok","timestamp":1704975288948,"user_tz":-60,"elapsed":14217,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}}},"execution_count":2,"outputs":[]},{"cell_type":"code","source":["def load_and_process_dataset():\n"," dataset = load_dataset(\"sst2\")\n"," dataset.remove_columns('idx')\n"," del dataset['test']\n"," dataset['test'] = dataset['validation']\n"," del dataset['validation']\n"," split_dataset = dataset['train'].train_test_split(test_size=1600)\n"," dataset['train'] = split_dataset['train']\n"," dataset['validation'] = split_dataset['test']\n"," return dataset"],"metadata":{"id":"-06dFJAqbWXm","executionInfo":{"status":"ok","timestamp":1704975288948,"user_tz":-60,"elapsed":3,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}}},"execution_count":3,"outputs":[]},{"cell_type":"code","source":["def compute_metrics(pred):\n"," labels = pred.label_ids\n"," preds = pred.predictions.argmax(-1)\n"," precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='binary')\n"," acc = accuracy_score(labels, preds)\n"," return {\n"," 'accuracy': acc,\n"," 'f1': f1,\n"," 'precision': precision,\n"," 'recall': recall\n"," }"],"metadata":{"id":"kw84V7XYbZOR","executionInfo":{"status":"ok","timestamp":1704975288949,"user_tz":-60,"elapsed":3,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}}},"execution_count":4,"outputs":[]},{"cell_type":"code","source":["dataset = load_and_process_dataset()\n","dataset"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"GdjWty2AbceZ","executionInfo":{"status":"ok","timestamp":1704975292515,"user_tz":-60,"elapsed":3569,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}},"outputId":"f0cfdb0e-8751-491e-d339-ea391511bc75"},"execution_count":5,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n","The secret `HF_TOKEN` does not exist in your Colab secrets.\n","To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n","You will be able to reuse this secret in all of your notebooks.\n","Please note that authentication is recommended but still optional to access public models or datasets.\n"," warnings.warn(\n"]},{"output_type":"execute_result","data":{"text/plain":["DatasetDict({\n"," train: Dataset({\n"," features: ['idx', 'sentence', 'label'],\n"," num_rows: 65749\n"," })\n"," test: Dataset({\n"," features: ['idx', 'sentence', 'label'],\n"," num_rows: 872\n"," })\n"," validation: Dataset({\n"," features: ['idx', 'sentence', 'label'],\n"," num_rows: 1600\n"," })\n","})"]},"metadata":{},"execution_count":5}]},{"cell_type":"code","source":["train = dataset['train']\n","validation = dataset['validation']\n","test = dataset['test']"],"metadata":{"id":"8_Bd-TWGbgMP","executionInfo":{"status":"ok","timestamp":1704975292515,"user_tz":-60,"elapsed":2,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}}},"execution_count":6,"outputs":[]},{"cell_type":"code","source":["configuration = GPT2Config()\n","\n","tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\n","tokenizer.pad_token = tokenizer.eos_token\n","\n","model = GPT2ForSequenceClassification(configuration).from_pretrained(\"gpt2\")\n","model.config.pad_token_id = model.config.eos_token_id"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Bihi_hAkbjst","executionInfo":{"status":"ok","timestamp":1704975295430,"user_tz":-60,"elapsed":2917,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}},"outputId":"ae9c5398-fbfa-4553-955b-12e88659c0b7"},"execution_count":7,"outputs":[{"output_type":"stream","name":"stderr","text":["Some weights of GPT2ForSequenceClassification were not initialized from the model checkpoint at gpt2 and are newly initialized: ['score.weight']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"]}]},{"cell_type":"code","source":["def tokenization(batched_text):\n"," return tokenizer(batched_text['sentence'], padding = True, truncation=True)\n","\n","\n","train_data = train.map(tokenization, batched = True, batch_size = len(train))\n","val_data = validation.map(tokenization, batched = True, batch_size = len(validation))\n","test_data = test.map(tokenization, batched = True, batch_size = len(test))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":81,"referenced_widgets":["8a4cd0fe025744e2a571185047c0b16d","84fdb66d0ab149cc9a77fa2f16346a15","2b4e41745b7f43dc92c6ab707d273a34","79cb1b3dfcab4f248237bb4176e0c8b0","269b128cbed84afb8017856dc2885669","4fe276bbe9bf449083438d902159eb3d","1437d395f89c4135a1190ef399e750e0","afc7a4c00fb34ee2b6e6aa37956c5b94","f14e0a1b7606419f9318228bcac386c4","8051c5ef57c14e14a5c01559e042cbf7","71d4f17b40b4444aa0a6195b9d353f6a","5975bb78688648f6929a860df3539dd1","45d488d39ce849c0a874628b4a48650e","b543b695bf7347c4bd212d7572d3c8db","5cc60e037e444a08a44d963f589710cd","9da5c3e95a4546d693d9a10b03943092","124e2c87eaf94821ad77e92f2c640f68","c9ca9d5a515e49cfb34d9e9314e52368","90e003cf18594727a9574dfed98b46e8","0f6dbc3278c7408cb2d7299d03d2eb44","99d3750d05f94e2c8ed225fa4f938c1a","207dd5fa45d0404394d1d871e6007a20"]},"id":"1JzFJ46PboS8","executionInfo":{"status":"ok","timestamp":1704975338103,"user_tz":-60,"elapsed":42675,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}},"outputId":"49d9a510-b0d1-4fc0-9f88-3ab858ea76bb"},"execution_count":8,"outputs":[{"output_type":"display_data","data":{"text/plain":["Map: 0%| | 0/65749 [00:00=4.34,<4.35'"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":911},"id":"jWKawt8bcPo3","executionInfo":{"status":"ok","timestamp":1704975352174,"user_tz":-60,"elapsed":14074,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}},"outputId":"49bf178a-04fa-4946-96fd-6e13709fde72"},"execution_count":10,"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting transformers[torch]<4.35,>=4.34\n"," Using cached transformers-4.34.1-py3-none-any.whl (7.7 MB)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (3.13.1)\n","Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (0.20.2)\n","Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (1.23.5)\n","Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (23.2)\n","Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (6.0.1)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (2023.6.3)\n","Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (2.31.0)\n","Collecting tokenizers<0.15,>=0.14 (from transformers[torch]<4.35,>=4.34)\n"," Using cached tokenizers-0.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.8 MB)\n","Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (0.4.1)\n","Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (4.66.1)\n","Requirement already satisfied: torch!=1.12.0,>=1.10 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (2.1.0+cu121)\n","Requirement already satisfied: accelerate>=0.20.3 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (0.26.0)\n","Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.20.3->transformers[torch]<4.35,>=4.34) (5.9.5)\n","Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers[torch]<4.35,>=4.34) (2023.6.0)\n","Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers[torch]<4.35,>=4.34) (4.5.0)\n","Collecting huggingface-hub<1.0,>=0.16.4 (from transformers[torch]<4.35,>=4.34)\n"," Using cached huggingface_hub-0.17.3-py3-none-any.whl (295 kB)\n","Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]<4.35,>=4.34) (1.12)\n","Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]<4.35,>=4.34) (3.2.1)\n","Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]<4.35,>=4.34) (3.1.2)\n","Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]<4.35,>=4.34) (2.1.0)\n","Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]<4.35,>=4.34) (3.3.2)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]<4.35,>=4.34) (3.6)\n","Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]<4.35,>=4.34) (2.0.7)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]<4.35,>=4.34) (2023.11.17)\n","Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch!=1.12.0,>=1.10->transformers[torch]<4.35,>=4.34) (2.1.3)\n","Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch!=1.12.0,>=1.10->transformers[torch]<4.35,>=4.34) (1.3.0)\n","Installing collected packages: huggingface-hub, tokenizers, transformers\n"," Attempting uninstall: huggingface-hub\n"," Found existing installation: huggingface-hub 0.20.2\n"," Uninstalling huggingface-hub-0.20.2:\n"," Successfully uninstalled huggingface-hub-0.20.2\n"," Attempting uninstall: tokenizers\n"," Found existing installation: tokenizers 0.15.0\n"," Uninstalling tokenizers-0.15.0:\n"," Successfully uninstalled tokenizers-0.15.0\n"," Attempting uninstall: transformers\n"," Found existing installation: transformers 4.36.2\n"," Uninstalling transformers-4.36.2:\n"," Successfully uninstalled transformers-4.36.2\n","\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n","datasets 2.16.1 requires huggingface-hub>=0.19.4, but you have huggingface-hub 0.17.3 which is incompatible.\u001b[0m\u001b[31m\n","\u001b[0mSuccessfully installed huggingface-hub-0.17.3 tokenizers-0.14.1 transformers-4.34.1\n"]},{"output_type":"display_data","data":{"application/vnd.colab-display-data+json":{"pip_warning":{"packages":["huggingface_hub","transformers"]}}},"metadata":{}}]},{"cell_type":"code","source":["training_args = TrainingArguments(\n"," output_dir = './results',\n"," num_train_epochs=4,\n"," per_device_train_batch_size = 4,\n"," gradient_accumulation_steps = 16,\n"," per_device_eval_batch_size= 8,\n"," evaluation_strategy = \"epoch\",\n"," disable_tqdm = False,\n"," load_best_model_at_end=False,\n"," warmup_steps=500,\n"," weight_decay=0.01,\n"," logging_steps = 8,\n"," fp16 = True,\n"," logging_dir='./logs',\n"," dataloader_num_workers = 2,\n"," run_name = 'gpt2-classification',\n"," optim=\"adamw_torch\"\n",")"],"metadata":{"id":"699qM1s5bx7s","executionInfo":{"status":"ok","timestamp":1704975352174,"user_tz":-60,"elapsed":4,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}}},"execution_count":11,"outputs":[]},{"cell_type":"code","source":["trainer = Trainer(\n"," model=model,\n"," args=training_args,\n"," compute_metrics=compute_metrics,\n"," train_dataset=train_data,\n"," eval_dataset=val_data,\n",")"],"metadata":{"id":"bEJzHGJmd-3x","executionInfo":{"status":"ok","timestamp":1704975672782,"user_tz":-60,"elapsed":1115,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}}},"execution_count":12,"outputs":[]},{"cell_type":"code","source":["trainer.train()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":290},"id":"uCdhQy7TeBfi","executionInfo":{"status":"ok","timestamp":1704979123491,"user_tz":-60,"elapsed":3448264,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}},"outputId":"25702174-21eb-47ae-fc58-997e96da6da3"},"execution_count":13,"outputs":[{"output_type":"stream","name":"stderr","text":["We strongly recommend passing in an `attention_mask` since your input_ids may be padded. See https://huggingface.co/docs/transformers/troubleshooting#incorrect-output-when-padding-tokens-arent-masked.\n","You may ignore this warning if your `pad_token_id` (50256) is identical to the `bos_token_id` (50256), `eos_token_id` (50256), or the `sep_token_id` (None), and your input is not padded.\n"]},{"output_type":"display_data","data":{"text/plain":[""],"text/html":["\n","
\n"," \n"," \n"," [4108/4108 57:21, Epoch 3/4]\n","
\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
EpochTraining LossValidation LossAccuracyF1PrecisionRecall
00.2537000.2179450.9087500.9159950.9245060.907640
10.1619000.1779210.9393750.9444760.9482760.940707
20.1058000.1869410.9425000.9469430.9579930.936146
30.0868000.1992870.9412500.9467120.9413750.952109

"]},"metadata":{}},{"output_type":"execute_result","data":{"text/plain":["TrainOutput(global_step=4108, training_loss=0.19071476609095162, metrics={'train_runtime': 3447.6287, 'train_samples_per_second': 76.283, 'train_steps_per_second': 1.192, 'total_flos': 8721133740933120.0, 'train_loss': 0.19071476609095162, 'epoch': 4.0})"]},"metadata":{},"execution_count":13}]},{"cell_type":"code","source":["trainer.evaluate()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":193},"id":"3M9QKQWufH1M","executionInfo":{"status":"ok","timestamp":1704979227055,"user_tz":-60,"elapsed":5237,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}},"outputId":"e3d28801-9ea6-4add-a560-1e33d348122f"},"execution_count":14,"outputs":[{"output_type":"display_data","data":{"text/plain":[""],"text/html":["\n","

\n"," \n"," \n"," [200/200 00:04]\n","
\n"," "]},"metadata":{}},{"output_type":"execute_result","data":{"text/plain":["{'eval_loss': 0.1992868036031723,\n"," 'eval_accuracy': 0.94125,\n"," 'eval_f1': 0.9467120181405895,\n"," 'eval_precision': 0.9413754227733935,\n"," 'eval_recall': 0.9521094640820981,\n"," 'eval_runtime': 4.8171,\n"," 'eval_samples_per_second': 332.147,\n"," 'eval_steps_per_second': 41.518,\n"," 'epoch': 4.0}"]},"metadata":{},"execution_count":14}]}]}