1 line
35 KiB
Plaintext
1 line
35 KiB
Plaintext
{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"gpuType":"T4","authorship_tag":"ABX9TyPAY/aWg/HHUkd3lVWeN1bw"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU","widgets":{"application/vnd.jupyter.widget-state+json":{"8a4cd0fe025744e2a571185047c0b16d":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_84fdb66d0ab149cc9a77fa2f16346a15","IPY_MODEL_2b4e41745b7f43dc92c6ab707d273a34","IPY_MODEL_79cb1b3dfcab4f248237bb4176e0c8b0"],"layout":"IPY_MODEL_269b128cbed84afb8017856dc2885669"}},"84fdb66d0ab149cc9a77fa2f16346a15":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_4fe276bbe9bf449083438d902159eb3d","placeholder":"","style":"IPY_MODEL_1437d395f89c4135a1190ef399e750e0","value":"Map: 100%"}},"2b4e41745b7f43dc92c6ab707d273a34":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_afc7a4c00fb34ee2b6e6aa37956c5b94","max":65749,"min":0,"orientation":"horizontal","style":"IPY_MODEL_f14e0a1b7606419f9318228bcac386c4","value":65749}},"79cb1b3dfcab4f248237bb4176e0c8b0":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_8051c5ef57c14e14a5c01559e042cbf7","placeholder":"","style":"IPY_MODEL_71d4f17b40b4444aa0a6195b9d353f6a","value":" 65749/65749 [00:34<00:00, 1923.91 examples/s]"}},"269b128cbed84afb8017856dc2885669":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"4fe276bbe9bf449083438d902159eb3d":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1437d395f89c4135a1190ef399e750e0":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"afc7a4c00fb34ee2b6e6aa37956c5b94":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f14e0a1b7606419f9318228bcac386c4":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"8051c5ef57c14e14a5c01559e042cbf7":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"71d4f17b40b4444aa0a6195b9d353f6a":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"5975bb78688648f6929a860df3539dd1":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_45d488d39ce849c0a874628b4a48650e","IPY_MODEL_b543b695bf7347c4bd212d7572d3c8db","IPY_MODEL_5cc60e037e444a08a44d963f589710cd"],"layout":"IPY_MODEL_9da5c3e95a4546d693d9a10b03943092"}},"45d488d39ce849c0a874628b4a48650e":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_124e2c87eaf94821ad77e92f2c640f68","placeholder":"","style":"IPY_MODEL_c9ca9d5a515e49cfb34d9e9314e52368","value":"Map: 100%"}},"b543b695bf7347c4bd212d7572d3c8db":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_90e003cf18594727a9574dfed98b46e8","max":1600,"min":0,"orientation":"horizontal","style":"IPY_MODEL_0f6dbc3278c7408cb2d7299d03d2eb44","value":1600}},"5cc60e037e444a08a44d963f589710cd":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_99d3750d05f94e2c8ed225fa4f938c1a","placeholder":"","style":"IPY_MODEL_207dd5fa45d0404394d1d871e6007a20","value":" 1600/1600 [00:00<00:00, 2134.10 examples/s]"}},"9da5c3e95a4546d693d9a10b03943092":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"124e2c87eaf94821ad77e92f2c640f68":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c9ca9d5a515e49cfb34d9e9314e52368":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"90e003cf18594727a9574dfed98b46e8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0f6dbc3278c7408cb2d7299d03d2eb44":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"99d3750d05f94e2c8ed225fa4f938c1a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"207dd5fa45d0404394d1d871e6007a20":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"cells":[{"cell_type":"code","execution_count":1,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"40SPtVYzaWER","executionInfo":{"status":"ok","timestamp":1704975274733,"user_tz":-60,"elapsed":29303,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}},"outputId":"066c2e3e-b0c1-4448-ec21-dc9dcd4f0189"},"outputs":[{"output_type":"stream","name":"stdout","text":["\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m330.3/330.3 kB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m22.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.2/8.2 MB\u001b[0m \u001b[31m66.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m52.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h"]}],"source":["!pip install -q datasets transformers"]},{"cell_type":"code","source":["from datasets import load_dataset\n","import torch\n","from transformers import AutoTokenizer, GPT2ForSequenceClassification, Trainer, TrainingArguments, GPT2Tokenizer,GPT2Config\n","from sklearn.metrics import accuracy_score, precision_recall_fscore_support"],"metadata":{"id":"SC8kQjrzbPEC","executionInfo":{"status":"ok","timestamp":1704975288948,"user_tz":-60,"elapsed":14217,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}}},"execution_count":2,"outputs":[]},{"cell_type":"code","source":["def load_and_process_dataset():\n"," dataset = load_dataset(\"sst2\")\n"," dataset.remove_columns('idx')\n"," del dataset['test']\n"," dataset['test'] = dataset['validation']\n"," del dataset['validation']\n"," split_dataset = dataset['train'].train_test_split(test_size=1600)\n"," dataset['train'] = split_dataset['train']\n"," dataset['validation'] = split_dataset['test']\n"," return dataset"],"metadata":{"id":"-06dFJAqbWXm","executionInfo":{"status":"ok","timestamp":1704975288948,"user_tz":-60,"elapsed":3,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}}},"execution_count":3,"outputs":[]},{"cell_type":"code","source":["def compute_metrics(pred):\n"," labels = pred.label_ids\n"," preds = pred.predictions.argmax(-1)\n"," precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='binary')\n"," acc = accuracy_score(labels, preds)\n"," return {\n"," 'accuracy': acc,\n"," 'f1': f1,\n"," 'precision': precision,\n"," 'recall': recall\n"," }"],"metadata":{"id":"kw84V7XYbZOR","executionInfo":{"status":"ok","timestamp":1704975288949,"user_tz":-60,"elapsed":3,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}}},"execution_count":4,"outputs":[]},{"cell_type":"code","source":["dataset = load_and_process_dataset()\n","dataset"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"GdjWty2AbceZ","executionInfo":{"status":"ok","timestamp":1704975292515,"user_tz":-60,"elapsed":3569,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}},"outputId":"f0cfdb0e-8751-491e-d339-ea391511bc75"},"execution_count":5,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n","The secret `HF_TOKEN` does not exist in your Colab secrets.\n","To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n","You will be able to reuse this secret in all of your notebooks.\n","Please note that authentication is recommended but still optional to access public models or datasets.\n"," warnings.warn(\n"]},{"output_type":"execute_result","data":{"text/plain":["DatasetDict({\n"," train: Dataset({\n"," features: ['idx', 'sentence', 'label'],\n"," num_rows: 65749\n"," })\n"," test: Dataset({\n"," features: ['idx', 'sentence', 'label'],\n"," num_rows: 872\n"," })\n"," validation: Dataset({\n"," features: ['idx', 'sentence', 'label'],\n"," num_rows: 1600\n"," })\n","})"]},"metadata":{},"execution_count":5}]},{"cell_type":"code","source":["train = dataset['train']\n","validation = dataset['validation']\n","test = dataset['test']"],"metadata":{"id":"8_Bd-TWGbgMP","executionInfo":{"status":"ok","timestamp":1704975292515,"user_tz":-60,"elapsed":2,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}}},"execution_count":6,"outputs":[]},{"cell_type":"code","source":["configuration = GPT2Config()\n","\n","tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\n","tokenizer.pad_token = tokenizer.eos_token\n","\n","model = GPT2ForSequenceClassification(configuration).from_pretrained(\"gpt2\")\n","model.config.pad_token_id = model.config.eos_token_id"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Bihi_hAkbjst","executionInfo":{"status":"ok","timestamp":1704975295430,"user_tz":-60,"elapsed":2917,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}},"outputId":"ae9c5398-fbfa-4553-955b-12e88659c0b7"},"execution_count":7,"outputs":[{"output_type":"stream","name":"stderr","text":["Some weights of GPT2ForSequenceClassification were not initialized from the model checkpoint at gpt2 and are newly initialized: ['score.weight']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"]}]},{"cell_type":"code","source":["def tokenization(batched_text):\n"," return tokenizer(batched_text['sentence'], padding = True, truncation=True)\n","\n","\n","train_data = train.map(tokenization, batched = True, batch_size = len(train))\n","val_data = validation.map(tokenization, batched = True, batch_size = len(validation))\n","test_data = test.map(tokenization, batched = True, batch_size = len(test))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":81,"referenced_widgets":["8a4cd0fe025744e2a571185047c0b16d","84fdb66d0ab149cc9a77fa2f16346a15","2b4e41745b7f43dc92c6ab707d273a34","79cb1b3dfcab4f248237bb4176e0c8b0","269b128cbed84afb8017856dc2885669","4fe276bbe9bf449083438d902159eb3d","1437d395f89c4135a1190ef399e750e0","afc7a4c00fb34ee2b6e6aa37956c5b94","f14e0a1b7606419f9318228bcac386c4","8051c5ef57c14e14a5c01559e042cbf7","71d4f17b40b4444aa0a6195b9d353f6a","5975bb78688648f6929a860df3539dd1","45d488d39ce849c0a874628b4a48650e","b543b695bf7347c4bd212d7572d3c8db","5cc60e037e444a08a44d963f589710cd","9da5c3e95a4546d693d9a10b03943092","124e2c87eaf94821ad77e92f2c640f68","c9ca9d5a515e49cfb34d9e9314e52368","90e003cf18594727a9574dfed98b46e8","0f6dbc3278c7408cb2d7299d03d2eb44","99d3750d05f94e2c8ed225fa4f938c1a","207dd5fa45d0404394d1d871e6007a20"]},"id":"1JzFJ46PboS8","executionInfo":{"status":"ok","timestamp":1704975338103,"user_tz":-60,"elapsed":42675,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}},"outputId":"49d9a510-b0d1-4fc0-9f88-3ab858ea76bb"},"execution_count":8,"outputs":[{"output_type":"display_data","data":{"text/plain":["Map: 0%| | 0/65749 [00:00<?, ? examples/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8a4cd0fe025744e2a571185047c0b16d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Map: 0%| | 0/1600 [00:00<?, ? examples/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5975bb78688648f6929a860df3539dd1"}},"metadata":{}}]},{"cell_type":"code","source":["train_data.set_format('torch', columns=['input_ids', 'sentence', 'label'])\n","val_data.set_format('torch', columns=['input_ids', 'sentence', 'label'])\n","test_data.set_format('torch', columns=['input_ids', 'sentence', 'label'])"],"metadata":{"id":"g4BX5iv8brRH","executionInfo":{"status":"ok","timestamp":1704975338103,"user_tz":-60,"elapsed":3,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}}},"execution_count":9,"outputs":[]},{"cell_type":"code","source":["!pip install 'transformers[torch]>=4.34,<4.35'"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":911},"id":"jWKawt8bcPo3","executionInfo":{"status":"ok","timestamp":1704975352174,"user_tz":-60,"elapsed":14074,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}},"outputId":"49bf178a-04fa-4946-96fd-6e13709fde72"},"execution_count":10,"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting transformers[torch]<4.35,>=4.34\n"," Using cached transformers-4.34.1-py3-none-any.whl (7.7 MB)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (3.13.1)\n","Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (0.20.2)\n","Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (1.23.5)\n","Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (23.2)\n","Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (6.0.1)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (2023.6.3)\n","Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (2.31.0)\n","Collecting tokenizers<0.15,>=0.14 (from transformers[torch]<4.35,>=4.34)\n"," Using cached tokenizers-0.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.8 MB)\n","Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (0.4.1)\n","Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (4.66.1)\n","Requirement already satisfied: torch!=1.12.0,>=1.10 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (2.1.0+cu121)\n","Requirement already satisfied: accelerate>=0.20.3 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]<4.35,>=4.34) (0.26.0)\n","Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.20.3->transformers[torch]<4.35,>=4.34) (5.9.5)\n","Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers[torch]<4.35,>=4.34) (2023.6.0)\n","Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers[torch]<4.35,>=4.34) (4.5.0)\n","Collecting huggingface-hub<1.0,>=0.16.4 (from transformers[torch]<4.35,>=4.34)\n"," Using cached huggingface_hub-0.17.3-py3-none-any.whl (295 kB)\n","Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]<4.35,>=4.34) (1.12)\n","Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]<4.35,>=4.34) (3.2.1)\n","Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]<4.35,>=4.34) (3.1.2)\n","Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]<4.35,>=4.34) (2.1.0)\n","Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]<4.35,>=4.34) (3.3.2)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]<4.35,>=4.34) (3.6)\n","Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]<4.35,>=4.34) (2.0.7)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]<4.35,>=4.34) (2023.11.17)\n","Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch!=1.12.0,>=1.10->transformers[torch]<4.35,>=4.34) (2.1.3)\n","Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch!=1.12.0,>=1.10->transformers[torch]<4.35,>=4.34) (1.3.0)\n","Installing collected packages: huggingface-hub, tokenizers, transformers\n"," Attempting uninstall: huggingface-hub\n"," Found existing installation: huggingface-hub 0.20.2\n"," Uninstalling huggingface-hub-0.20.2:\n"," Successfully uninstalled huggingface-hub-0.20.2\n"," Attempting uninstall: tokenizers\n"," Found existing installation: tokenizers 0.15.0\n"," Uninstalling tokenizers-0.15.0:\n"," Successfully uninstalled tokenizers-0.15.0\n"," Attempting uninstall: transformers\n"," Found existing installation: transformers 4.36.2\n"," Uninstalling transformers-4.36.2:\n"," Successfully uninstalled transformers-4.36.2\n","\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n","datasets 2.16.1 requires huggingface-hub>=0.19.4, but you have huggingface-hub 0.17.3 which is incompatible.\u001b[0m\u001b[31m\n","\u001b[0mSuccessfully installed huggingface-hub-0.17.3 tokenizers-0.14.1 transformers-4.34.1\n"]},{"output_type":"display_data","data":{"application/vnd.colab-display-data+json":{"pip_warning":{"packages":["huggingface_hub","transformers"]}}},"metadata":{}}]},{"cell_type":"code","source":["training_args = TrainingArguments(\n"," output_dir = './results',\n"," num_train_epochs=4,\n"," per_device_train_batch_size = 4,\n"," gradient_accumulation_steps = 16,\n"," per_device_eval_batch_size= 8,\n"," evaluation_strategy = \"epoch\",\n"," disable_tqdm = False,\n"," load_best_model_at_end=False,\n"," warmup_steps=500,\n"," weight_decay=0.01,\n"," logging_steps = 8,\n"," fp16 = True,\n"," logging_dir='./logs',\n"," dataloader_num_workers = 2,\n"," run_name = 'gpt2-classification',\n"," optim=\"adamw_torch\"\n",")"],"metadata":{"id":"699qM1s5bx7s","executionInfo":{"status":"ok","timestamp":1704975352174,"user_tz":-60,"elapsed":4,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}}},"execution_count":11,"outputs":[]},{"cell_type":"code","source":["trainer = Trainer(\n"," model=model,\n"," args=training_args,\n"," compute_metrics=compute_metrics,\n"," train_dataset=train_data,\n"," eval_dataset=val_data,\n",")"],"metadata":{"id":"bEJzHGJmd-3x","executionInfo":{"status":"ok","timestamp":1704975672782,"user_tz":-60,"elapsed":1115,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}}},"execution_count":12,"outputs":[]},{"cell_type":"code","source":["trainer.train()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":290},"id":"uCdhQy7TeBfi","executionInfo":{"status":"ok","timestamp":1704979123491,"user_tz":-60,"elapsed":3448264,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}},"outputId":"25702174-21eb-47ae-fc58-997e96da6da3"},"execution_count":13,"outputs":[{"output_type":"stream","name":"stderr","text":["We strongly recommend passing in an `attention_mask` since your input_ids may be padded. See https://huggingface.co/docs/transformers/troubleshooting#incorrect-output-when-padding-tokens-arent-masked.\n","You may ignore this warning if your `pad_token_id` (50256) is identical to the `bos_token_id` (50256), `eos_token_id` (50256), or the `sep_token_id` (None), and your input is not padded.\n"]},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n"," <div>\n"," \n"," <progress value='4108' max='4108' style='width:300px; height:20px; vertical-align: middle;'></progress>\n"," [4108/4108 57:21, Epoch 3/4]\n"," </div>\n"," <table border=\"1\" class=\"dataframe\">\n"," <thead>\n"," <tr style=\"text-align: left;\">\n"," <th>Epoch</th>\n"," <th>Training Loss</th>\n"," <th>Validation Loss</th>\n"," <th>Accuracy</th>\n"," <th>F1</th>\n"," <th>Precision</th>\n"," <th>Recall</th>\n"," </tr>\n"," </thead>\n"," <tbody>\n"," <tr>\n"," <td>0</td>\n"," <td>0.253700</td>\n"," <td>0.217945</td>\n"," <td>0.908750</td>\n"," <td>0.915995</td>\n"," <td>0.924506</td>\n"," <td>0.907640</td>\n"," </tr>\n"," <tr>\n"," <td>1</td>\n"," <td>0.161900</td>\n"," <td>0.177921</td>\n"," <td>0.939375</td>\n"," <td>0.944476</td>\n"," <td>0.948276</td>\n"," <td>0.940707</td>\n"," </tr>\n"," <tr>\n"," <td>2</td>\n"," <td>0.105800</td>\n"," <td>0.186941</td>\n"," <td>0.942500</td>\n"," <td>0.946943</td>\n"," <td>0.957993</td>\n"," <td>0.936146</td>\n"," </tr>\n"," <tr>\n"," <td>3</td>\n"," <td>0.086800</td>\n"," <td>0.199287</td>\n"," <td>0.941250</td>\n"," <td>0.946712</td>\n"," <td>0.941375</td>\n"," <td>0.952109</td>\n"," </tr>\n"," </tbody>\n","</table><p>"]},"metadata":{}},{"output_type":"execute_result","data":{"text/plain":["TrainOutput(global_step=4108, training_loss=0.19071476609095162, metrics={'train_runtime': 3447.6287, 'train_samples_per_second': 76.283, 'train_steps_per_second': 1.192, 'total_flos': 8721133740933120.0, 'train_loss': 0.19071476609095162, 'epoch': 4.0})"]},"metadata":{},"execution_count":13}]},{"cell_type":"code","source":["trainer.evaluate()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":193},"id":"3M9QKQWufH1M","executionInfo":{"status":"ok","timestamp":1704979227055,"user_tz":-60,"elapsed":5237,"user":{"displayName":"Marcin Rostkowski","userId":"16749256502154511679"}},"outputId":"e3d28801-9ea6-4add-a560-1e33d348122f"},"execution_count":14,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n"," <div>\n"," \n"," <progress value='200' max='200' style='width:300px; height:20px; vertical-align: middle;'></progress>\n"," [200/200 00:04]\n"," </div>\n"," "]},"metadata":{}},{"output_type":"execute_result","data":{"text/plain":["{'eval_loss': 0.1992868036031723,\n"," 'eval_accuracy': 0.94125,\n"," 'eval_f1': 0.9467120181405895,\n"," 'eval_precision': 0.9413754227733935,\n"," 'eval_recall': 0.9521094640820981,\n"," 'eval_runtime': 4.8171,\n"," 'eval_samples_per_second': 332.147,\n"," 'eval_steps_per_second': 41.518,\n"," 'epoch': 4.0}"]},"metadata":{},"execution_count":14}]}]} |