{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# GPT2" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "pycharm": { "is_executing": true } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "02/16/2022 00:13:42 - INFO - __main__ - Distributed environment: NO\n", "Num processes: 1\n", "Process index: 0\n", "Local process index: 0\n", "Device: cpu\n", "Use FP16 precision: False\n", "\n", "02/16/2022 00:13:43 - WARNING - datasets.builder - Using custom data configuration default-67c9d932a627b7b8\n", "02/16/2022 00:13:43 - WARNING - datasets.builder - Reusing dataset json (C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-67c9d932a627b7b8\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426)\n", "\n", " 0%| | 0/3 [00:00\n", "02/16/2022 00:13:50 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-67c9d932a627b7b8\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426\\cache-18c6f53370629db4.arrow\n", "02/16/2022 00:13:50 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-67c9d932a627b7b8\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426\\cache-da48038acf63cb08.arrow\n", "\n", "Running tokenizer on dataset: 0%| | 0/1 [00:00\n", "02/16/2022 00:27:29 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-67c9d932a627b7b8\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426\\cache-ba0dca0006a47e01.arrow\n", "\n", "Running tokenizer on dataset: 0%| | 0/1 [00:00\n", "02/16/2022 00:33:08 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-67c9d932a627b7b8\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426\\cache-f4385b00908c069e.arrow\n", "\n", "Running tokenizer on dataset: 0%| | 0/1 [00:00\n", "02/16/2022 00:39:16 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-67c9d932a627b7b8\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426\\cache-5a65b7038a57b5cc.arrow\n", "\n", "Running tokenizer on dataset: 0%| | 0/1 [00:00\n", "\n", "Running tokenizer on dataset: 0%| | 0/5 [00:00> loading configuration file out/tweet/roberta_version_2\\config.json\n", "[INFO|configuration_utils.py:625] 2022-02-17 17:22:05,893 >> Model config RobertaConfig {\n", " \"_name_or_path\": \"roberta-base\",\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "02/17/2022 17:22:05 - INFO - __main__ - Training/evaluation parameters TrainingArguments(\n", "_n_gpu=0,\n", "adafactor=False,\n", "adam_beta1=0.9,\n", "adam_beta2=0.999,\n", "adam_epsilon=1e-08,\n", "dataloader_drop_last=False,\n", "dataloader_num_workers=0,\n", "dataloader_pin_memory=True,\n", "ddp_find_unused_parameters=None,\n", "debug=[],\n", "deepspeed=None,\n", "disable_tqdm=False,\n", "do_eval=True,\n", "do_predict=False,\n", "do_train=False,\n", "eval_accumulation_steps=None,\n", "eval_steps=None,\n", "evaluation_strategy=IntervalStrategy.NO,\n", "fp16=False,\n", "fp16_backend=auto,\n", "fp16_full_eval=False,\n", "fp16_opt_level=O1,\n", "gradient_accumulation_steps=1,\n", "gradient_checkpointing=False,\n", "greater_is_better=None,\n", "group_by_length=False,\n", "hub_model_id=None,\n", "hub_strategy=HubStrategy.EVERY_SAVE,\n", "hub_token=,\n", "ignore_data_skip=False,\n", "label_names=None,\n", "label_smoothing_factor=0.0,\n", "learning_rate=5e-05,\n", "length_column_name=length,\n", "load_best_model_at_end=False,\n", "local_rank=-1,\n", "log_level=-1,\n", "log_level_replica=-1,\n", "log_on_each_node=True,\n", "logging_dir=out/tweet/roberta_version_2-evaluation\\runs\\Feb17_17-22-05_DESKTOP-K706NKK,\n", "logging_first_step=False,\n", "logging_nan_inf_filter=True,\n", "logging_steps=500,\n", "logging_strategy=IntervalStrategy.STEPS,\n", "lr_scheduler_type=SchedulerType.LINEAR,\n", "max_grad_norm=1.0,\n", "max_steps=-1,\n", "metric_for_best_model=None,\n", "mp_parameters=,\n", "no_cuda=False,\n", "num_train_epochs=3.0,\n", "output_dir=out/tweet/roberta_version_2-evaluation,\n", "overwrite_output_dir=False,\n", "past_index=-1,\n", "per_device_eval_batch_size=24,\n", "per_device_train_batch_size=8,\n", "prediction_loss_only=False,\n", "push_to_hub=False,\n", "push_to_hub_model_id=None,\n", "push_to_hub_organization=None,\n", "push_to_hub_token=,\n", "remove_unused_columns=True,\n", "report_to=[],\n", "resume_from_checkpoint=None,\n", "run_name=out/tweet/roberta_version_2-evaluation,\n", "save_on_each_node=False,\n", "save_steps=500,\n", "save_strategy=IntervalStrategy.STEPS,\n", "save_total_limit=None,\n", "seed=42,\n", "sharded_ddp=[],\n", "skip_memory_metrics=True,\n", "tpu_metrics_debug=False,\n", "tpu_num_cores=None,\n", "use_legacy_prediction_loop=False,\n", "warmup_ratio=0.0,\n", "warmup_steps=0,\n", "weight_decay=0.0,\n", "xpu_backend=None,\n", ")\n", "02/17/2022 17:22:05 - INFO - __main__ - load a local file for train: data/train.json\n", "02/17/2022 17:22:05 - INFO - __main__ - load a local file for validation: data/valid.json\n", "02/17/2022 17:22:05 - WARNING - datasets.builder - Using custom data configuration default-f2672b914d9c5a33\n", "02/17/2022 17:22:05 - INFO - datasets.builder - Overwrite dataset info from restored data version.\n", "02/17/2022 17:22:05 - INFO - datasets.info - Loading Dataset info from C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-f2672b914d9c5a33\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426\n", "02/17/2022 17:22:05 - WARNING - datasets.builder - Reusing dataset json (C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-f2672b914d9c5a33\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426)\n", "02/17/2022 17:22:05 - INFO - datasets.info - Loading Dataset info from C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-f2672b914d9c5a33\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426\n", "02/17/2022 17:22:05 - INFO - __main__ - Return hidden states from model: True\n", "02/17/2022 17:22:05 - INFO - __main__ - Using implementation from: RobertaForSequenceClassificationCustomAlternative\n", "02/17/2022 17:22:07 - INFO - datasets.arrow_dataset - Caching processed dataset at C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-f2672b914d9c5a33\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426\\cache-d1d24efe1f314f1d.arrow\n", "02/17/2022 17:22:07 - INFO - datasets.arrow_dataset - Caching processed dataset at C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-f2672b914d9c5a33\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426\\cache-74073ef035f90484.arrow\n", "02/17/2022 17:22:08 - INFO - __main__ - *** Evaluate ***\n", "***** eval metrics *****\n", " eval_accuracy = 0.938\n", " eval_loss = 0.673\n", " eval_runtime = 0:00:46.31\n", " eval_samples = 500\n", " eval_samples_per_second = 10.795\n", " eval_steps_per_second = 0.453\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " \"architectures\": [\n", " \"RobertaForSequenceClassification\"\n", " ],\n", " \"attention_probs_dropout_prob\": 0.1,\n", " \"bos_token_id\": 0,\n", " \"classifier_dropout\": null,\n", " \"eos_token_id\": 2,\n", " \"hidden_act\": \"gelu\",\n", " \"hidden_dropout_prob\": 0.1,\n", " \"hidden_size\": 768,\n", " \"id2label\": {\n", " \"0\": 0,\n", " \"1\": 1\n", " },\n", " \"initializer_range\": 0.02,\n", " \"intermediate_size\": 3072,\n", " \"label2id\": {\n", " \"0\": 0,\n", " \"1\": 1\n", " },\n", " \"layer_norm_eps\": 1e-05,\n", " \"max_position_embeddings\": 514,\n", " \"model_type\": \"roberta\",\n", " \"num_attention_heads\": 12,\n", " \"num_hidden_layers\": 12,\n", " \"pad_token_id\": 1,\n", " \"position_embedding_type\": \"absolute\",\n", " \"problem_type\": \"single_label_classification\",\n", " \"torch_dtype\": \"float32\",\n", " \"transformers_version\": \"4.12.5\",\n", " \"type_vocab_size\": 1,\n", " \"use_cache\": true,\n", " \"vocab_size\": 50265\n", "}\n", "\n", "[INFO|tokenization_utils_base.py:1671] 2022-02-17 17:22:05,900 >> Didn't find file out/tweet/roberta_version_2\\added_tokens.json. We won't load it.\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-17 17:22:05,900 >> loading file out/tweet/roberta_version_2\\vocab.json\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-17 17:22:05,900 >> loading file out/tweet/roberta_version_2\\merges.txt\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-17 17:22:05,900 >> loading file out/tweet/roberta_version_2\\tokenizer.json\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-17 17:22:05,900 >> loading file None\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-17 17:22:05,900 >> loading file out/tweet/roberta_version_2\\special_tokens_map.json\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-17 17:22:05,900 >> loading file out/tweet/roberta_version_2\\tokenizer_config.json\n", "[INFO|modeling_utils.py:1349] 2022-02-17 17:22:05,959 >> loading weights file out/tweet/roberta_version_2\\pytorch_model.bin\n", "[WARNING|modeling_utils.py:1609] 2022-02-17 17:22:07,196 >> Some weights of the model checkpoint at out/tweet/roberta_version_2 were not used when initializing RobertaForSequenceClassificationCustomAlternative: ['classifier.dense.weight', 'classifier.dense.bias']\n", "- This IS expected if you are initializing RobertaForSequenceClassificationCustomAlternative from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", "- This IS NOT expected if you are initializing RobertaForSequenceClassificationCustomAlternative from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", "[WARNING|modeling_utils.py:1620] 2022-02-17 17:22:07,196 >> Some weights of RobertaForSequenceClassificationCustomAlternative were not initialized from the model checkpoint at out/tweet/roberta_version_2 and are newly initialized: ['classifier.dense_1_hidden.weight', 'classifier.dense_2.weight', 'classifier.dense_1_input.weight', 'classifier.dense_1_hidden.bias', 'classifier.dense_1_input.bias', 'classifier.dense_2.bias']\n", "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", "\n", "Running tokenizer on dataset: 0%| | 0/5 [00:00> The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassificationCustomAlternative.forward` and have been ignored: tweet.\n", "[INFO|trainer.py:2243] 2022-02-17 17:22:08,392 >> ***** Running Evaluation *****\n", "[INFO|trainer.py:2245] 2022-02-17 17:22:08,392 >> Num examples = 500\n", "[INFO|trainer.py:2248] 2022-02-17 17:22:08,392 >> Batch size = 24\n", "\n", " 0%| | 0/21 [00:00> Dropping the following result as it does not have all the necessary fields:\n", "{'task': {'name': 'Text Classification', 'type': 'text-classification'}}\n" ] } ], "source": [ "#valid\n", "!python run_glue.py \\\n", "--model_name_or_path out/tweet/roberta_version_2 \\\n", "--output_dir out/tweet/roberta_version_2-evaluation \\\n", "--return_hidden_states --custom_model \\\n", "--train_file data/train.json --validation_file data/valid.json \\\n", "--do_eval \\\n", "--per_device_eval_batch_size 24 --max_seq_length 128 \\\n", "--return_hidden_states --custom_model" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "02/16/2022 01:12:34 - WARNING - __main__ - Process rank: -1, device: cpu, n_gpu: 0distributed training: False, 16-bits training: False\n", "02/16/2022 01:12:34 - INFO - __main__ - Training/evaluation parameters TrainingArguments(\n", "_n_gpu=0,\n", "adafactor=False,\n", "adam_beta1=0.9,\n", "adam_beta2=0.999,\n", "adam_epsilon=1e-08,\n", "dataloader_drop_last=False,\n", "dataloader_num_workers=0,\n", "dataloader_pin_memory=True,\n", "ddp_find_unused_parameters=None," ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", " 0%| | 0/2 [00:00> loading configuration file out/tweet/roberta_version_4\\config.json\n", "[INFO|configuration_utils.py:625] 2022-02-16 01:12:34,776 >> Model config RobertaConfig {\n", " \"_name_or_path\": \"roberta-base\",\n", " \"architectures\": [\n", " \"RobertaForSequenceClassificationCustomAlternative\"\n", " ],\n", " \"attention_probs_dropout_prob\": 0.1,\n", " \"bos_token_id\": 0,\n", " \"classifier_dropout\": null,\n", " \"eos_token_id\": 2,\n", " \"hidden_act\": \"gelu\",\n", " \"hidden_dropout_prob\": 0.1,\n", " \"hidden_size\": 768,\n", " \"id2label\": {\n", " \"0\": 0,\n", " \"1\": 1\n", " },\n", " \"initializer_range\": 0.02,\n", " \"intermediate_size\": 3072,\n", " \"label2id\": {\n", " \"0\": 0,\n", " \"1\": 1\n", " },\n", " \"layer_norm_eps\": 1e-05,\n", " \"max_position_embeddings\": 514,\n", " \"model_type\": \"roberta\",\n", " \"num_attention_heads\": 12,\n", " \"num_hidden_layers\": 12,\n", " \"pad_token_id\": 1,\n", " \"position_embedding_type\": \"absolute\",\n", " \"problem_type\": \"single_label_classification\",\n", " \"torch_dtype\": \"float32\",\n", " \"transformers_version\": \"4.12.5\",\n", " \"type_vocab_size\": 1,\n", " \"use_cache\": true,\n", " \"vocab_size\": 50265\n", "}\n", "\n", "[INFO|tokenization_utils_base.py:1671] 2022-02-16 01:12:34,779 >> Didn't find file out/tweet/roberta_version_4\\added_tokens.json. We won't load it.\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-16 01:12:34,779 >> loading file out/tweet/roberta_version_4\\vocab.json\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-16 01:12:34,779 >> loading file out/tweet/roberta_version_4\\merges.txt\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-16 01:12:34,779 >> loading file out/tweet/roberta_version_4\\tokenizer.json\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-16 01:12:34,779 >> loading file None\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-16 01:12:34,779 >> loading file out/tweet/roberta_version_4\\special_tokens_map.json\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-16 01:12:34,780 >> loading file out/tweet/roberta_version_4\\tokenizer_config.json\n", "[INFO|modeling_utils.py:1349] 2022-02-16 01:12:34,829 >> loading weights file out/tweet/roberta_version_4\\pytorch_model.bin\n", "[INFO|modeling_utils.py:1618] 2022-02-16 01:12:35,990 >> All model checkpoint weights were used when initializing RobertaForSequenceClassificationCustomAlternative.\n", "\n", "[INFO|modeling_utils.py:1626] 2022-02-16 01:12:35,990 >> All the weights of RobertaForSequenceClassificationCustomAlternative were initialized from the model checkpoint at out/tweet/roberta_version_4.\n", "If your task is similar to the task the model of the checkpoint was trained on, you can already use RobertaForSequenceClassificationCustomAlternative for predictions without further training.\n", "\n", "Running tokenizer on dataset: 0%| | 0/1 [00:00> The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassificationCustomAlternative.forward` and have been ignored: tweet.\n", "[INFO|trainer.py:2243] 2022-02-16 01:12:36,823 >> ***** Running Evaluation *****\n", "[INFO|trainer.py:2245] 2022-02-16 01:12:36,824 >> Num examples = 500\n", "[INFO|trainer.py:2248] 2022-02-16 01:12:36,824 >> Batch size = 24\n", "\n", " 0%| | 0/21 [00:00,\n", "ignore_data_skip=False,\n", "label_names=None,\n", "label_smoothing_factor=0.0,\n", "learning_rate=5e-05,\n", "length_column_name=length,\n", "load_best_model_at_end=False,\n", "local_rank=-1,\n", "log_level=-1,\n", "log_level_replica=-1,\n", "log_on_each_node=True,\n", "logging_dir=out/tweet/roberta_version_4-evaluation\\runs\\Feb16_01-12-34_DESKTOP-K706NKK,\n", "logging_first_step=False,\n", "logging_nan_inf_filter=True,\n", "logging_steps=500,\n", "logging_strategy=IntervalStrategy.STEPS,\n", "lr_scheduler_type=SchedulerType.LINEAR,\n", "max_grad_norm=1.0,\n", "max_steps=-1,\n", "metric_for_best_model=None,\n", "mp_parameters=,\n", "no_cuda=False,\n", "num_train_epochs=3.0,\n", "output_dir=out/tweet/roberta_version_4-evaluation,\n", "overwrite_output_dir=False,\n", "past_index=-1,\n", "per_device_eval_batch_size=24,\n", "per_device_train_batch_size=8,\n", "prediction_loss_only=False,\n", "push_to_hub=False,\n", "push_to_hub_model_id=None,\n", "push_to_hub_organization=None,\n", "push_to_hub_token=,\n", "remove_unused_columns=True,\n", "report_to=[],\n", "resume_from_checkpoint=None,\n", "run_name=out/tweet/roberta_version_4-evaluation,\n", "save_on_each_node=False,\n", "save_steps=500,\n", "save_strategy=IntervalStrategy.STEPS,\n", "save_total_limit=None,\n", "seed=42,\n", "sharded_ddp=[],\n", "skip_memory_metrics=True,\n", "tpu_metrics_debug=False,\n", "tpu_num_cores=None,\n", "use_legacy_prediction_loop=False,\n", "warmup_ratio=0.0,\n", "warmup_steps=0,\n", "weight_decay=0.0,\n", "xpu_backend=None,\n", ")\n", "02/16/2022 01:12:34 - INFO - __main__ - load a local file for train: data/train.json\n", "02/16/2022 01:12:34 - INFO - __main__ - load a local file for validation: data/test.json\n", "02/16/2022 01:12:34 - WARNING - datasets.builder - Using custom data configuration default-aa408910693fa782\n", "02/16/2022 01:12:34 - INFO - datasets.builder - Overwrite dataset info from restored data version.\n", "02/16/2022 01:12:34 - INFO - datasets.info - Loading Dataset info from C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-aa408910693fa782\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426\n", "02/16/2022 01:12:34 - WARNING - datasets.builder - Reusing dataset json (C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-aa408910693fa782\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426)\n", "02/16/2022 01:12:34 - INFO - datasets.info - Loading Dataset info from C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-aa408910693fa782\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426\n", "02/16/2022 01:12:34 - INFO - __main__ - Return hidden states from model: True\n", "02/16/2022 01:12:34 - INFO - __main__ - Using implementation from: RobertaForSequenceClassificationCustomAlternative\n", "02/16/2022 01:12:36 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-aa408910693fa782\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426\\cache-7c7dda0a4623bcbe.arrow\n", "02/16/2022 01:12:36 - INFO - datasets.arrow_dataset - Caching processed dataset at C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-aa408910693fa782\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426\\cache-eec123a569b1837d.arrow\n", "02/16/2022 01:12:36 - INFO - __main__ - *** Evaluate ***\n", "***** eval metrics *****\n", " eval_accuracy = 1.0\n", " eval_loss = 0.6472\n", " eval_runtime = 0:00:45.49\n", " eval_samples = 500\n", " eval_samples_per_second = 10.991\n", " eval_steps_per_second = 0.462\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", " 48%|####7 | 10/21 [00:19<00:23, 2.17s/it]\n", " 52%|#####2 | 11/21 [00:21<00:21, 2.16s/it]\n", " 57%|#####7 | 12/21 [00:24<00:19, 2.18s/it]\n", " 62%|######1 | 13/21 [00:26<00:17, 2.18s/it]\n", " 67%|######6 | 14/21 [00:28<00:15, 2.18s/it]\n", " 71%|#######1 | 15/21 [00:30<00:13, 2.17s/it]\n", " 76%|#######6 | 16/21 [00:32<00:10, 2.18s/it]\n", " 81%|######## | 17/21 [00:34<00:08, 2.19s/it]\n", " 86%|########5 | 18/21 [00:37<00:06, 2.19s/it]\n", " 90%|######### | 19/21 [00:39<00:04, 2.19s/it]\n", " 95%|#########5| 20/21 [00:41<00:02, 2.17s/it]\n", "100%|##########| 21/21 [00:43<00:00, 2.06s/it]\n", "100%|##########| 21/21 [00:43<00:00, 2.06s/it]\n", "[INFO|modelcard.py:449] 2022-02-16 01:13:22,843 >> Dropping the following result as it does not have all the necessary fields:\n", "{'task': {'name': 'Text Classification', 'type': 'text-classification'}}\n" ] } ], "source": [ "#test\n", "!python run_glue.py \\\n", "--model_name_or_path out/tweet/roberta_version_4 \\\n", "--output_dir out/tweet/roberta_version_4-evaluation \\\n", "--return_hidden_states --custom_model \\\n", "--train_file data/train.json --validation_file data/test.json \\\n", "--do_eval \\\n", "--per_device_eval_batch_size 24 --max_seq_length 128 \\\n", "--return_hidden_states --custom_model" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# GPT2" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "02/17/2022 17:25:29 - WARNING - __main__ - Process rank: -1, device: cpu, n_gpu: 0distributed training: False, 16-bits training: False\n", "02/17/2022 17:25:29 - INFO - __main__ - Training/evaluation parameters TrainingArguments(\n", "_n_gpu=0,\n", "adafactor=False,\n", "adam_beta1=0.9,\n", "adam_beta2=0.999,\n", "adam_epsilon=1e-08,\n", "dataloader_drop_last=False,\n", "dataloader_num_workers=0,\n", "dataloader_pin_memory=True,\n", "ddp_find_unused_parameters=None,\n", "debug=[],\n", "deepspeed=None,\n", "disable_tqdm=False,\n", "do_eval=True,\n", "do_predict=False,\n", "do_train=False,\n", "eval_accumulation_steps=None,\n", "eval_steps=None,\n", "evaluation_strategy=IntervalStrategy.NO,\n", "fp16=False,\n", "fp16_backend=auto,\n", "fp16_full_eval=False,\n", "fp16_opt_level=O1,\n", "gradient_accumulation_steps=1,\n", "gradient_checkpointing=False,\n", "greater_is_better=None,\n", "group_by_length=False,\n", "hub_model_id=None,\n", "hub_strategy=HubStrategy.EVERY_SAVE,\n", "hub_token=,\n", "ignore_data_skip=False,\n", "label_names=None,\n", "label_smoothing_factor=0.0,\n", "learning_rate=5e-05,\n", "length_column_name=length,\n", "load_best_model_at_end=False,\n", "local_rank=-1,\n", "log_level=-1,\n", "log_level_replica=-1,\n", "log_on_each_node=True,\n", "logging_dir=out/tweet/gpt2_version_2-evaluation\\runs\\Feb17_17-25-29_DESKTOP-K706NKK,\n", "logging_first_step=False,\n", "logging_nan_inf_filter=True,\n", "logging_steps=500,\n", "logging_strategy=IntervalStrategy.STEPS,\n", "lr_scheduler_type=SchedulerType.LINEAR,\n", "max_grad_norm=1.0,\n", "max_steps=-1,\n", "metric_for_best_model=None,\n", "mp_parameters=,\n", "no_cuda=False,\n", "num_train_epochs=3.0,\n", "output_dir=out/tweet/gpt2_version_2-evaluation,\n", "overwrite_output_dir=False,\n", "past_index=-1,\n", "per_device_eval_batch_size=24,\n", "per_device_train_batch_size=8,\n", "prediction_loss_only=False,\n", "push_to_hub=False,\n", "push_to_hub_model_id=None,\n", "push_to_hub_organization=None,\n", "push_to_hub_token=,\n", "remove_unused_columns=True,\n", "report_to=[],\n", "resume_from_checkpoint=None,\n", "run_name=out/tweet/gpt2_version_2-evaluation,\n", "save_on_each_node=False,\n", "save_steps=500,\n", "save_strategy=IntervalStrategy.STEPS,\n", "save_total_limit=None,\n", "seed=42,\n", "sharded_ddp=[],\n", "skip_memory_metrics=True,\n", "tpu_metrics_debug=False,\n", "tpu_num_cores=None,\n", "use_legacy_prediction_loop=False,\n", "warmup_ratio=0.0,\n", "warmup_steps=0,\n", "weight_decay=0.0,\n", "xpu_backend=None,\n", ")\n", "02/17/2022 17:25:29 - INFO - __main__ - load a local file for train: data/train.json\n", "02/17/2022 17:25:29 - INFO - __main__ - load a local file for validation: data/valid.json\n", "02/17/2022 17:25:29 - WARNING - datasets.builder - Using custom data configuration default-f2672b914d9c5a33\n", "02/17/2022 17:25:29 - INFO - datasets.builder - Overwrite dataset info from restored data version.\n", "02/17/2022 17:25:29 - INFO - datasets.info - Loading Dataset info from C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-f2672b914d9c5a33\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426\n", "02/17/2022 17:25:29 - WARNING - datasets.builder - Reusing dataset json (C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-f2672b914d9c5a33\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426)\n", "02/17/2022 17:25:29 - INFO - datasets.info - Loading Dataset info from C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-f2672b914d9c5a33\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426\n", "02/17/2022 17:25:29 - INFO - __main__ - Return hidden states from model: True\n", "02/17/2022 17:25:29 - INFO - __main__ - Using implementation from: GPT2ForSequenceClassificationCustom\n", "02/17/2022 17:25:31 - INFO - datasets.arrow_dataset - Caching processed dataset at C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-f2672b914d9c5a33\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426\\cache-212f78cac2ca92a1.arrow\n", "02/17/2022 17:25:31 - INFO - datasets.arrow_dataset - Caching processed dataset at C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-f2672b914d9c5a33\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426\\cache-95c22eb06b0faad8.arrow\n", "02/17/2022 17:25:32 - INFO - __main__ - *** Evaluate ***\n", "***** eval metrics *****\n", " eval_accuracy = 0.938\n", " eval_loss = 0.4886\n", " eval_runtime = 0:01:01.53\n", " eval_samples = 500\n", " eval_samples_per_second = 8.126\n", " eval_steps_per_second = 0.341\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", " 0%| | 0/2 [00:00> loading configuration file out/tweet/gpt2_version_2\\config.json\n", "[INFO|configuration_utils.py:625] 2022-02-17 17:25:29,864 >> Model config GPT2Config {\n", " \"_name_or_path\": \"gpt2\",\n", " \"activation_function\": \"gelu_new\",\n", " \"architectures\": [\n", " \"GPT2ForSequenceClassification\"\n", " ],\n", " \"attn_pdrop\": 0.1,\n", " \"bos_token_id\": 50256,\n", " \"embd_pdrop\": 0.1,\n", " \"eos_token_id\": 50256,\n", " \"id2label\": {\n", " \"0\": 0,\n", " \"1\": 1\n", " },\n", " \"initializer_range\": 0.02,\n", " \"label2id\": {\n", " \"0\": 0,\n", " \"1\": 1\n", " },\n", " \"layer_norm_epsilon\": 1e-05,\n", " \"model_type\": \"gpt2\",\n", " \"n_ctx\": 1024,\n", " \"n_embd\": 768,\n", " \"n_head\": 12,\n", " \"n_inner\": null,\n", " \"n_layer\": 12,\n", " \"n_positions\": 1024,\n", " \"pad_token_id\": 50256,\n", " \"reorder_and_upcast_attn\": false,\n", " \"resid_pdrop\": 0.1,\n", " \"scale_attn_by_inverse_layer_idx\": false,\n", " \"scale_attn_weights\": true,\n", " \"summary_activation\": null,\n", " \"summary_first_dropout\": 0.1,\n", " \"summary_proj_to_labels\": true,\n", " \"summary_type\": \"cls_index\",\n", " \"summary_use_proj\": true,\n", " \"task_specific_params\": {\n", " \"text-generation\": {\n", " \"do_sample\": true,\n", " \"max_length\": 50\n", " }\n", " },\n", " \"torch_dtype\": \"float32\",\n", " \"transformers_version\": \"4.12.5\",\n", " \"use_cache\": true,\n", " \"vocab_size\": 50257\n", "}\n", "\n", "[INFO|tokenization_utils_base.py:1671] 2022-02-17 17:25:29,868 >> Didn't find file out/tweet/gpt2_version_2\\added_tokens.json. We won't load it.\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-17 17:25:29,869 >> loading file out/tweet/gpt2_version_2\\vocab.json\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-17 17:25:29,869 >> loading file out/tweet/gpt2_version_2\\merges.txt\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-17 17:25:29,869 >> loading file out/tweet/gpt2_version_2\\tokenizer.json\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-17 17:25:29,869 >> loading file None\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-17 17:25:29,869 >> loading file out/tweet/gpt2_version_2\\special_tokens_map.json\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-17 17:25:29,869 >> loading file out/tweet/gpt2_version_2\\tokenizer_config.json\n", "[INFO|modeling_utils.py:1349] 2022-02-17 17:25:29,927 >> loading weights file out/tweet/gpt2_version_2\\pytorch_model.bin\n", "[WARNING|modeling_utils.py:1609] 2022-02-17 17:25:31,677 >> Some weights of the model checkpoint at out/tweet/gpt2_version_2 were not used when initializing GPT2ForSequenceClassificationCustom: ['score.weight']\n", "- This IS expected if you are initializing GPT2ForSequenceClassificationCustom from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", "- This IS NOT expected if you are initializing GPT2ForSequenceClassificationCustom from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", "[WARNING|modeling_utils.py:1620] 2022-02-17 17:25:31,677 >> Some weights of GPT2ForSequenceClassificationCustom were not initialized from the model checkpoint at out/tweet/gpt2_version_2 and are newly initialized: ['score.out_proj.weight', 'score.dense_1_input.bias', 'score.dense_1_hidden.bias', 'score.dense_2.weight', 'score.dense_2.bias', 'score.dense_1_hidden.weight', 'score.dense_1_input.weight']\n", "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", "\n", "Running tokenizer on dataset: 0%| | 0/5 [00:00> The following columns in the evaluation set don't have a corresponding argument in `GPT2ForSequenceClassificationCustom.forward` and have been ignored: tweet.\n", "[INFO|trainer.py:2243] 2022-02-17 17:25:32,737 >> ***** Running Evaluation *****\n", "[INFO|trainer.py:2245] 2022-02-17 17:25:32,737 >> Num examples = 500\n", "[INFO|trainer.py:2248] 2022-02-17 17:25:32,737 >> Batch size = 24\n", "\n", " 0%| | 0/21 [00:00> Dropping the following result as it does not have all the necessary fields:\n", "{'task': {'name': 'Text Classification', 'type': 'text-classification'}}\n" ] } ], "source": [ "#valid\n", "!python run_glue.py \\\n", "--model_name_or_path out/tweet/gpt2_version_2 \\\n", "--output_dir out/tweet/gpt2_version_2-evaluation \\\n", "--return_hidden_states --custom_model \\\n", "--train_file data/train.json --validation_file data/valid.json \\\n", "--do_eval \\\n", "--per_device_eval_batch_size 24 --max_seq_length 128 \\\n", "--return_hidden_states --custom_model" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# T5" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "02/17/2022 17:36:52 - WARNING - __main__ - Process rank: -1, device: cpu, n_gpu: 0distributed training: False, 16-bits training: False\n", "02/17/2022 17:36:52 - INFO - __main__ - Training/evaluation parameters Seq2SeqTrainingArguments(\n", "_n_gpu=0,\n", "adafactor=False,\n", "adam_beta1=0.9,\n", "adam_beta2=0.999,\n", "adam_epsilon=1e-08,\n", "dataloader_drop_last=False,\n", "dataloader_num_workers=0,\n", "dataloader_pin_memory=True,\n", "ddp_find_unused_parameters=None,\n", "debug=[],\n", "deepspeed=None,\n", "disable_tqdm=False,\n", "do_eval=True,\n", "do_predict=False,\n", "do_train=False,\n", "eval_accumulation_steps=None,\n", "eval_steps=None,\n", "evaluation_strategy=IntervalStrategy.NO,\n", "fp16=False,\n", "fp16_backend=auto,\n", "fp16_full_eval=False,\n", "fp16_opt_level=O1,\n", "generation_max_length=None,\n", "generation_num_beams=None,\n", "gradient_accumulation_steps=1,\n", "gradient_checkpointing=False,\n", "greater_is_better=None,\n", "group_by_length=False,\n", "hub_model_id=None,\n", "hub_strategy=HubStrategy.EVERY_SAVE,\n", "hub_token=,\n", "ignore_data_skip=False,\n", "label_names=None,\n", "label_smoothing_factor=0.0,\n", "learning_rate=5e-05,\n", "length_column_name=length,\n", "load_best_model_at_end=False,\n", "local_rank=-1,\n", "log_level=-1,\n", "log_level_replica=-1,\n", "log_on_each_node=True,\n", "logging_dir=out/tweet/t5-evaluation\\runs\\Feb17_17-36-52_DESKTOP-K706NKK,\n", "logging_first_step=False,\n", "logging_nan_inf_filter=True,\n", "logging_steps=500,\n", "logging_strategy=IntervalStrategy.STEPS,\n", "lr_scheduler_type=SchedulerType.LINEAR,\n", "max_grad_norm=1.0,\n", "max_steps=-1,\n", "metric_for_best_model=None,\n", "mp_parameters=,\n", "no_cuda=False,\n", "num_train_epochs=3.0,\n", "output_dir=out/tweet/t5-evaluation,\n", "overwrite_output_dir=False,\n", "past_index=-1,\n", "per_device_eval_batch_size=16,\n", "per_device_train_batch_size=8,\n", "predict_with_generate=True,\n", "prediction_loss_only=False,\n", "push_to_hub=False,\n", "push_to_hub_model_id=None,\n", "push_to_hub_organization=None,\n", "push_to_hub_token=,\n", "remove_unused_columns=True,\n", "report_to=[],\n", "resume_from_checkpoint=None,\n", "run_name=out/tweet/t5-evaluation,\n", "save_on_each_node=False,\n", "save_steps=500,\n", "save_strategy=IntervalStrategy.STEPS,\n", "save_total_limit=None,\n", "seed=42,\n", "sharded_ddp=[],\n", "skip_memory_metrics=True,\n", "sortish_sampler=False,\n", "tpu_metrics_debug=False,\n", "tpu_num_cores=None,\n", "use_legacy_prediction_loop=False,\n", "warmup_ratio=0.0,\n", "warmup_steps=0,\n", "weight_decay=0.0,\n", "xpu_backend=None,\n", ")\n", "02/17/2022 17:36:52 - WARNING - datasets.builder - Using custom data configuration default-6d5bc754bbaa91d7\n", "02/17/2022 17:36:52 - INFO - datasets.builder - Overwrite dataset info from restored data version.\n", "02/17/2022 17:36:52 - INFO - datasets.info - Loading Dataset info from C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-6d5bc754bbaa91d7\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426\n", "02/17/2022 17:36:52 - WARNING - datasets.builder - Reusing dataset json (C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-6d5bc754bbaa91d7\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426)\n", "02/17/2022 17:36:52 - INFO - datasets.info - Loading Dataset info from C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-6d5bc754bbaa91d7\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426\n", "02/17/2022 17:36:53 - INFO - __main__ - Using translation prefix: \"tweet classification: \"\n", "02/17/2022 17:36:53 - INFO - datasets.arrow_dataset - Caching processed dataset at C:\\Users\\Foka\\.cache\\huggingface\\datasets\\json\\default-6d5bc754bbaa91d7\\0.0.0\\c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426\\cache-96f3d337ad66e082.arrow\n", "02/17/2022 17:36:55 - INFO - __main__ - *** Evaluate ***\n", "02/17/2022 17:37:09 - INFO - datasets.metric - Removing C:\\Users\\Foka\\.cache\\huggingface\\metrics\\accuracy\\default\\default_experiment-1-0.arrow\n", "02/17/2022 17:37:09 - INFO - datasets.metric - Removing C:\\Users\\Foka\\.cache\\huggingface\\metrics\\sacrebleu\\default\\default_experiment-1-0.arrow\n", "***** eval metrics *****\n", " eval_accuracy = 1.0\n", " eval_bleu = 0.0\n", " eval_gen_len = 2.272\n", " eval_loss = 0.5538\n", " eval_runtime = 0:00:14.42\n", " eval_samples = 500\n", " eval_samples_per_second = 34.659\n", " eval_steps_per_second = 2.218\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", " 0%| | 0/2 [00:00> loading configuration file out/tweet/t5_version_2\\config.json\n", "[INFO|configuration_utils.py:625] 2022-02-17 17:36:52,677 >> Model config T5Config {\n", " \"_name_or_path\": \"t5-small\",\n", " \"architectures\": [\n", " \"T5ForConditionalGeneration\"\n", " ],\n", " \"d_ff\": 2048,\n", " \"d_kv\": 64,\n", " \"d_model\": 512,\n", " \"decoder_start_token_id\": 0,\n", " \"dropout_rate\": 0.1,\n", " \"eos_token_id\": 1,\n", " \"feed_forward_proj\": \"relu\",\n", " \"initializer_factor\": 1.0,\n", " \"is_encoder_decoder\": true,\n", " \"layer_norm_epsilon\": 1e-06,\n", " \"model_type\": \"t5\",\n", " \"n_positions\": 512,\n", " \"num_decoder_layers\": 6,\n", " \"num_heads\": 8,\n", " \"num_layers\": 6,\n", " \"output_past\": true,\n", " \"pad_token_id\": 0,\n", " \"relative_attention_num_buckets\": 32,\n", " \"task_specific_params\": {\n", " \"summarization\": {\n", " \"early_stopping\": true,\n", " \"length_penalty\": 2.0,\n", " \"max_length\": 200,\n", " \"min_length\": 30,\n", " \"no_repeat_ngram_size\": 3,\n", " \"num_beams\": 4,\n", " \"prefix\": \"summarize: \"\n", " },\n", " \"translation_en_to_de\": {\n", " \"early_stopping\": true,\n", " \"max_length\": 300,\n", " \"num_beams\": 4,\n", " \"prefix\": \"translate English to German: \"\n", " },\n", " \"translation_en_to_fr\": {\n", " \"early_stopping\": true,\n", " \"max_length\": 300,\n", " \"num_beams\": 4,\n", " \"prefix\": \"translate English to French: \"\n", " },\n", " \"translation_en_to_ro\": {\n", " \"early_stopping\": true,\n", " \"max_length\": 300,\n", " \"num_beams\": 4,\n", " \"prefix\": \"translate English to Romanian: \"\n", " }\n", " },\n", " \"torch_dtype\": \"float32\",\n", " \"transformers_version\": \"4.12.5\",\n", " \"use_cache\": true,\n", " \"vocab_size\": 32100\n", "}\n", "\n", "[INFO|tokenization_utils_base.py:1671] 2022-02-17 17:36:52,677 >> Didn't find file out/tweet/t5_version_2\\added_tokens.json. We won't load it.\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-17 17:36:52,677 >> loading file out/tweet/t5_version_2\\spiece.model\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-17 17:36:52,677 >> loading file out/tweet/t5_version_2\\tokenizer.json\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-17 17:36:52,677 >> loading file None\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-17 17:36:52,677 >> loading file out/tweet/t5_version_2\\special_tokens_map.json\n", "[INFO|tokenization_utils_base.py:1740] 2022-02-17 17:36:52,677 >> loading file out/tweet/t5_version_2\\tokenizer_config.json\n", "[INFO|modeling_utils.py:1349] 2022-02-17 17:36:52,771 >> loading weights file out/tweet/t5_version_2\\pytorch_model.bin\n", "[INFO|modeling_utils.py:1618] 2022-02-17 17:36:53,190 >> All model checkpoint weights were used when initializing T5ForConditionalGeneration.\n", "\n", "[INFO|modeling_utils.py:1626] 2022-02-17 17:36:53,190 >> All the weights of T5ForConditionalGeneration were initialized from the model checkpoint at out/tweet/t5_version_2.\n", "If your task is similar to the task the model of the checkpoint was trained on, you can already use T5ForConditionalGeneration for predictions without further training.\n", "\n", "Running tokenizer on validation dataset: 0%| | 0/1 [00:00> ***** Running Evaluation *****\n", "[INFO|trainer.py:2245] 2022-02-17 17:36:55,016 >> Num examples = 500\n", "[INFO|trainer.py:2248] 2022-02-17 17:36:55,016 >> Batch size = 16\n", "\n", " 0%| | 0/32 [00:00> Dropping the following result as it does not have all the necessary fields:\n", "{'task': {'name': 'Translation', 'type': 'translation'}}\n" ] } ], "source": [ "#train and test\n", "!python run_translation.py \\\n", "--model_name_or_path out/tweet/t5_version_2 \\\n", "--output_dir out/tweet/t5-evaluation \\\n", "--train_file data/translations-train.json \\\n", "--validation_file data/translations-test.json \\\n", "--do_eval \\\n", "--per_device_eval_batch_size 16 \\\n", "--source_lang text \\\n", "--target_lang label \\\n", "--source_prefix \"tweet classification\" \\\n", "--max_source_length 256 \\\n", "--max_target_length 128 \\\n", "--predict_with_generate" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.9" } }, "nbformat": 4, "nbformat_minor": 1 }