DL_RNN/RNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RNN\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Installation of packages\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: torch in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (2.3.0)\n",
      "Requirement already satisfied: filelock in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from torch) (3.14.0)\n",
      "Requirement already satisfied: typing-extensions>=4.8.0 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from torch) (4.10.0)\n",
      "Requirement already satisfied: sympy in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from torch) (1.12)\n",
      "Requirement already satisfied: networkx in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from torch) (3.2.1)\n",
      "Requirement already satisfied: jinja2 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from torch) (3.1.3)\n",
      "Requirement already satisfied: fsspec in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from torch) (2024.3.1)\n",
      "Requirement already satisfied: mkl<=2021.4.0,>=2021.1.1 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from torch) (2021.4.0)\n",
      "Requirement already satisfied: intel-openmp==2021.* in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from mkl<=2021.4.0,>=2021.1.1->torch) (2021.4.0)\n",
      "Requirement already satisfied: tbb==2021.* in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from mkl<=2021.4.0,>=2021.1.1->torch) (2021.12.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from jinja2->torch) (2.1.5)\n",
      "Requirement already satisfied: mpmath>=0.19 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from sympy->torch) (1.3.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Requirement already satisfied: torchtext in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (0.18.0)\n",
      "Requirement already satisfied: tqdm in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from torchtext) (4.66.2)\n",
      "Requirement already satisfied: requests in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from torchtext) (2.31.0)\n",
      "Requirement already satisfied: torch>=2.3.0 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from torchtext) (2.3.0)\n",
      "Requirement already satisfied: numpy in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from torchtext) (1.26.3)\n",
      "Requirement already satisfied: filelock in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from torch>=2.3.0->torchtext) (3.14.0)\n",
      "Requirement already satisfied: typing-extensions>=4.8.0 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from torch>=2.3.0->torchtext) (4.10.0)\n",
      "Requirement already satisfied: sympy in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from torch>=2.3.0->torchtext) (1.12)\n",
      "Requirement already satisfied: networkx in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from torch>=2.3.0->torchtext) (3.2.1)\n",
      "Requirement already satisfied: jinja2 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from torch>=2.3.0->torchtext) (3.1.3)\n",
      "Requirement already satisfied: fsspec in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from torch>=2.3.0->torchtext) (2024.3.1)\n",
      "Requirement already satisfied: mkl<=2021.4.0,>=2021.1.1 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from torch>=2.3.0->torchtext) (2021.4.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from requests->torchtext) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from requests->torchtext) (3.6)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from requests->torchtext) (2.2.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from requests->torchtext) (2024.2.2)\n",
      "Requirement already satisfied: colorama in c:\\users\\skype\\appdata\\roaming\\python\\python312\\site-packages (from tqdm->torchtext) (0.4.6)\n",
      "Requirement already satisfied: intel-openmp==2021.* in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from mkl<=2021.4.0,>=2021.1.1->torch>=2.3.0->torchtext) (2021.4.0)\n",
      "Requirement already satisfied: tbb==2021.* in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from mkl<=2021.4.0,>=2021.1.1->torch>=2.3.0->torchtext) (2021.12.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from jinja2->torch>=2.3.0->torchtext) (2.1.5)\n",
      "Requirement already satisfied: mpmath>=0.19 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from sympy->torch>=2.3.0->torchtext) (1.3.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Requirement already satisfied: datasets in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (2.19.1)\n",
      "Requirement already satisfied: filelock in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from datasets) (3.14.0)\n",
      "Requirement already satisfied: numpy>=1.17 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from datasets) (1.26.3)\n",
      "Requirement already satisfied: pyarrow>=12.0.0 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from datasets) (15.0.2)\n",
      "Requirement already satisfied: pyarrow-hotfix in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from datasets) (0.6)\n",
      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from datasets) (0.3.8)\n",
      "Requirement already satisfied: pandas in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from datasets) (2.2.1)\n",
      "Requirement already satisfied: requests>=2.19.0 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from datasets) (2.31.0)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from datasets) (4.66.2)\n",
      "Requirement already satisfied: xxhash in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from datasets) (3.4.1)\n",
      "Requirement already satisfied: multiprocess in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from datasets) (0.70.16)\n",
      "Requirement already satisfied: fsspec<=2024.3.1,>=2023.1.0 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from fsspec[http]<=2024.3.1,>=2023.1.0->datasets) (2024.3.1)\n",
      "Requirement already satisfied: aiohttp in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from datasets) (3.9.5)\n",
      "Requirement already satisfied: huggingface-hub>=0.21.2 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from datasets) (0.23.1)\n",
      "Requirement already satisfied: packaging in c:\\users\\skype\\appdata\\roaming\\python\\python312\\site-packages (from datasets) (23.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from datasets) (6.0.1)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from aiohttp->datasets) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from aiohttp->datasets) (23.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from aiohttp->datasets) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from aiohttp->datasets) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from aiohttp->datasets) (1.9.4)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from huggingface-hub>=0.21.2->datasets) (4.10.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from requests>=2.19.0->datasets) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from requests>=2.19.0->datasets) (3.6)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from requests>=2.19.0->datasets) (2.2.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from requests>=2.19.0->datasets) (2024.2.2)\n",
      "Requirement already satisfied: colorama in c:\\users\\skype\\appdata\\roaming\\python\\python312\\site-packages (from tqdm>=4.62.1->datasets) (0.4.6)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in c:\\users\\skype\\appdata\\roaming\\python\\python312\\site-packages (from pandas->datasets) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from pandas->datasets) (2024.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from pandas->datasets) (2024.1)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\skype\\appdata\\roaming\\python\\python312\\site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Requirement already satisfied: pandas in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (2.2.1)\n",
      "Requirement already satisfied: numpy<2,>=1.26.0 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from pandas) (1.26.3)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in c:\\users\\skype\\appdata\\roaming\\python\\python312\\site-packages (from pandas) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from pandas) (2024.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from pandas) (2024.1)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\skype\\appdata\\roaming\\python\\python312\\site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Requirement already satisfied: scikit-learn in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (1.4.1.post1)\n",
      "Requirement already satisfied: numpy<2.0,>=1.19.5 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from scikit-learn) (1.26.3)\n",
      "Requirement already satisfied: scipy>=1.6.0 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from scikit-learn) (1.12.0)\n",
      "Requirement already satisfied: joblib>=1.2.0 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from scikit-learn) (1.3.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\users\\skype\\appdata\\local\\programs\\python\\python312\\lib\\site-packages (from scikit-learn) (3.3.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install torch\n",
    "%pip install torchtext\n",
    "%pip install datasets\n",
    "%pip install pandas\n",
    "%pip install scikit-learn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importing libraries\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 377,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "import torch\n",
    "import pandas as pd\n",
    "from torchtext.vocab import vocab\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read datasets\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 378,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_data():\n",
    "    train_dataset = pd.read_csv(\n",
    "        \"train/train.tsv.xz\", compression=\"xz\", sep=\"\\t\", names=[\"Label\", \"Text\"]\n",
    "    )\n",
    "    dev_0_dataset = pd.read_csv(\"dev-0/in.tsv\", sep=\"\\t\", names=[\"Text\"])\n",
    "    dev_0_labels = pd.read_csv(\"dev-0/expected.tsv\", sep=\"\\t\", names=[\"Label\"])\n",
    "    test_A_dataset = pd.read_csv(\"test-A/in.tsv\", sep=\"\\t\", names=[\"Text\"])\n",
    "\n",
    "    return train_dataset, dev_0_dataset, dev_0_labels, test_A_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 379,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset, dev_0_dataset, dev_0_labels, test_A_dataset = read_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split the training data into training and validation sets\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 380,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_texts, val_texts, train_labels, val_labels = train_test_split(\n",
    "    train_dataset[\"Text\"], train_dataset[\"Label\"], test_size=0.1, random_state=42\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = pd.DataFrame({\"Text\": train_texts, \"Label\": train_labels})\n",
    "val_dataset = pd.DataFrame({\"Text\": val_texts, \"Label\": val_labels})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenize the text and labels\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset[\"tokenized_text\"] = train_dataset[\"Text\"].apply(lambda x: x.split())\n",
    "train_dataset[\"tokenized_labels\"] = train_dataset[\"Label\"].apply(lambda x: x.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 383,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_dataset[\"tokenized_text\"] = val_dataset[\"Text\"].apply(lambda x: x.split())\n",
    "val_dataset[\"tokenized_labels\"] = val_dataset[\"Label\"].apply(lambda x: x.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_0_dataset[\"tokenized_text\"] = dev_0_dataset[\"Text\"].apply(lambda x: x.split())\n",
    "dev_0_dataset[\"tokenized_labels\"] = dev_0_labels[\"Label\"].apply(lambda x: x.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 385,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_A_dataset[\"tokenized_text\"] = test_A_dataset[\"Text\"].apply(lambda x: x.split())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a vocab object which maps tokens to indices\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 386,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_vocab(dataset):\n",
    "    counter = Counter()\n",
    "    for document in dataset:\n",
    "        counter.update(document)\n",
    "    return vocab(counter, specials=[\"<unk>\", \"<pad>\", \"<bos>\", \"<eos>\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 387,
   "metadata": {},
   "outputs": [],
   "source": [
    "v = build_vocab(train_dataset[\"tokenized_text\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Map indices to tokens\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 388,
   "metadata": {},
   "outputs": [],
   "source": [
    "itos = v.get_itos()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Number of tokens in the vocabulary\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 389,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22154"
      ]
     },
     "execution_count": 389,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(itos)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Index of the 'rejects' token\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 390,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9086"
      ]
     },
     "execution_count": 390,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v[\"rejects\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Index of the '\\<unk\\>' token\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 391,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 391,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v[\"<unk>\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set the default index to the unknown token\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 392,
   "metadata": {},
   "outputs": [],
   "source": [
    "v.set_default_index(v[\"<unk>\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use cuda if available\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 393,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vectorize the data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 394,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_process(dt):\n",
    "    return [\n",
    "        torch.tensor(\n",
    "            [v[\"<bos>\"]] + [v[token] for token in document] + [v[\"<eos>\"]],\n",
    "            dtype=torch.long,\n",
    "            device=device,\n",
    "        )\n",
    "        for document in dt\n",
    "    ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 395,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_tokens_ids = data_process(train_dataset[\"tokenized_text\"])\n",
    "val_tokens_ids = data_process(val_dataset[\"tokenized_text\"])\n",
    "dev_0_tokens_ids = data_process(dev_0_dataset[\"tokenized_text\"])\n",
    "test_A_tokens_ids = data_process(test_A_dataset[\"tokenized_text\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a mapping from label to index\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 396,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = [\"O\", \"B-PER\", \"I-PER\", \"B-ORG\", \"I-ORG\", \"B-LOC\", \"I-LOC\", \"B-MISC\", \"I-MISC\"]\n",
    "\n",
    "label_to_index = {label: idx for idx, label in enumerate(labels)}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vectorize the labels (NER)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 397,
   "metadata": {},
   "outputs": [],
   "source": [
    "def labels_process(dt, label_to_index):\n",
    "    return [\n",
    "        torch.tensor(\n",
    "            [0] + [label_to_index[label] for label in document] + [0],\n",
    "            dtype=torch.long,\n",
    "            device=device,\n",
    "        )\n",
    "        for document in dt\n",
    "    ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 398,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_labels = labels_process(train_dataset[\"tokenized_labels\"], label_to_index)\n",
    "val_labels = labels_process(val_dataset[\"tokenized_labels\"], label_to_index)\n",
    "dev_0_labels = labels_process(dev_0_dataset[\"tokenized_labels\"], label_to_index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function for evaluation (returns precision, recall, and F1 score)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 399,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_scores(y_true, y_pred):\n",
    "    acc_score = 0\n",
    "    tp = 0\n",
    "    fp = 0\n",
    "    selected_items = 0\n",
    "    relevant_items = 0\n",
    "\n",
    "    for p, t in zip(y_pred, y_true):\n",
    "        if p == t:\n",
    "            acc_score += 1\n",
    "\n",
    "        if p > 0 and p == t:\n",
    "            tp += 1\n",
    "\n",
    "        if p > 0:\n",
    "            selected_items += 1\n",
    "\n",
    "        if t > 0:\n",
    "            relevant_items += 1\n",
    "\n",
    "    if selected_items == 0:\n",
    "        precision = 1.0\n",
    "    else:\n",
    "        precision = tp / selected_items\n",
    "\n",
    "    if relevant_items == 0:\n",
    "        recall = 1.0\n",
    "    else:\n",
    "        recall = tp / relevant_items\n",
    "\n",
    "    if precision + recall == 0.0:\n",
    "        f1 = 0.0\n",
    "    else:\n",
    "        f1 = 2 * precision * recall / (precision + recall)\n",
    "\n",
    "    return precision, recall, f1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate the number of unique tags\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 400,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n"
     ]
    }
   ],
   "source": [
    "all_label_indices = [\n",
    "    label_to_index[label]\n",
    "    for document in train_dataset[\"tokenized_labels\"]\n",
    "    for label in document\n",
    "]\n",
    "\n",
    "num_tags = max(all_label_indices) + 1\n",
    "\n",
    "print(num_tags)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Implementation of a recurrent neural network LSTM\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 401,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSTM(torch.nn.Module):\n",
    "\n",
    "    def __init__(self, vocab_size, embedding_dim, hidden_dim, num_layers, num_tags):\n",
    "        super(LSTM, self).__init__()\n",
    "        self.embedding = torch.nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.rec = torch.nn.LSTM(\n",
    "            embedding_dim, hidden_dim, num_layers, batch_first=True, bidirectional=True\n",
    "        )\n",
    "        self.fc1 = torch.nn.Linear(hidden_dim * 2, num_tags)\n",
    "\n",
    "    def forward(self, x):\n",
    "        embedding = torch.relu(self.embedding(x))\n",
    "        lstm_output, _ = self.rec(embedding)\n",
    "        out_weights = self.fc1(lstm_output)\n",
    "        return out_weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize the LSTM model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 402,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm = LSTM(len(v.get_itos()), 100, 100, 1, num_tags).to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define the loss function\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 403,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = torch.nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define the optimizer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 404,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(lstm.parameters())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function for model evaluation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 405,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_model(dataset_tokens, dataset_labels, model):\n",
    "    Y_true = []\n",
    "    Y_pred = []\n",
    "    for i in tqdm(range(len(dataset_labels))):\n",
    "        batch_tokens = dataset_tokens[i].unsqueeze(0)\n",
    "        tags = list(dataset_labels[i].cpu().numpy())\n",
    "        Y_true += tags\n",
    "\n",
    "        Y_batch_pred_weights = model(batch_tokens).squeeze(0)\n",
    "        Y_batch_pred = torch.argmax(Y_batch_pred_weights, 1)\n",
    "        Y_pred += list(Y_batch_pred.cpu().numpy())\n",
    "\n",
    "    return get_scores(Y_true, Y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function for returning the predictions labels\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 406,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pred_labels(dataset_tokens, model, label_to_index):\n",
    "    Y_pred = []\n",
    "    inv_label_to_index = {\n",
    "        v: k for k, v in label_to_index.items()\n",
    "    }  # Create the inverse mapping\n",
    "\n",
    "    dataset_tokens = dataset_tokens[1:-1]\n",
    "\n",
    "    for i in tqdm(range(len(dataset_tokens))):\n",
    "        batch_tokens = dataset_tokens[i].unsqueeze(0)\n",
    "        Y_batch_pred_weights = model(batch_tokens).squeeze(0)\n",
    "        Y_batch_pred = torch.argmax(Y_batch_pred_weights, 1)\n",
    "        predicted_labels = [inv_label_to_index[label.item()] for label in Y_batch_pred]\n",
    "        Y_pred.append(\" \".join(predicted_labels))\n",
    "\n",
    "    return Y_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 407,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_EPOCHS = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 408,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0587944344f4bf1950126a01ac8ad07",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/850 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7131c4d76dbe415ab561eb386f73a9e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.7132680320569902, 0.23037100949094047, 0.3482608695652174)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "972aac01482b4736bb64a8057d3f38e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/850 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f70ae0eb2e074348aaf9b08840b32842",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.7749537892791127, 0.4823123382226057, 0.5945754298883177)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c12ce79cf9f3419f808cd67920124d44",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/850 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "950fa18832c44b2f8b39a76cfae9e4f5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.7864864864864864, 0.5858498705780846, 0.6715015658480303)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e64c3ac1406b47319a913c276659741b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/850 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "563008ad95664e9a93fda3bad47a8286",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.7872340425531915, 0.6384814495254529, 0.7050976655550262)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b169c3c7bd8f46afbb4613e53d3f3004",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/850 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "29392ac1478d4b41b58dc7cbb047977e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.7830188679245284, 0.6683922922059247, 0.7211792086889061)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e3671434bef46748a6303fae19e37f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/850 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e208db80dfc413a9ca2d0194eb0dfbb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.7966499162479062, 0.6839229220592464, 0.7359950479727638)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "428f27d053314b2baa730a11e0bf2109",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/850 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "09aa77d56e88401980661735666d5e84",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.8300609343263372, 0.7052056370434282, 0.7625563675944643)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a7005c70fd9542ca8a215ea99f939e83",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/850 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ef6c9b82a014300853a1bedffc072f7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.8456536618754278, 0.7106701179177451, 0.7723081731520549)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9dd3d20480224ee2829eee65fbce6a56",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/850 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9952fc82ecaf4608b90423b06e9ca9b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.8593695878074126, 0.7135461604831751, 0.7796983029541169)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3aa663321a094110982f975c44a9f89c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/850 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2db3d63d612f45dfa6877f98866d2f90",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.8427757291317466, 0.723037100949094, 0.778328173374613)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "acee67bf61d9414eae672c396d223498",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/850 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "853b2b1bba1e4803bc403328ca500587",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.8629512793550649, 0.7080816796088583, 0.7778830963665087)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6caff7e4c24c4a92aa7d1fffb871ac32",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/850 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a8fea3e3457485fa393f32197555dc0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.854050279329609, 0.7034800115041703, 0.7714871471376755)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "530a91d496044e3eb3a195f575e081dd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/850 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "678cb14ce65d4a8abe9f519526a946f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.8393868710429857, 0.724475122231809, 0.7777091694967583)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "600450b7660547128a1de002251da5fd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/850 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "631a76105d884c958f6a2c4da5ccfc98",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.8705717292178183, 0.7138337647397182, 0.7844500632111252)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "25e6be58b58a46efb29efc02a32bfad3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/850 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f06164feed74871ad2e8d7a3837df16",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.8473666554847367, 0.72648835202761, 0.7822855373180551)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "19761ae688124f6eb3dba461896533bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/850 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "51ba4f5b67694431adfae6ecfcb179e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.8214173228346456, 0.7500719010641358, 0.7841250751653638)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f73c9d22b17b40f4ab69c5b1454545f7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/850 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "afba4e9112ed415aaf3647ae8dfe34df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.8588936734017887, 0.7457578372159908, 0.7983374384236454)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "12f030871ee34c698598afe497560488",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/850 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33f2653607e14978bf29480c899c349e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.8506047728015691, 0.7483462755248778, 0.7962056303549571)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cb3464cfa7544252a723d7ee1ce66c91",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/850 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3df77e255ed42af90bcee1fe73dd9d2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.8432908912830558, 0.7428817946505608, 0.789908256880734)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e1c7776d0be546ffa459ab3199edd9de",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/850 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c32c359edab44cf8b8e556afa06c45e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.8448275862068966, 0.7469082542421628, 0.7928560525110669)\n"
     ]
    }
   ],
   "source": [
    "for i in range(NUM_EPOCHS):\n",
    "    lstm.train()\n",
    "    for i in tqdm(range(len(train_labels))):\n",
    "        batch_tokens = train_tokens_ids[i].unsqueeze(0)\n",
    "        tags = train_labels[i].unsqueeze(1)\n",
    "\n",
    "        predicted_tags = lstm(batch_tokens)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss = criterion(predicted_tags.squeeze(0), tags.squeeze(1))\n",
    "\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "    lstm.eval()\n",
    "    print(eval_model(val_tokens_ids, val_labels, lstm))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 409,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2600c9749c804cbcbfb4e5cd0d2a266a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(0.8448275862068966, 0.7469082542421628, 0.7928560525110669)"
      ]
     },
     "execution_count": 409,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_model(val_tokens_ids, val_labels, lstm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 410,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "529b42c0bd364a6b92b334c003af2a7c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/215 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(0.871974921630094, 0.8103006292239571, 0.8400072476897988)"
      ]
     },
     "execution_count": 410,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_model(dev_0_tokens_ids, dev_0_labels, lstm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 411,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2e41550d0583427e96f7525ac627a08f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/213 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dev_0_predictons = pred_labels(dev_0_tokens_ids, lstm, label_to_index)\n",
    "dev_0_predictons = pd.DataFrame(dev_0_predictons, columns=[\"Label\"])\n",
    "dev_0_predictons.to_csv(\"dev-0/out.tsv\", index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 412,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4983214de09b42f1b2240dd80987353f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/228 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_A_predictions = pred_labels(test_A_tokens_ids, lstm, label_to_index)\n",
    "test_A_predictions = pd.DataFrame(test_A_predictions, columns=[\"Label\"])\n",
    "test_A_predictions.to_csv(\"test-A/out.tsv\", index=False, header=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}