{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "![Logo 1](https://git.wmi.amu.edu.pl/AITech/Szablon/raw/branch/master/Logotyp_AITech1.jpg)\n", "
\n", "

Ekstrakcja informacji

\n", "

11. NER RNN [ćwiczenia]

\n", "

Jakub Pokrywka (2021)

\n", "
\n", "\n", "![Logo 2](https://git.wmi.amu.edu.pl/AITech/Szablon/raw/branch/master/Logotyp_AITech2.jpg)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Podejście softmax z embeddingami na przykładzie NER" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import gensim\n", "import torch\n", "import pandas as pd\n", "import seaborn as sns\n", "from sklearn.model_selection import train_test_split\n", "\n", "from datasets import load_dataset\n", "import torchtext\n", "#from torchtext.vocab import vocab\n", "from collections import Counter\n", "\n", "from sklearn.datasets import fetch_20newsgroups\n", "# https://scikit-learn.org/0.19/datasets/twenty_newsgroups.html\n", "\n", "from sklearn.feature_extraction.text import TfidfVectorizer\n", "from sklearn.metrics import accuracy_score\n", "\n", "from tqdm.notebook import tqdm\n", "\n", "import torch" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Reusing dataset conll2003 (/home/kuba/.cache/huggingface/datasets/conll2003/conll2003/1.0.0/63f4ebd1bcb7148b1644497336fd74643d4ce70123334431a3c053b7ee4e96ee)\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "7c9a8ca324914c40b7606ab8cd487df2", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/3 [00:00', '', '', ''])\n", " vocab.set_default_index(0)\n", " return vocab" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "vocab = build_vocab(dataset['train']['tokens'])" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "21" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "vocab['on']" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "def data_process(dt):\n", " return [ torch.tensor([vocab['']] +[vocab[token] for token in document ] + [vocab['']], dtype = torch.long) for document in dt]" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "def labels_process(dt):\n", " return [ torch.tensor([0] + document + [0], dtype = torch.long) for document in dt]\n" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "train_tokens_ids = data_process(dataset['train']['tokens'])" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "test_tokens_ids = data_process(dataset['test']['tokens'])" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "validation_tokens_ids = data_process(dataset['validation']['tokens'])" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "scrolled": true }, "outputs": [], "source": [ "train_labels = labels_process(dataset['train']['ner_tags'])" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "validation_labels = labels_process(dataset['validation']['ner_tags'])" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "test_labels = labels_process(dataset['test']['ner_tags'])" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([ 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 3])" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "train_tokens_ids[0]" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'id': '0',\n", " 'tokens': ['EU',\n", " 'rejects',\n", " 'German',\n", " 'call',\n", " 'to',\n", " 'boycott',\n", " 'British',\n", " 'lamb',\n", " '.'],\n", " 'pos_tags': [22, 42, 16, 21, 35, 37, 16, 21, 7],\n", " 'chunk_tags': [11, 21, 11, 12, 21, 22, 11, 12, 0],\n", " 'ner_tags': [3, 0, 7, 0, 0, 0, 7, 0, 0]}" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dataset['train'][0]" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/plain": [ "tensor([0, 3, 0, 7, 0, 0, 0, 7, 0, 0, 0])" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "train_labels[0]" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [], "source": [ "def get_scores(y_true, y_pred):\n", " acc_score = 0\n", " tp = 0\n", " fp = 0\n", " selected_items = 0\n", " relevant_items = 0 \n", "\n", " for p,t in zip(y_pred, y_true):\n", " if p == t:\n", " acc_score +=1\n", "\n", " if p > 0 and p == t:\n", " tp +=1\n", "\n", " if p > 0:\n", " selected_items += 1\n", "\n", " if t > 0 :\n", " relevant_items +=1\n", "\n", " \n", " \n", " if selected_items == 0:\n", " precision = 1.0\n", " else:\n", " precision = tp / selected_items\n", " \n", " \n", " if relevant_items == 0:\n", " recall = 1.0\n", " else:\n", " recall = tp / relevant_items\n", " \n", " \n", " if precision + recall == 0.0 :\n", " f1 = 0.0\n", " else:\n", " f1 = 2* precision * recall / (precision + recall)\n", "\n", " return precision, recall, f1" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "num_tags = max([max(x) for x in dataset['train']['ner_tags'] if x]) + 1 " ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "class LSTM(torch.nn.Module):\n", "\n", " def __init__(self):\n", " super(LSTM, self).__init__()\n", " self.emb = torch.nn.Embedding(len(vocab.get_itos()),100)\n", " self.rec = torch.nn.LSTM(100, 256, 1, batch_first = True)\n", " self.fc1 = torch.nn.Linear( 256 , 9)\n", "\n", " def forward(self, x):\n", " emb = torch.relu(self.emb(x))\n", " \n", " lstm_output, (h_n, c_n) = self.rec(emb)\n", " \n", " out_weights = self.fc1(lstm_output)\n", "\n", " return out_weights" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "lstm = LSTM()" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [], "source": [ "criterion = torch.nn.CrossEntropyLoss()" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [], "source": [ "optimizer = torch.optim.Adam(lstm.parameters())" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [], "source": [ "def eval_model(dataset_tokens, dataset_labels, model):\n", " Y_true = []\n", " Y_pred = []\n", " for i in tqdm(range(len(dataset_labels))):\n", " batch_tokens = dataset_tokens[i].unsqueeze(0)\n", " tags = list(dataset_labels[i].numpy())\n", " Y_true += tags\n", " \n", " Y_batch_pred_weights = model(batch_tokens).squeeze(0)\n", " Y_batch_pred = torch.argmax(Y_batch_pred_weights,1)\n", " Y_pred += list(Y_batch_pred.numpy())\n", " \n", "\n", " return get_scores(Y_true, Y_pred)\n", " " ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [], "source": [ "NUM_EPOCHS = 5" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "scrolled": true }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "59e268fa2b29414fb6306ec4ee44d51f", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/500 [00:00 1 ?\n", "- co zrobić, żeby sieć uwzględniała następne tokeny, a nie tylko poprzednie?\n", "- w jaki sposób wykorzystać taką sieć do zadania zwykłej klasyfikacji?" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Zadanie na zajęcia ( 20 minut)\n", "\n", "zmodyfikować sieć tak, żeby była używała dwuwarstwowej, dwukierunkowej warstwy GRU oraz dropoutu. Dropout ma nałożony na embeddingi.\n" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [], "source": [ "class GRU(torch.nn.Module):\n", "\n", " def __init__(self):\n", " super(GRU, self).__init__()\n", " self.emb = torch.nn.Embedding(len(vocab.get_itos()),100)\n", " self.dropout = torch.nn.Dropout(0.2)\n", " self.rec = torch.nn.GRU(100, 256, 2, batch_first = True, bidirectional = True)\n", " self.fc1 = torch.nn.Linear(2* 256 , 9)\n", " \n", " def forward(self, x):\n", " emb = torch.relu(self.emb(x))\n", " emb = self.dropout(emb)\n", " \n", " gru_output, h_n = self.rec(emb)\n", " \n", " out_weights = self.fc1(gru_output)\n", "\n", " return out_weights" ] }, { "cell_type": "code", "execution_count": 30, "metadata": {}, "outputs": [], "source": [ "gru = GRU()" ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, "outputs": [], "source": [ "criterion = torch.nn.CrossEntropyLoss()" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [], "source": [ "optimizer = torch.optim.Adam(gru.parameters())" ] }, { "cell_type": "code", "execution_count": 33, "metadata": {}, "outputs": [], "source": [ "NUM_EPOCHS = 5" ] }, { "cell_type": "code", "execution_count": 34, "metadata": { "scrolled": true }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "109e0891142545ee8315c040cb231fb2", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/500 [00:00