From 15854bb61f5b95c4b14832ddb58bb1d2c984c226 Mon Sep 17 00:00:00 2001 From: kubapok Date: Wed, 2 Jun 2021 13:27:34 +0200 Subject: [PATCH] rnn --- cw/11_NER_RNN.ipynb | 828 +++++++++++++++++++++++ cw/11_NER_RNN_ODPOWIEDZI.ipynb | 1137 ++++++++++++++++++++++++++++++++ 2 files changed, 1965 insertions(+) create mode 100644 cw/11_NER_RNN.ipynb create mode 100644 cw/11_NER_RNN_ODPOWIEDZI.ipynb diff --git a/cw/11_NER_RNN.ipynb b/cw/11_NER_RNN.ipynb new file mode 100644 index 0000000..4965188 --- /dev/null +++ b/cw/11_NER_RNN.ipynb @@ -0,0 +1,828 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Podejście softmax z embeddingami na przykładzie NER" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/media/kuba/ssdsam/anaconda3/lib/python3.8/site-packages/gensim/similarities/__init__.py:15: UserWarning: The gensim.similarities.levenshtein submodule is disabled, because the optional Levenshtein package is unavailable. Install Levenhstein (e.g. `pip install python-Levenshtein`) to suppress this warning.\n", + " warnings.warn(msg)\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "import gensim\n", + "import torch\n", + "import pandas as pd\n", + "import seaborn as sns\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "from datasets import load_dataset\n", + "from torchtext.vocab import Vocab\n", + "from collections import Counter\n", + "\n", + "from sklearn.datasets import fetch_20newsgroups\n", + "# https://scikit-learn.org/0.19/datasets/twenty_newsgroups.html\n", + "\n", + "from sklearn.feature_extraction.text import TfidfVectorizer\n", + "from sklearn.metrics import accuracy_score\n", + "\n", + "from tqdm.notebook import tqdm\n", + "\n", + "import torch" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Reusing dataset conll2003 (/home/kuba/.cache/huggingface/datasets/conll2003/conll2003/1.0.0/40e7cb6bcc374f7c349c83acd1e9352a4f09474eb691f64f364ee62eb65d0ca6)\n" + ] + } + ], + "source": [ + "dataset = load_dataset(\"conll2003\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def build_vocab(dataset):\n", + " counter = Counter()\n", + " for document in dataset:\n", + " counter.update(document)\n", + " return Vocab(counter, specials=['', '', '', ''])" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "vocab = build_vocab(dataset['train']['tokens'])" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "23627" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(vocab.itos)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "15" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vocab['on']" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "def data_process(dt):\n", + " return [ torch.tensor([vocab['']] +[vocab[token] for token in document ] + [vocab['']], dtype = torch.long) for document in dt]" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "def labels_process(dt):\n", + " return [ torch.tensor([0] + document + [0], dtype = torch.long) for document in dt]\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "train_tokens_ids = data_process(dataset['train']['tokens'])" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "test_tokens_ids = data_process(dataset['test']['tokens'])" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "validation_tokens_ids = data_process(dataset['validation']['tokens'])" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "train_labels = labels_process(dataset['train']['ner_tags'])" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "validation_labels = labels_process(dataset['validation']['ner_tags'])" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "test_labels = labels_process(dataset['test']['ner_tags'])" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([ 2, 966, 22409, 238, 773, 9, 4588, 212, 7686, 4,\n", + " 3])" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "train_tokens_ids[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'chunk_tags': [11, 21, 11, 12, 21, 22, 11, 12, 0],\n", + " 'id': '0',\n", + " 'ner_tags': [3, 0, 7, 0, 0, 0, 7, 0, 0],\n", + " 'pos_tags': [22, 42, 16, 21, 35, 37, 16, 21, 7],\n", + " 'tokens': ['EU',\n", + " 'rejects',\n", + " 'German',\n", + " 'call',\n", + " 'to',\n", + " 'boycott',\n", + " 'British',\n", + " 'lamb',\n", + " '.']}" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset['train'][0]" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([0, 3, 0, 7, 0, 0, 0, 7, 0, 0, 0])" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "train_labels[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "def get_scores(y_true, y_pred):\n", + " acc_score = 0\n", + " tp = 0\n", + " fp = 0\n", + " selected_items = 0\n", + " relevant_items = 0 \n", + "\n", + " for p,t in zip(y_pred, y_true):\n", + " if p == t:\n", + " acc_score +=1\n", + "\n", + " if p > 0 and p == t:\n", + " tp +=1\n", + "\n", + " if p > 0:\n", + " selected_items += 1\n", + "\n", + " if t > 0 :\n", + " relevant_items +=1\n", + "\n", + " \n", + " \n", + " if selected_items == 0:\n", + " precision = 1.0\n", + " else:\n", + " precision = tp / selected_items\n", + " \n", + " \n", + " if relevant_items == 0:\n", + " recall = 1.0\n", + " else:\n", + " recall = tp / relevant_items\n", + " \n", + " \n", + " if precision + recall == 0.0 :\n", + " f1 = 0.0\n", + " else:\n", + " f1 = 2* precision * recall / (precision + recall)\n", + "\n", + " return precision, recall, f1" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "num_tags = max([max(x) for x in dataset['train']['ner_tags'] ]) + 1 " + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "class LSTM(torch.nn.Module):\n", + "\n", + " def __init__(self):\n", + " super(LSTM, self).__init__()\n", + " self.emb = torch.nn.Embedding(len(vocab.itos),100)\n", + " self.rec = torch.nn.LSTM(100, 256, 1, batch_first = True)\n", + " self.fc1 = torch.nn.Linear( 256 , 9)\n", + "\n", + " def forward(self, x):\n", + " emb = torch.relu(self.emb(x))\n", + " \n", + " lstm_output, (h_n, c_n) = self.rec(emb)\n", + " \n", + " out_weights = self.fc1(lstm_output)\n", + "\n", + " return out_weights" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "lstm = LSTM()" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "criterion = torch.nn.CrossEntropyLoss()" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = torch.optim.Adam(lstm.parameters())" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "def eval_model(dataset_tokens, dataset_labels, model):\n", + " Y_true = []\n", + " Y_pred = []\n", + " for i in tqdm(range(len(dataset_labels))):\n", + " batch_tokens = dataset_tokens[i].unsqueeze(0)\n", + " tags = list(dataset_labels[i].numpy())\n", + " Y_true += tags\n", + " \n", + " Y_batch_pred_weights = model(batch_tokens).squeeze(0)\n", + " Y_batch_pred = torch.argmax(Y_batch_pred_weights,1)\n", + " Y_pred += list(Y_batch_pred.numpy())\n", + " \n", + "\n", + " return get_scores(Y_true, Y_pred)\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "NUM_EPOCHS = 5" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "75184b632ce54ae690b3444778f44651", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "74a55a414fa948a3b251b89f780564d0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "(0.5068524970963996, 0.5072649075903755, 0.5070586184860281)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0c9c580076fb4ec48b7ea2f300878594", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0be8681c67f64aca95ce5d3c44f10538", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "(0.653649243957614, 0.6381494827385795, 0.6458063757205035)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2dec403004bb4ae298bc73553ea3f4bc", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "eebed0407ba343e29cf8c2d607f631dc", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "(0.7140486069946651, 0.7001046146693014, 0.7070078647728607)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "70792f22eea343c8916bcfcf9215c298", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5d400bf1b656433ba2091cf750ec2d78", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "(0.756327964151629, 0.725909566430315, 0.7408066429418744)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "604c4fa13c03435d81bf68be37977d74", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2f78871f366f4fd1b7de6c4be5303906", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "(0.7963248522230789, 0.7203301174009067, 0.7564235581324383)\n" + ] + } + ], + "source": [ + "for i in range(NUM_EPOCHS):\n", + " lstm.train()\n", + " #for i in tqdm(range(500)):\n", + " for i in tqdm(range(len(train_labels))):\n", + " batch_tokens = train_tokens_ids[i].unsqueeze(0)\n", + " tags = train_labels[i].unsqueeze(1)\n", + " \n", + " \n", + " predicted_tags = lstm(batch_tokens)\n", + "\n", + " \n", + " optimizer.zero_grad()\n", + " loss = criterion(predicted_tags.squeeze(0),tags.squeeze(1))\n", + " \n", + " loss.backward()\n", + " optimizer.step()\n", + " \n", + " lstm.eval()\n", + " print(eval_model(validation_tokens_ids, validation_labels, lstm))" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5159f7a61c3a439bab45573f15ea55b2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "(0.7963248522230789, 0.7203301174009067, 0.7564235581324383)" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "eval_model(validation_tokens_ids, validation_labels, lstm)" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4b604bbb796f4d4cb99528fad98cfdff", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=3453.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "(0.7450810185185185, 0.6348619329388561, 0.685569755058573)" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "eval_model(test_tokens_ids, test_labels, lstm)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "14041" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(train_tokens_ids)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## pytania\n", + "\n", + "- co zrobić z trenowaniem na batchach > 1 ?\n", + "- co zrobić, żeby sieć uwzględniała następne tokeny, a nie tylko poprzednie?\n", + "- w jaki sposób wykorzystać taką sieć do zadania zwykłej klasyfikacji?" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Zadanie na zajęcia ( 20 minut)\n", + "\n", + "zmodyfikować sieć tak, żeby była używała dwuwarstwowej, dwukierunkowej warstwy GRU oraz dropoutu. Dropout ma nałożony na embeddingi.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Zadanie domowe\n", + "\n", + "\n", + "- sklonować repozytorium https://git.wmi.amu.edu.pl/kubapok/en-ner-conll-2003\n", + "- stworzyć model seq labelling bazujący na sieci neuronowej opisanej w punkcie niżej (można bazować na tym jupyterze lub nie).\n", + "- model sieci to GRU (o dowolnych parametrach) + CRF w pytorchu korzystając z modułu CRF z poprzednich zajęć- - stworzyć predykcje w plikach dev-0/out.tsv oraz test-A/out.tsv\n", + "- wynik fscore sprawdzony za pomocą narzędzia geval (patrz poprzednie zadanie) powinien wynosić conajmniej 0.65\n", + "- proszę umieścić predykcję oraz skrypty generujące (w postaci tekstowej a nie jupyter) w repo, a w MS TEAMS umieścić link do swojego repo\n", + "termin 22.06, 60 punktów, za najlepszy wynik- 100 punktów\n", + " " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/cw/11_NER_RNN_ODPOWIEDZI.ipynb b/cw/11_NER_RNN_ODPOWIEDZI.ipynb new file mode 100644 index 0000000..03aceae --- /dev/null +++ b/cw/11_NER_RNN_ODPOWIEDZI.ipynb @@ -0,0 +1,1137 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Podejście softmax z embeddingami na przykładzie NER" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/media/kuba/ssdsam/anaconda3/lib/python3.8/site-packages/gensim/similarities/__init__.py:15: UserWarning: The gensim.similarities.levenshtein submodule is disabled, because the optional Levenshtein package is unavailable. Install Levenhstein (e.g. `pip install python-Levenshtein`) to suppress this warning.\n", + " warnings.warn(msg)\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "import gensim\n", + "import torch\n", + "import pandas as pd\n", + "import seaborn as sns\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "from datasets import load_dataset\n", + "from torchtext.vocab import Vocab\n", + "from collections import Counter\n", + "\n", + "from sklearn.datasets import fetch_20newsgroups\n", + "# https://scikit-learn.org/0.19/datasets/twenty_newsgroups.html\n", + "\n", + "from sklearn.feature_extraction.text import TfidfVectorizer\n", + "from sklearn.metrics import accuracy_score\n", + "\n", + "from tqdm.notebook import tqdm\n", + "\n", + "import torch" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Reusing dataset conll2003 (/home/kuba/.cache/huggingface/datasets/conll2003/conll2003/1.0.0/40e7cb6bcc374f7c349c83acd1e9352a4f09474eb691f64f364ee62eb65d0ca6)\n" + ] + } + ], + "source": [ + "dataset = load_dataset(\"conll2003\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def build_vocab(dataset):\n", + " counter = Counter()\n", + " for document in dataset:\n", + " counter.update(document)\n", + " return Vocab(counter, specials=['', '', '', ''])" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "vocab = build_vocab(dataset['train']['tokens'])" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "23627" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(vocab.itos)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "15" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "vocab['on']" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "def data_process(dt):\n", + " return [ torch.tensor([vocab['']] +[vocab[token] for token in document ] + [vocab['']], dtype = torch.long) for document in dt]" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "def labels_process(dt):\n", + " return [ torch.tensor([0] + document + [0], dtype = torch.long) for document in dt]\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "train_tokens_ids = data_process(dataset['train']['tokens'])" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "test_tokens_ids = data_process(dataset['test']['tokens'])" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "validation_tokens_ids = data_process(dataset['validation']['tokens'])" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "train_labels = labels_process(dataset['train']['ner_tags'])" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "validation_labels = labels_process(dataset['validation']['ner_tags'])" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "test_labels = labels_process(dataset['test']['ner_tags'])" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([ 2, 966, 22409, 238, 773, 9, 4588, 212, 7686, 4,\n", + " 3])" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "train_tokens_ids[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'chunk_tags': [11, 21, 11, 12, 21, 22, 11, 12, 0],\n", + " 'id': '0',\n", + " 'ner_tags': [3, 0, 7, 0, 0, 0, 7, 0, 0],\n", + " 'pos_tags': [22, 42, 16, 21, 35, 37, 16, 21, 7],\n", + " 'tokens': ['EU',\n", + " 'rejects',\n", + " 'German',\n", + " 'call',\n", + " 'to',\n", + " 'boycott',\n", + " 'British',\n", + " 'lamb',\n", + " '.']}" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset['train'][0]" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([0, 3, 0, 7, 0, 0, 0, 7, 0, 0, 0])" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "train_labels[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "def get_scores(y_true, y_pred):\n", + " acc_score = 0\n", + " tp = 0\n", + " fp = 0\n", + " selected_items = 0\n", + " relevant_items = 0 \n", + "\n", + " for p,t in zip(y_pred, y_true):\n", + " if p == t:\n", + " acc_score +=1\n", + "\n", + " if p > 0 and p == t:\n", + " tp +=1\n", + "\n", + " if p > 0:\n", + " selected_items += 1\n", + "\n", + " if t > 0 :\n", + " relevant_items +=1\n", + "\n", + " \n", + " \n", + " if selected_items == 0:\n", + " precision = 1.0\n", + " else:\n", + " precision = tp / selected_items\n", + " \n", + " \n", + " if relevant_items == 0:\n", + " recall = 1.0\n", + " else:\n", + " recall = tp / relevant_items\n", + " \n", + " \n", + " if precision + recall == 0.0 :\n", + " f1 = 0.0\n", + " else:\n", + " f1 = 2* precision * recall / (precision + recall)\n", + "\n", + " return precision, recall, f1" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "num_tags = max([max(x) for x in dataset['train']['ner_tags'] ]) + 1 " + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "class LSTM(torch.nn.Module):\n", + "\n", + " def __init__(self):\n", + " super(LSTM, self).__init__()\n", + " self.emb = torch.nn.Embedding(len(vocab.itos),100)\n", + " self.rec = torch.nn.LSTM(100, 256, 1, batch_first = True)\n", + " self.fc1 = torch.nn.Linear( 256 , 9)\n", + "\n", + " def forward(self, x):\n", + " emb = torch.relu(self.emb(x))\n", + " \n", + " lstm_output, (h_n, c_n) = self.rec(emb)\n", + " \n", + " out_weights = self.fc1(lstm_output)\n", + "\n", + " return out_weights" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "lstm = LSTM()" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "criterion = torch.nn.CrossEntropyLoss()" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = torch.optim.Adam(lstm.parameters())" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "def eval_model(dataset_tokens, dataset_labels, model):\n", + " Y_true = []\n", + " Y_pred = []\n", + " for i in tqdm(range(len(dataset_labels))):\n", + " batch_tokens = dataset_tokens[i].unsqueeze(0)\n", + " tags = list(dataset_labels[i].numpy())\n", + " Y_true += tags\n", + " \n", + " Y_batch_pred_weights = model(batch_tokens).squeeze(0)\n", + " Y_batch_pred = torch.argmax(Y_batch_pred_weights,1)\n", + " Y_pred += list(Y_batch_pred.numpy())\n", + " \n", + "\n", + " return get_scores(Y_true, Y_pred)\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "NUM_EPOCHS = 5" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "75184b632ce54ae690b3444778f44651", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "74a55a414fa948a3b251b89f780564d0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "(0.5068524970963996, 0.5072649075903755, 0.5070586184860281)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0c9c580076fb4ec48b7ea2f300878594", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0be8681c67f64aca95ce5d3c44f10538", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "(0.653649243957614, 0.6381494827385795, 0.6458063757205035)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2dec403004bb4ae298bc73553ea3f4bc", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "eebed0407ba343e29cf8c2d607f631dc", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "(0.7140486069946651, 0.7001046146693014, 0.7070078647728607)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "70792f22eea343c8916bcfcf9215c298", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5d400bf1b656433ba2091cf750ec2d78", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "(0.756327964151629, 0.725909566430315, 0.7408066429418744)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "604c4fa13c03435d81bf68be37977d74", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2f78871f366f4fd1b7de6c4be5303906", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "(0.7963248522230789, 0.7203301174009067, 0.7564235581324383)\n" + ] + } + ], + "source": [ + "for i in range(NUM_EPOCHS):\n", + " lstm.train()\n", + " #for i in tqdm(range(500)):\n", + " for i in tqdm(range(len(train_labels))):\n", + " batch_tokens = train_tokens_ids[i].unsqueeze(0)\n", + " tags = train_labels[i].unsqueeze(1)\n", + " \n", + " \n", + " predicted_tags = lstm(batch_tokens)\n", + "\n", + " \n", + " optimizer.zero_grad()\n", + " loss = criterion(predicted_tags.squeeze(0),tags.squeeze(1))\n", + " \n", + " loss.backward()\n", + " optimizer.step()\n", + " \n", + " lstm.eval()\n", + " print(eval_model(validation_tokens_ids, validation_labels, lstm))" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5159f7a61c3a439bab45573f15ea55b2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "(0.7963248522230789, 0.7203301174009067, 0.7564235581324383)" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "eval_model(validation_tokens_ids, validation_labels, lstm)" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4b604bbb796f4d4cb99528fad98cfdff", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=3453.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "(0.7450810185185185, 0.6348619329388561, 0.685569755058573)" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "eval_model(test_tokens_ids, test_labels, lstm)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "14041" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(train_tokens_ids)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## pytania\n", + "\n", + "- co zrobić z trenowaniem na batchach > 1 ?\n", + "- co zrobić, żeby sieć uwzględniała następne tokeny, a nie tylko poprzednie?\n", + "- w jaki sposób wykorzystać taką sieć do zadania zwykłej klasyfikacji?" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Zadanie na zajęcia ( 20 minut)\n", + "\n", + "zmodyfikować sieć tak, żeby była używała dwuwarstwowej, dwukierunkowej warstwy GRU oraz dropoutu. Dropout ma nałożony na embeddingi.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "class GRU(torch.nn.Module):\n", + "\n", + " def __init__(self):\n", + " super(GRU, self).__init__()\n", + " self.emb = torch.nn.Embedding(len(vocab.itos),100)\n", + " self.dropout = torch.nn.Dropout(0.2)\n", + " self.rec = torch.nn.GRU(100, 256, 2, batch_first = True, bidirectional = True)\n", + " self.fc1 = torch.nn.Linear(2* 256 , 9)\n", + " \n", + " def forward(self, x):\n", + " emb = torch.relu(self.emb(x))\n", + " emb = self.dropout(emb)\n", + " \n", + " gru_output, h_n = self.rec(emb)\n", + " \n", + " out_weights = self.fc1(gru_output)\n", + "\n", + " return out_weights" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "gru = GRU()" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [], + "source": [ + "criterion = torch.nn.CrossEntropyLoss()" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = torch.optim.Adam(gru.parameters())" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [], + "source": [ + "NUM_EPOCHS = 5" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a0b15b129c294730ab2a6035b9a98b47", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b7b3bc93dc7349949b59f97751ebdec2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "(0.6431379891406104, 0.39927932116703474, 0.49268502581755597)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0fa832ed33e3451ebc241046ea299d2e", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "953a62a22eb44de4a33b0e1c53a472f0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "(0.6638910917261432, 0.5838660932232942, 0.6213123879027769)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5efce69cb2e246d2adababcc18780083", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "cb7cde559dd544b0961fa4c855b856d8", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "(0.6697936210131332, 0.7054515866558178, 0.6871603260869565)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d7211ba79faa4de3aeafff4ef39bbaf1", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2a8dbd5393fa4ee2a8ee72738efa2a57", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "(0.7091097308488613, 0.7166104847146344, 0.7128403769439787)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "84d0b4b0e1ec4c1a88aff0698836a362", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "31a2f0a285e44a55a7b583a81511dbc2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "(0.7708963348252087, 0.740788097175404, 0.755542382928275)\n" + ] + } + ], + "source": [ + "for i in range(NUM_EPOCHS):\n", + " gru.train()\n", + " #for i in tqdm(range(50)):\n", + " for i in tqdm(range(len(train_labels))):\n", + " batch_tokens = train_tokens_ids[i].unsqueeze(0)\n", + " tags = train_labels[i].unsqueeze(1)\n", + " \n", + " \n", + " predicted_tags = gru(batch_tokens)\n", + "\n", + " \n", + " optimizer.zero_grad()\n", + " loss = criterion(predicted_tags.squeeze(0),tags.squeeze(1))\n", + " \n", + " loss.backward()\n", + " optimizer.step()\n", + " \n", + " \n", + " gru.eval()\n", + " print(eval_model(validation_tokens_ids, validation_labels, gru))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Zadanie domowe\n", + "\n", + "\n", + "- sklonować repozytorium https://git.wmi.amu.edu.pl/kubapok/en-ner-conll-2003\n", + "- stworzyć model seq labelling bazujący na sieci neuronowej opisanej w punkcie niżej (można bazować na tym jupyterze lub nie).\n", + "- model sieci to GRU (o dowolnych parametrach) + CRF w pytorchu korzystając z modułu CRF z poprzednich zajęć- - stworzyć predykcje w plikach dev-0/out.tsv oraz test-A/out.tsv\n", + "- wynik fscore sprawdzony za pomocą narzędzia geval (patrz poprzednie zadanie) powinien wynosić conajmniej 0.65\n", + "- proszę umieścić predykcję oraz skrypty generujące (w postaci tekstowej a nie jupyter) w repo, a w MS TEAMS umieścić link do swojego repo\n", + "termin 22.06, 60 punktów, za najlepszy wynik- 100 punktów\n", + " " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}