forked from filipg/aitech-eks-pub
1138 lines
26 KiB
Plaintext
1138 lines
26 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Podejście softmax z embeddingami na przykładzie NER"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"/media/kuba/ssdsam/anaconda3/lib/python3.8/site-packages/gensim/similarities/__init__.py:15: UserWarning: The gensim.similarities.levenshtein submodule is disabled, because the optional Levenshtein package <https://pypi.org/project/python-Levenshtein/> is unavailable. Install Levenhstein (e.g. `pip install python-Levenshtein`) to suppress this warning.\n",
|
|
" warnings.warn(msg)\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import numpy as np\n",
|
|
"import gensim\n",
|
|
"import torch\n",
|
|
"import pandas as pd\n",
|
|
"import seaborn as sns\n",
|
|
"from sklearn.model_selection import train_test_split\n",
|
|
"\n",
|
|
"from datasets import load_dataset\n",
|
|
"from torchtext.vocab import Vocab\n",
|
|
"from collections import Counter\n",
|
|
"\n",
|
|
"from sklearn.datasets import fetch_20newsgroups\n",
|
|
"# https://scikit-learn.org/0.19/datasets/twenty_newsgroups.html\n",
|
|
"\n",
|
|
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
|
|
"from sklearn.metrics import accuracy_score\n",
|
|
"\n",
|
|
"from tqdm.notebook import tqdm\n",
|
|
"\n",
|
|
"import torch"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {
|
|
"scrolled": false
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Reusing dataset conll2003 (/home/kuba/.cache/huggingface/datasets/conll2003/conll2003/1.0.0/40e7cb6bcc374f7c349c83acd1e9352a4f09474eb691f64f364ee62eb65d0ca6)\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"dataset = load_dataset(\"conll2003\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def build_vocab(dataset):\n",
|
|
" counter = Counter()\n",
|
|
" for document in dataset:\n",
|
|
" counter.update(document)\n",
|
|
" return Vocab(counter, specials=['<unk>', '<pad>', '<bos>', '<eos>'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"vocab = build_vocab(dataset['train']['tokens'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"23627"
|
|
]
|
|
},
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"len(vocab.itos)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"15"
|
|
]
|
|
},
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"vocab['on']"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def data_process(dt):\n",
|
|
" return [ torch.tensor([vocab['<bos>']] +[vocab[token] for token in document ] + [vocab['<eos>']], dtype = torch.long) for document in dt]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def labels_process(dt):\n",
|
|
" return [ torch.tensor([0] + document + [0], dtype = torch.long) for document in dt]\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"train_tokens_ids = data_process(dataset['train']['tokens'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"test_tokens_ids = data_process(dataset['test']['tokens'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"validation_tokens_ids = data_process(dataset['validation']['tokens'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 12,
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"train_labels = labels_process(dataset['train']['ner_tags'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 13,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"validation_labels = labels_process(dataset['validation']['ner_tags'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 14,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"test_labels = labels_process(dataset['test']['ner_tags'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 15,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor([ 2, 966, 22409, 238, 773, 9, 4588, 212, 7686, 4,\n",
|
|
" 3])"
|
|
]
|
|
},
|
|
"execution_count": 15,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"train_tokens_ids[0]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 16,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"{'chunk_tags': [11, 21, 11, 12, 21, 22, 11, 12, 0],\n",
|
|
" 'id': '0',\n",
|
|
" 'ner_tags': [3, 0, 7, 0, 0, 0, 7, 0, 0],\n",
|
|
" 'pos_tags': [22, 42, 16, 21, 35, 37, 16, 21, 7],\n",
|
|
" 'tokens': ['EU',\n",
|
|
" 'rejects',\n",
|
|
" 'German',\n",
|
|
" 'call',\n",
|
|
" 'to',\n",
|
|
" 'boycott',\n",
|
|
" 'British',\n",
|
|
" 'lamb',\n",
|
|
" '.']}"
|
|
]
|
|
},
|
|
"execution_count": 16,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"dataset['train'][0]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 17,
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor([0, 3, 0, 7, 0, 0, 0, 7, 0, 0, 0])"
|
|
]
|
|
},
|
|
"execution_count": 17,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"train_labels[0]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 18,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def get_scores(y_true, y_pred):\n",
|
|
" acc_score = 0\n",
|
|
" tp = 0\n",
|
|
" fp = 0\n",
|
|
" selected_items = 0\n",
|
|
" relevant_items = 0 \n",
|
|
"\n",
|
|
" for p,t in zip(y_pred, y_true):\n",
|
|
" if p == t:\n",
|
|
" acc_score +=1\n",
|
|
"\n",
|
|
" if p > 0 and p == t:\n",
|
|
" tp +=1\n",
|
|
"\n",
|
|
" if p > 0:\n",
|
|
" selected_items += 1\n",
|
|
"\n",
|
|
" if t > 0 :\n",
|
|
" relevant_items +=1\n",
|
|
"\n",
|
|
" \n",
|
|
" \n",
|
|
" if selected_items == 0:\n",
|
|
" precision = 1.0\n",
|
|
" else:\n",
|
|
" precision = tp / selected_items\n",
|
|
" \n",
|
|
" \n",
|
|
" if relevant_items == 0:\n",
|
|
" recall = 1.0\n",
|
|
" else:\n",
|
|
" recall = tp / relevant_items\n",
|
|
" \n",
|
|
" \n",
|
|
" if precision + recall == 0.0 :\n",
|
|
" f1 = 0.0\n",
|
|
" else:\n",
|
|
" f1 = 2* precision * recall / (precision + recall)\n",
|
|
"\n",
|
|
" return precision, recall, f1"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 19,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"num_tags = max([max(x) for x in dataset['train']['ner_tags'] ]) + 1 "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 20,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class LSTM(torch.nn.Module):\n",
|
|
"\n",
|
|
" def __init__(self):\n",
|
|
" super(LSTM, self).__init__()\n",
|
|
" self.emb = torch.nn.Embedding(len(vocab.itos),100)\n",
|
|
" self.rec = torch.nn.LSTM(100, 256, 1, batch_first = True)\n",
|
|
" self.fc1 = torch.nn.Linear( 256 , 9)\n",
|
|
"\n",
|
|
" def forward(self, x):\n",
|
|
" emb = torch.relu(self.emb(x))\n",
|
|
" \n",
|
|
" lstm_output, (h_n, c_n) = self.rec(emb)\n",
|
|
" \n",
|
|
" out_weights = self.fc1(lstm_output)\n",
|
|
"\n",
|
|
" return out_weights"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 21,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"lstm = LSTM()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 22,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"criterion = torch.nn.CrossEntropyLoss()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 23,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"optimizer = torch.optim.Adam(lstm.parameters())"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 24,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def eval_model(dataset_tokens, dataset_labels, model):\n",
|
|
" Y_true = []\n",
|
|
" Y_pred = []\n",
|
|
" for i in tqdm(range(len(dataset_labels))):\n",
|
|
" batch_tokens = dataset_tokens[i].unsqueeze(0)\n",
|
|
" tags = list(dataset_labels[i].numpy())\n",
|
|
" Y_true += tags\n",
|
|
" \n",
|
|
" Y_batch_pred_weights = model(batch_tokens).squeeze(0)\n",
|
|
" Y_batch_pred = torch.argmax(Y_batch_pred_weights,1)\n",
|
|
" Y_pred += list(Y_batch_pred.numpy())\n",
|
|
" \n",
|
|
"\n",
|
|
" return get_scores(Y_true, Y_pred)\n",
|
|
" "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 25,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"NUM_EPOCHS = 5"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 26,
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "75184b632ce54ae690b3444778f44651",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "74a55a414fa948a3b251b89f780564d0",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n",
|
|
"(0.5068524970963996, 0.5072649075903755, 0.5070586184860281)\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "0c9c580076fb4ec48b7ea2f300878594",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "0be8681c67f64aca95ce5d3c44f10538",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n",
|
|
"(0.653649243957614, 0.6381494827385795, 0.6458063757205035)\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "2dec403004bb4ae298bc73553ea3f4bc",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "eebed0407ba343e29cf8c2d607f631dc",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n",
|
|
"(0.7140486069946651, 0.7001046146693014, 0.7070078647728607)\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "70792f22eea343c8916bcfcf9215c298",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "5d400bf1b656433ba2091cf750ec2d78",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n",
|
|
"(0.756327964151629, 0.725909566430315, 0.7408066429418744)\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "604c4fa13c03435d81bf68be37977d74",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "2f78871f366f4fd1b7de6c4be5303906",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n",
|
|
"(0.7963248522230789, 0.7203301174009067, 0.7564235581324383)\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"for i in range(NUM_EPOCHS):\n",
|
|
" lstm.train()\n",
|
|
" #for i in tqdm(range(500)):\n",
|
|
" for i in tqdm(range(len(train_labels))):\n",
|
|
" batch_tokens = train_tokens_ids[i].unsqueeze(0)\n",
|
|
" tags = train_labels[i].unsqueeze(1)\n",
|
|
" \n",
|
|
" \n",
|
|
" predicted_tags = lstm(batch_tokens)\n",
|
|
"\n",
|
|
" \n",
|
|
" optimizer.zero_grad()\n",
|
|
" loss = criterion(predicted_tags.squeeze(0),tags.squeeze(1))\n",
|
|
" \n",
|
|
" loss.backward()\n",
|
|
" optimizer.step()\n",
|
|
" \n",
|
|
" lstm.eval()\n",
|
|
" print(eval_model(validation_tokens_ids, validation_labels, lstm))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 27,
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "5159f7a61c3a439bab45573f15ea55b2",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"(0.7963248522230789, 0.7203301174009067, 0.7564235581324383)"
|
|
]
|
|
},
|
|
"execution_count": 27,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"eval_model(validation_tokens_ids, validation_labels, lstm)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 28,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "4b604bbb796f4d4cb99528fad98cfdff",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=3453.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"(0.7450810185185185, 0.6348619329388561, 0.685569755058573)"
|
|
]
|
|
},
|
|
"execution_count": 28,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"eval_model(test_tokens_ids, test_labels, lstm)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 29,
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"14041"
|
|
]
|
|
},
|
|
"execution_count": 29,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"len(train_tokens_ids)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## pytania\n",
|
|
"\n",
|
|
"- co zrobić z trenowaniem na batchach > 1 ?\n",
|
|
"- co zrobić, żeby sieć uwzględniała następne tokeny, a nie tylko poprzednie?\n",
|
|
"- w jaki sposób wykorzystać taką sieć do zadania zwykłej klasyfikacji?"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Zadanie na zajęcia ( 20 minut)\n",
|
|
"\n",
|
|
"zmodyfikować sieć tak, żeby była używała dwuwarstwowej, dwukierunkowej warstwy GRU oraz dropoutu. Dropout ma nałożony na embeddingi.\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 30,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"\n",
|
|
"class GRU(torch.nn.Module):\n",
|
|
"\n",
|
|
" def __init__(self):\n",
|
|
" super(GRU, self).__init__()\n",
|
|
" self.emb = torch.nn.Embedding(len(vocab.itos),100)\n",
|
|
" self.dropout = torch.nn.Dropout(0.2)\n",
|
|
" self.rec = torch.nn.GRU(100, 256, 2, batch_first = True, bidirectional = True)\n",
|
|
" self.fc1 = torch.nn.Linear(2* 256 , 9)\n",
|
|
" \n",
|
|
" def forward(self, x):\n",
|
|
" emb = torch.relu(self.emb(x))\n",
|
|
" emb = self.dropout(emb)\n",
|
|
" \n",
|
|
" gru_output, h_n = self.rec(emb)\n",
|
|
" \n",
|
|
" out_weights = self.fc1(gru_output)\n",
|
|
"\n",
|
|
" return out_weights"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 31,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"gru = GRU()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 32,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"criterion = torch.nn.CrossEntropyLoss()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 33,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"optimizer = torch.optim.Adam(gru.parameters())"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 34,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"NUM_EPOCHS = 5"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 35,
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "a0b15b129c294730ab2a6035b9a98b47",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "b7b3bc93dc7349949b59f97751ebdec2",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n",
|
|
"(0.6431379891406104, 0.39927932116703474, 0.49268502581755597)\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "0fa832ed33e3451ebc241046ea299d2e",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "953a62a22eb44de4a33b0e1c53a472f0",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n",
|
|
"(0.6638910917261432, 0.5838660932232942, 0.6213123879027769)\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "5efce69cb2e246d2adababcc18780083",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "cb7cde559dd544b0961fa4c855b856d8",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n",
|
|
"(0.6697936210131332, 0.7054515866558178, 0.6871603260869565)\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "d7211ba79faa4de3aeafff4ef39bbaf1",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "2a8dbd5393fa4ee2a8ee72738efa2a57",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n",
|
|
"(0.7091097308488613, 0.7166104847146344, 0.7128403769439787)\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "84d0b4b0e1ec4c1a88aff0698836a362",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "31a2f0a285e44a55a7b583a81511dbc2",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n",
|
|
"(0.7708963348252087, 0.740788097175404, 0.755542382928275)\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"for i in range(NUM_EPOCHS):\n",
|
|
" gru.train()\n",
|
|
" #for i in tqdm(range(50)):\n",
|
|
" for i in tqdm(range(len(train_labels))):\n",
|
|
" batch_tokens = train_tokens_ids[i].unsqueeze(0)\n",
|
|
" tags = train_labels[i].unsqueeze(1)\n",
|
|
" \n",
|
|
" \n",
|
|
" predicted_tags = gru(batch_tokens)\n",
|
|
"\n",
|
|
" \n",
|
|
" optimizer.zero_grad()\n",
|
|
" loss = criterion(predicted_tags.squeeze(0),tags.squeeze(1))\n",
|
|
" \n",
|
|
" loss.backward()\n",
|
|
" optimizer.step()\n",
|
|
" \n",
|
|
" \n",
|
|
" gru.eval()\n",
|
|
" print(eval_model(validation_tokens_ids, validation_labels, gru))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Zadanie domowe\n",
|
|
"\n",
|
|
"\n",
|
|
"- sklonować repozytorium https://git.wmi.amu.edu.pl/kubapok/en-ner-conll-2003\n",
|
|
"- stworzyć model seq labelling bazujący na sieci neuronowej opisanej w punkcie niżej (można bazować na tym jupyterze lub nie).\n",
|
|
"- model sieci to GRU (o dowolnych parametrach) + CRF w pytorchu korzystając z modułu CRF z poprzednich zajęć- - stworzyć predykcje w plikach dev-0/out.tsv oraz test-A/out.tsv\n",
|
|
"- wynik fscore sprawdzony za pomocą narzędzia geval (patrz poprzednie zadanie) powinien wynosić conajmniej 0.65\n",
|
|
"- proszę umieścić predykcję oraz skrypty generujące (w postaci tekstowej a nie jupyter) w repo, a w MS TEAMS umieścić link do swojego repo\n",
|
|
"termin 22.06, 60 punktów, za najlepszy wynik- 100 punktów\n",
|
|
" "
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.8.5"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 4
|
|
}
|