570 lines
13 KiB
Plaintext
570 lines
13 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"![Logo 1](https://git.wmi.amu.edu.pl/AITech/Szablon/raw/branch/master/Logotyp_AITech1.jpg)\n",
|
|
"<div class=\"alert alert-block alert-info\">\n",
|
|
"<h1> Ekstrakcja informacji </h1>\n",
|
|
"<h2> 10. <i>CRF</i> [ćwiczenia]</h2> \n",
|
|
"<h3> Jakub Pokrywka (2021)</h3>\n",
|
|
"</div>\n",
|
|
"\n",
|
|
"![Logo 2](https://git.wmi.amu.edu.pl/AITech/Szablon/raw/branch/master/Logotyp_AITech2.jpg)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Podejście softmax z embeddingami na przykładzie NER"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"source": [
|
|
"https://pytorch-crf.readthedocs.io/en/stable/"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"https://www.aclweb.org/anthology/W03-0419.pdf"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"conda env export -n <env-name> > environment.yml\n",
|
|
" \n",
|
|
" \n",
|
|
"conda env create -f path/to/environment.yml"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Requirement already satisfied: pytorch-crf in /home/kuba/anaconda3/envs/zajeciaei/lib/python3.10/site-packages (0.7.2)\r\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"!pip install pytorch-crf"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import numpy as np\n",
|
|
"import gensim\n",
|
|
"import torch\n",
|
|
"import pandas as pd\n",
|
|
"import seaborn as sns\n",
|
|
"import torchtext\n",
|
|
"from sklearn.model_selection import train_test_split\n",
|
|
"\n",
|
|
"from datasets import load_dataset\n",
|
|
"from torchtext.vocab import Vocab\n",
|
|
"from collections import Counter\n",
|
|
"\n",
|
|
"from sklearn.datasets import fetch_20newsgroups\n",
|
|
"# https://scikit-learn.org/0.19/datasets/twenty_newsgroups.html\n",
|
|
"\n",
|
|
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
|
|
"from sklearn.metrics import accuracy_score\n",
|
|
"\n",
|
|
"from tqdm.notebook import tqdm\n",
|
|
"\n",
|
|
"import torch\n",
|
|
"from torchcrf import CRF"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {
|
|
"scrolled": false
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Reusing dataset conll2003 (/home/kuba/.cache/huggingface/datasets/conll2003/conll2003/1.0.0/63f4ebd1bcb7148b1644497336fd74643d4ce70123334431a3c053b7ee4e96ee)\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "960f4cf0de594e48ad7a84740cf301a3",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
" 0%| | 0/3 [00:00<?, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"dataset = load_dataset(\"conll2003\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def build_vocab(dataset):\n",
|
|
" counter = Counter()\n",
|
|
" for document in dataset:\n",
|
|
" counter.update(document)\n",
|
|
" vocab = torchtext.vocab.vocab(counter, specials=['<unk>', '<pad>', '<bos>', '<eos>'])\n",
|
|
" vocab.set_default_index(0)\n",
|
|
" return vocab"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"vocab = build_vocab(dataset['train']['tokens'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"21"
|
|
]
|
|
},
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"vocab['on']"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def data_process(dt):\n",
|
|
" return [ torch.tensor([vocab['<bos>']] +[vocab[token] for token in document ] + [vocab['<eos>']], dtype = torch.long) for document in dt]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def labels_process(dt):\n",
|
|
" return [ torch.tensor([0] + document + [0], dtype = torch.long) for document in dt]\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"train_tokens_ids = data_process(dataset['train']['tokens'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"test_tokens_ids = data_process(dataset['test']['tokens'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"validation_tokens_ids = data_process(dataset['validation']['tokens'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 12,
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"train_labels = labels_process(dataset['train']['ner_tags'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 13,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"validation_labels = labels_process(dataset['validation']['ner_tags'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 14,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"test_labels = labels_process(dataset['test']['ner_tags'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 15,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor([ 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 3])"
|
|
]
|
|
},
|
|
"execution_count": 15,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"train_tokens_ids[0]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 16,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def get_scores(y_true, y_pred):\n",
|
|
" acc_score = 0\n",
|
|
" tp = 0\n",
|
|
" fp = 0\n",
|
|
" selected_items = 0\n",
|
|
" relevant_items = 0 \n",
|
|
"\n",
|
|
" for p,t in zip(y_pred, y_true):\n",
|
|
" if p == t:\n",
|
|
" acc_score +=1\n",
|
|
"\n",
|
|
" if p > 0 and p == t:\n",
|
|
" tp +=1\n",
|
|
"\n",
|
|
" if p > 0:\n",
|
|
" selected_items += 1\n",
|
|
"\n",
|
|
" if t > 0 :\n",
|
|
" relevant_items +=1\n",
|
|
"\n",
|
|
" \n",
|
|
" \n",
|
|
" if selected_items == 0:\n",
|
|
" precision = 1.0\n",
|
|
" else:\n",
|
|
" precision = tp / selected_items\n",
|
|
" \n",
|
|
" \n",
|
|
" if relevant_items == 0:\n",
|
|
" recall = 1.0\n",
|
|
" else:\n",
|
|
" recall = tp / relevant_items\n",
|
|
" \n",
|
|
" \n",
|
|
" if precision + recall == 0.0 :\n",
|
|
" f1 = 0.0\n",
|
|
" else:\n",
|
|
" f1 = 2* precision * recall / (precision + recall)\n",
|
|
"\n",
|
|
" return precision, recall, f1"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 17,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"num_tags = max([max(x) for x in dataset['train']['ner_tags'] if x]) + 1 "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 18,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class FF(torch.nn.Module):\n",
|
|
"\n",
|
|
" def __init__(self,):\n",
|
|
" super(FF, self).__init__()\n",
|
|
" self.emb = torch.nn.Embedding(23627,200)\n",
|
|
" self.fc1 = torch.nn.Linear(200,num_tags)\n",
|
|
" \n",
|
|
"\n",
|
|
" def forward(self, x):\n",
|
|
" x = self.emb(x)\n",
|
|
" x = self.fc1(x)\n",
|
|
" return x"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 19,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"ff = FF()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 20,
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"crf = CRF(num_tags)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 21,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"params = list(ff.parameters()) + list(crf.parameters())\n",
|
|
"\n",
|
|
"optimizer = torch.optim.Adam(params)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 22,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def eval_model(dataset_tokens, dataset_labels):\n",
|
|
" Y_true = []\n",
|
|
" Y_pred = []\n",
|
|
" ff.eval()\n",
|
|
" crf.eval()\n",
|
|
" for i in tqdm(range(len(dataset_labels))):\n",
|
|
" batch_tokens = dataset_tokens[i]\n",
|
|
" tags = list(dataset_labels[i].numpy())\n",
|
|
" emissions = ff(batch_tokens).unsqueeze(1)\n",
|
|
" Y_pred += crf.decode(emissions)[0]\n",
|
|
" Y_true += tags\n",
|
|
"\n",
|
|
" return get_scores(Y_true, Y_pred)\n",
|
|
" "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 23,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"NUM_EPOCHS = 4"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"scrolled": false
|
|
},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "caddabb06f894f529ba7143ce62b8c2e",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
" 0%| | 0/14042 [00:00<?, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"/home/kuba/anaconda3/envs/zajeciaei/lib/python3.10/site-packages/torchcrf/__init__.py:249: UserWarning: where received a uint8 condition tensor. This behavior is deprecated and will be removed in a future version of PyTorch. Use a boolean condition instead. (Triggered internally at /opt/conda/conda-bld/pytorch_1646755897462/work/aten/src/ATen/native/TensorCompare.cpp:333.)\n",
|
|
" score = torch.where(mask[i].unsqueeze(1), next_score, score)\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"for i in range(NUM_EPOCHS):\n",
|
|
" ff.train()\n",
|
|
" crf.train()\n",
|
|
" for i in tqdm(range(len(train_labels))):\n",
|
|
" batch_tokens = train_tokens_ids[i]\n",
|
|
" tags = train_labels[i].unsqueeze(1)\n",
|
|
" emissions = ff(batch_tokens).unsqueeze(1)\n",
|
|
"\n",
|
|
" optimizer.zero_grad()\n",
|
|
" loss = -crf(emissions,tags)\n",
|
|
" #import pdb; pdb.set_trace()\n",
|
|
" loss.backward()\n",
|
|
" optimizer.step()\n",
|
|
" \n",
|
|
" ff.eval()\n",
|
|
" crf.eval()\n",
|
|
" print(eval_model(validation_tokens_ids, validation_labels))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"dir(crf)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"crf.transitions"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"list(crf.parameters())"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"eval_model(validation_tokens_ids, validation_labels)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"eval_model(test_tokens_ids, test_labels)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"len(train_tokens_ids)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Zadanie domowe\n",
|
|
"\n",
|
|
"- en-ner-conll-2003\n",
|
|
"- stworzyć klasyfikator bazujący na sieci neuronowej feed forward w pytorchu + CRF (można bazować na tym jupyterze lub nie).\n",
|
|
"- sieć feedforward powinna obejmować aktualne słowo, poprzednie i następne + dodatkowe cechy (np. długość wyrazu, czy wyraz zaczyna się od wielkiej litery, stemmming słowa, czy zawiera cyfrę)\n",
|
|
"- stworzyć predykcje w plikach dev-0/out.tsv oraz test-A/out.tsv\n",
|
|
"- wynik fscore sprawdzony za pomocą narzędzia geval (patrz poprzednie zadanie) powinien wynosić conajmniej 0.65\n",
|
|
"- 60 punktów, za najlepszy wynik- 100 punktów\n"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"author": "Jakub Pokrywka",
|
|
"email": "kubapok@wmi.amu.edu.pl",
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"lang": "pl",
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.10.4"
|
|
},
|
|
"subtitle": "10.CRF[ćwiczenia]",
|
|
"title": "Ekstrakcja informacji",
|
|
"year": "2021"
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 4
|
|
}
|