kubapok/aitech-eks-pub-22
4
5
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
55f0bea16b
aitech-eks-pub-22/cw/10_CRF.ipynb
Jakub Pokrywka 55f0bea16b 10
2022-06-01 09:50:08 +02:00

20 KiB
Raw Blame History

Logo 1

Ekstrakcja informacji

10. CRF [ćwiczenia]

Jakub Pokrywka (2021)

Logo 2

Podejście softmax z embeddingami na przykładzie NER

https://pytorch-crf.readthedocs.io/en/stable/

https://www.aclweb.org/anthology/W03-0419.pdf

!pip install pytorch-crf
Requirement already satisfied: pytorch-crf in /home/kuba/anaconda3/envs/zajeciaei/lib/python3.10/site-packages (0.7.2)

import numpy as np
import gensim
import torch
import pandas as pd
import seaborn as sns
import torchtext
from sklearn.model_selection import train_test_split

from datasets import load_dataset
from torchtext.vocab import Vocab
from collections import Counter

from sklearn.datasets import fetch_20newsgroups
# https://scikit-learn.org/0.19/datasets/twenty_newsgroups.html

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import accuracy_score

from tqdm.notebook import tqdm

import torch
from torchcrf import CRF
dataset = load_dataset("conll2003")
Reusing dataset conll2003 (/home/kuba/.cache/huggingface/datasets/conll2003/conll2003/1.0.0/63f4ebd1bcb7148b1644497336fd74643d4ce70123334431a3c053b7ee4e96ee)

  0%|          | 0/3 [00:00<?, ?it/s]
def build_vocab(dataset):
    counter = Counter()
    for document in dataset:
        counter.update(document)
    vocab = torchtext.vocab.vocab(counter, specials=['<unk>', '<pad>', '<bos>', '<eos>'])
    vocab.set_default_index(0)
    return vocab
vocab = build_vocab(dataset['train']['tokens'])
vocab['on']
21
def data_process(dt):
    return [ torch.tensor([vocab['<bos>']] +[vocab[token]  for token in  document ] + [vocab['<eos>']], dtype = torch.long) for document in dt]
def labels_process(dt):
    return [ torch.tensor([0] + document + [0], dtype = torch.long) for document in dt]

train_tokens_ids = data_process(dataset['train']['tokens'])
test_tokens_ids = data_process(dataset['test']['tokens'])
validation_tokens_ids =  data_process(dataset['validation']['tokens'])
train_labels = labels_process(dataset['train']['ner_tags'])
validation_labels = labels_process(dataset['validation']['ner_tags'])
test_labels = labels_process(dataset['test']['ner_tags'])
train_tokens_ids[0]
tensor([ 2,  4,  5,  6,  7,  8,  9, 10, 11, 12,  3])
def get_scores(y_true, y_pred):
    acc_score = 0
    tp = 0
    fp = 0
    selected_items = 0
    relevant_items = 0 

    for p,t in zip(y_pred, y_true):
        if p == t:
            acc_score +=1

        if p > 0 and p == t:
            tp +=1

        if p > 0:
            selected_items += 1

        if t > 0 :
            relevant_items +=1

            
            
    if selected_items == 0:
        precision = 1.0
    else:
        precision = tp / selected_items
        
            
    if relevant_items == 0:
        recall = 1.0
    else:
        recall = tp / relevant_items
    
    
    if precision + recall == 0.0 :
        f1 = 0.0
    else:
        f1 = 2* precision * recall  / (precision + recall)

    return precision, recall, f1
num_tags = max([max(x) for x in dataset['train']['ner_tags'] if x]) + 1 
class FF(torch.nn.Module):

    def __init__(self,):
        super(FF, self).__init__()
        self.emb = torch.nn.Embedding(23627,200)
        self.fc1 = torch.nn.Linear(200,num_tags)
       

    def forward(self, x):
        x = self.emb(x)
        x = self.fc1(x)
        return x
ff = FF()
crf = CRF(num_tags)
params = list(ff.parameters()) + list(crf.parameters())

optimizer = torch.optim.Adam(params)
def eval_model(dataset_tokens, dataset_labels):
    Y_true = []
    Y_pred = []
    ff.eval()
    crf.eval()
    for i in tqdm(range(len(dataset_labels))):
        batch_tokens = dataset_tokens[i]
        tags = list(dataset_labels[i].numpy())
        emissions = ff(batch_tokens).unsqueeze(1)
        Y_pred += crf.decode(emissions)[0]
        Y_true += tags

    return get_scores(Y_true, Y_pred)
        
NUM_EPOCHS = 4
for i in range(NUM_EPOCHS):
    ff.train()
    crf.train()
    for i in tqdm(range(len(train_labels))):
        batch_tokens = train_tokens_ids[i]
        tags = train_labels[i].unsqueeze(1)
        emissions = ff(batch_tokens).unsqueeze(1)

        optimizer.zero_grad()
        loss  = -crf(emissions,tags)
        import pdb; pdb.set_trace()
        loss.backward()
        optimizer.step()
        
    ff.eval()
    crf.eval()
    print(eval_model(validation_tokens_ids, validation_labels))
  0%|          | 0/14042 [00:00<?, ?it/s]
> /tmp/ipykernel_306568/4048919537.py(12)<cell line: 1>()
     10         loss  = -crf(emissions,tags)
     11         import pdb; pdb.set_trace()
---> 12         loss.backward()
     13         optimizer.step()
     14 

ipdb> batch_tokens
tensor([ 2,  4,  5,  6,  7,  8,  9, 10, 11, 12,  3])
ipdb> tags.shape
torch.Size([11, 1])
ipdb> tags
tensor([[0],
        [3],
        [0],
        [7],
        [0],
        [0],
        [0],
        [7],
        [0],
        [0],
        [0]])

dir(crf)

['T_destination',
 '__annotations__',
 '__call__',
 '__class__',
 '__delattr__',
 '__dict__',
 '__dir__',
 '__doc__',
 '__eq__',
 '__format__',
 '__ge__',
 '__getattr__',
 '__getattribute__',
 '__gt__',
 '__hash__',
 '__init__',
 '__init_subclass__',
 '__le__',
 '__lt__',
 '__module__',
 '__ne__',
 '__new__',
 '__reduce__',
 '__reduce_ex__',
 '__repr__',
 '__setattr__',
 '__setstate__',
 '__sizeof__',
 '__str__',
 '__subclasshook__',
 '__weakref__',
 '_apply',
 '_backward_hooks',
 '_buffers',
 '_call_impl',
 '_compute_normalizer',
 '_compute_score',
 '_forward_hooks',
 '_forward_pre_hooks',
 '_get_backward_hooks',
 '_get_name',
 '_is_full_backward_hook',
 '_load_from_state_dict',
 '_load_state_dict_pre_hooks',
 '_maybe_warn_non_full_backward_hook',
 '_modules',
 '_named_members',
 '_non_persistent_buffers_set',
 '_parameters',
 '_register_load_state_dict_pre_hook',
 '_register_state_dict_hook',
 '_replicate_for_data_parallel',
 '_save_to_state_dict',
 '_slow_forward',
 '_state_dict_hooks',
 '_validate',
 '_version',
 '_viterbi_decode',
 'add_module',
 'apply',
 'batch_first',
 'bfloat16',
 'buffers',
 'children',
 'cpu',
 'cuda',
 'decode',
 'double',
 'dump_patches',
 'end_transitions',
 'eval',
 'extra_repr',
 'float',
 'forward',
 'get_buffer',
 'get_extra_state',
 'get_parameter',
 'get_submodule',
 'half',
 'load_state_dict',
 'modules',
 'named_buffers',
 'named_children',
 'named_modules',
 'named_parameters',
 'num_tags',
 'parameters',
 'register_backward_hook',
 'register_buffer',
 'register_forward_hook',
 'register_forward_pre_hook',
 'register_full_backward_hook',
 'register_module',
 'register_parameter',
 'requires_grad_',
 'reset_parameters',
 'set_extra_state',
 'share_memory',
 'start_transitions',
 'state_dict',
 'to',
 'to_empty',
 'train',
 'training',
 'transitions',
 'type',
 'xpu',
 'zero_grad']
crf.transitions
Parameter containing:
tensor([[ 0.1427,  0.0082, -0.0852, -0.0714, -0.0514,  0.0753,  0.0389,  0.0018,
         -0.0806],
        [-0.0809, -0.0508,  0.0520, -0.0619,  0.0181, -0.0729, -0.1430, -0.1055,
          0.0384],
        [-0.0011, -0.1476,  0.0425, -0.0081, -0.1181, -0.0098, -0.0567,  0.0311,
         -0.0696],
        [-0.0443, -0.0741,  0.0463, -0.0967, -0.0403, -0.0243,  0.0098, -0.0063,
         -0.0811],
        [ 0.0632, -0.1175, -0.0992,  0.0198,  0.0310, -0.0059,  0.0191, -0.1303,
         -0.1423],
        [ 0.0029,  0.0296,  0.0152, -0.0418, -0.1068, -0.0920, -0.0380,  0.0461,
          0.0167],
        [-0.1167, -0.0559, -0.0428, -0.0115, -0.1006, -0.1511,  0.0035, -0.0273,
         -0.1201],
        [-0.0378,  0.0481, -0.1474, -0.0154,  0.0347, -0.0392, -0.0755, -0.1227,
          0.0448],
        [-0.0383, -0.0402,  0.0054,  0.0145, -0.1353, -0.0460,  0.0257, -0.0322,
          0.0023]], requires_grad=True)
list(crf.parameters())
[Parameter containing:
 tensor([-0.0432, -0.1150, -0.1045, -0.0779, -0.0858,  0.0287, -0.1437, -0.1446,
          0.0335], requires_grad=True),
 Parameter containing:
 tensor([ 0.0838, -0.0097, -0.1136,  0.0010, -0.1177,  0.0225,  0.0292, -0.0837,
         -0.1063], requires_grad=True),
 Parameter containing:
 tensor([[ 0.1427,  0.0082, -0.0852, -0.0714, -0.0514,  0.0753,  0.0389,  0.0018,
          -0.0806],
         [-0.0809, -0.0508,  0.0520, -0.0619,  0.0181, -0.0729, -0.1430, -0.1055,
           0.0384],
         [-0.0011, -0.1476,  0.0425, -0.0081, -0.1181, -0.0098, -0.0567,  0.0311,
          -0.0696],
         [-0.0443, -0.0741,  0.0463, -0.0967, -0.0403, -0.0243,  0.0098, -0.0063,
          -0.0811],
         [ 0.0632, -0.1175, -0.0992,  0.0198,  0.0310, -0.0059,  0.0191, -0.1303,
          -0.1423],
         [ 0.0029,  0.0296,  0.0152, -0.0418, -0.1068, -0.0920, -0.0380,  0.0461,
           0.0167],
         [-0.1167, -0.0559, -0.0428, -0.0115, -0.1006, -0.1511,  0.0035, -0.0273,
          -0.1201],
         [-0.0378,  0.0481, -0.1474, -0.0154,  0.0347, -0.0392, -0.0755, -0.1227,
           0.0448],
         [-0.0383, -0.0402,  0.0054,  0.0145, -0.1353, -0.0460,  0.0257, -0.0322,
           0.0023]], requires_grad=True)]
eval_model(validation_tokens_ids, validation_labels)
eval_model(test_tokens_ids, test_labels)
len(train_tokens_ids)

Zadanie domowe

  • en-ner-conll-2003
  • stworzyć klasyfikator bazujący na sieci neuronowej feed forward w pytorchu + CRF (można bazować na tym jupyterze lub nie).
  • sieć feedforward powinna obejmować aktualne słowo, poprzednie i następne + dodatkowe cechy (np. długość wyrazu, czy wyraz zaczyna się od wielkiej litery, stemmming słowa, czy zawiera cyfrę)
  • stworzyć predykcje w plikach dev-0/out.tsv oraz test-A/out.tsv
  • wynik fscore sprawdzony za pomocą narzędzia geval (patrz poprzednie zadanie) powinien wynosić conajmniej 0.65
  • 60 punktów, za najlepszy wynik- 100 punktów