aitech-eks-pub-22/cw/10_CRF.ipynb

Ekstrakcja informacji

10. CRF [ćwiczenia]

Jakub Pokrywka (2021)

Podejście softmax z embeddingami na przykładzie NER

https://pytorch-crf.readthedocs.io/en/stable/

https://www.aclweb.org/anthology/W03-0419.pdf

conda env export -n > environment.yml

conda env create -f path/to/environment.yml

!pip install pytorch-crf

Requirement already satisfied: pytorch-crf in /home/kuba/anaconda3/envs/zajeciaei/lib/python3.10/site-packages (0.7.2)

import numpy as np
import gensim
import torch
import pandas as pd
import seaborn as sns
import torchtext
from sklearn.model_selection import train_test_split

from datasets import load_dataset
from torchtext.vocab import Vocab
from collections import Counter

from sklearn.datasets import fetch_20newsgroups
# https://scikit-learn.org/0.19/datasets/twenty_newsgroups.html

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import accuracy_score

from tqdm.notebook import tqdm

import torch
from torchcrf import CRF

dataset = load_dataset("conll2003")

Reusing dataset conll2003 (/home/kuba/.cache/huggingface/datasets/conll2003/conll2003/1.0.0/63f4ebd1bcb7148b1644497336fd74643d4ce70123334431a3c053b7ee4e96ee)

  0%|          | 0/3 [00:00<?, ?it/s]

def build_vocab(dataset):
    counter = Counter()
    for document in dataset:
        counter.update(document)
    vocab = torchtext.vocab.vocab(counter, specials=['<unk>', '<pad>', '<bos>', '<eos>'])
    vocab.set_default_index(0)
    return vocab

vocab = build_vocab(dataset['train']['tokens'])

vocab['on']

21

def data_process(dt):
    return [ torch.tensor([vocab['<bos>']] +[vocab[token]  for token in  document ] + [vocab['<eos>']], dtype = torch.long) for document in dt]

def labels_process(dt):
    return [ torch.tensor([0] + document + [0], dtype = torch.long) for document in dt]

train_tokens_ids = data_process(dataset['train']['tokens'])

test_tokens_ids = data_process(dataset['test']['tokens'])

validation_tokens_ids =  data_process(dataset['validation']['tokens'])

train_labels = labels_process(dataset['train']['ner_tags'])

validation_labels = labels_process(dataset['validation']['ner_tags'])

test_labels = labels_process(dataset['test']['ner_tags'])

train_tokens_ids[0]

tensor([ 2,  4,  5,  6,  7,  8,  9, 10, 11, 12,  3])

def get_scores(y_true, y_pred):
    acc_score = 0
    tp = 0
    fp = 0
    selected_items = 0
    relevant_items = 0 

    for p,t in zip(y_pred, y_true):
        if p == t:
            acc_score +=1

        if p > 0 and p == t:
            tp +=1

        if p > 0:
            selected_items += 1

        if t > 0 :
            relevant_items +=1

            
            
    if selected_items == 0:
        precision = 1.0
    else:
        precision = tp / selected_items
        
            
    if relevant_items == 0:
        recall = 1.0
    else:
        recall = tp / relevant_items
    
    
    if precision + recall == 0.0 :
        f1 = 0.0
    else:
        f1 = 2* precision * recall  / (precision + recall)

    return precision, recall, f1

num_tags = max([max(x) for x in dataset['train']['ner_tags'] if x]) + 1 

class FF(torch.nn.Module):

    def __init__(self,):
        super(FF, self).__init__()
        self.emb = torch.nn.Embedding(23627,200)
        self.fc1 = torch.nn.Linear(200,num_tags)
       

    def forward(self, x):
        x = self.emb(x)
        x = self.fc1(x)
        return x

ff = FF()

crf = CRF(num_tags)

params = list(ff.parameters()) + list(crf.parameters())

optimizer = torch.optim.Adam(params)

def eval_model(dataset_tokens, dataset_labels):
    Y_true = []
    Y_pred = []
    ff.eval()
    crf.eval()
    for i in tqdm(range(len(dataset_labels))):
        batch_tokens = dataset_tokens[i]
        tags = list(dataset_labels[i].numpy())
        emissions = ff(batch_tokens).unsqueeze(1)
        Y_pred += crf.decode(emissions)[0]
        Y_true += tags

    return get_scores(Y_true, Y_pred)
        

NUM_EPOCHS = 4

for i in range(NUM_EPOCHS):
    ff.train()
    crf.train()
    for i in tqdm(range(len(train_labels))):
        batch_tokens = train_tokens_ids[i]
        tags = train_labels[i].unsqueeze(1)
        emissions = ff(batch_tokens).unsqueeze(1)

        optimizer.zero_grad()
        loss  = -crf(emissions,tags)
        #import pdb; pdb.set_trace()
        loss.backward()
        optimizer.step()
        
    ff.eval()
    crf.eval()
    print(eval_model(validation_tokens_ids, validation_labels))

  0%|          | 0/14042 [00:00<?, ?it/s]

/home/kuba/anaconda3/envs/zajeciaei/lib/python3.10/site-packages/torchcrf/__init__.py:249: UserWarning: where received a uint8 condition tensor. This behavior is deprecated and will be removed in a future version of PyTorch. Use a boolean condition instead. (Triggered internally at  /opt/conda/conda-bld/pytorch_1646755897462/work/aten/src/ATen/native/TensorCompare.cpp:333.)
  score = torch.where(mask[i].unsqueeze(1), next_score, score)

dir(crf)

crf.transitions

list(crf.parameters())

eval_model(validation_tokens_ids, validation_labels)

eval_model(test_tokens_ids, test_labels)

len(train_tokens_ids)

Zadanie domowe

• en-ner-conll-2003
• stworzyć klasyfikator bazujący na sieci neuronowej feed forward w pytorchu + CRF (można bazować na tym jupyterze lub nie).
• sieć feedforward powinna obejmować aktualne słowo, poprzednie i następne + dodatkowe cechy (np. długość wyrazu, czy wyraz zaczyna się od wielkiej litery, stemmming słowa, czy zawiera cyfrę)
• stworzyć predykcje w plikach dev-0/out.tsv oraz test-A/out.tsv
• wynik fscore sprawdzony za pomocą narzędzia geval (patrz poprzednie zadanie) powinien wynosić conajmniej 0.65
• 60 punktów, za najlepszy wynik- 100 punktów