aitech-eks-pub/cw/11_NER_RNN_ODPOWIEDZI.ipynb
2021-09-27 12:34:44 +02:00

42 KiB

Logo 1

Ekstrakcja informacji

11. NER RNN [ćwiczenia]

Jakub Pokrywka (2021)

Logo 2

Podejście softmax z embeddingami na przykładzie NER

import numpy as np
import gensim
import torch
import pandas as pd
import seaborn as sns
from sklearn.model_selection import train_test_split

from datasets import load_dataset
from torchtext.vocab import Vocab
from collections import Counter

from sklearn.datasets import fetch_20newsgroups
# https://scikit-learn.org/0.19/datasets/twenty_newsgroups.html

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import accuracy_score

from tqdm.notebook import tqdm

import torch
/media/kuba/ssdsam/anaconda3/lib/python3.8/site-packages/gensim/similarities/__init__.py:15: UserWarning: The gensim.similarities.levenshtein submodule is disabled, because the optional Levenshtein package <https://pypi.org/project/python-Levenshtein/> is unavailable. Install Levenhstein (e.g. `pip install python-Levenshtein`) to suppress this warning.
  warnings.warn(msg)
dataset = load_dataset("conll2003")
Reusing dataset conll2003 (/home/kuba/.cache/huggingface/datasets/conll2003/conll2003/1.0.0/40e7cb6bcc374f7c349c83acd1e9352a4f09474eb691f64f364ee62eb65d0ca6)
def build_vocab(dataset):
    counter = Counter()
    for document in dataset:
        counter.update(document)
    return Vocab(counter, specials=['<unk>', '<pad>', '<bos>', '<eos>'])
vocab = build_vocab(dataset['train']['tokens'])
len(vocab.itos)
23627
vocab['on']
15
def data_process(dt):
    return [ torch.tensor([vocab['<bos>']] +[vocab[token]  for token in  document ] + [vocab['<eos>']], dtype = torch.long) for document in dt]
def labels_process(dt):
    return [ torch.tensor([0] + document + [0], dtype = torch.long) for document in dt]
train_tokens_ids = data_process(dataset['train']['tokens'])
test_tokens_ids = data_process(dataset['test']['tokens'])
validation_tokens_ids =  data_process(dataset['validation']['tokens'])
train_labels = labels_process(dataset['train']['ner_tags'])
validation_labels = labels_process(dataset['validation']['ner_tags'])
test_labels = labels_process(dataset['test']['ner_tags'])
train_tokens_ids[0]
tensor([    2,   966, 22409,   238,   773,     9,  4588,   212,  7686,     4,
            3])
dataset['train'][0]
{'chunk_tags': [11, 21, 11, 12, 21, 22, 11, 12, 0],
 'id': '0',
 'ner_tags': [3, 0, 7, 0, 0, 0, 7, 0, 0],
 'pos_tags': [22, 42, 16, 21, 35, 37, 16, 21, 7],
 'tokens': ['EU',
  'rejects',
  'German',
  'call',
  'to',
  'boycott',
  'British',
  'lamb',
  '.']}
train_labels[0]
tensor([0, 3, 0, 7, 0, 0, 0, 7, 0, 0, 0])
def get_scores(y_true, y_pred):
    acc_score = 0
    tp = 0
    fp = 0
    selected_items = 0
    relevant_items = 0 

    for p,t in zip(y_pred, y_true):
        if p == t:
            acc_score +=1

        if p > 0 and p == t:
            tp +=1

        if p > 0:
            selected_items += 1

        if t > 0 :
            relevant_items +=1

            
            
    if selected_items == 0:
        precision = 1.0
    else:
        precision = tp / selected_items
        
            
    if relevant_items == 0:
        recall = 1.0
    else:
        recall = tp / relevant_items
    
    
    if precision + recall == 0.0 :
        f1 = 0.0
    else:
        f1 = 2* precision * recall  / (precision + recall)

    return precision, recall, f1
num_tags = max([max(x) for x in dataset['train']['ner_tags'] ]) + 1 
class LSTM(torch.nn.Module):

    def __init__(self):
        super(LSTM, self).__init__()
        self.emb = torch.nn.Embedding(len(vocab.itos),100)
        self.rec = torch.nn.LSTM(100, 256, 1, batch_first = True)
        self.fc1 = torch.nn.Linear( 256 , 9)

    def forward(self, x):
        emb = torch.relu(self.emb(x))
        
        lstm_output, (h_n, c_n) = self.rec(emb)
        
        out_weights = self.fc1(lstm_output)

        return out_weights
lstm = LSTM()
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(lstm.parameters())
def eval_model(dataset_tokens, dataset_labels, model):
    Y_true = []
    Y_pred = []
    for i in tqdm(range(len(dataset_labels))):
        batch_tokens = dataset_tokens[i].unsqueeze(0)
        tags = list(dataset_labels[i].numpy())
        Y_true += tags
        
        Y_batch_pred_weights = model(batch_tokens).squeeze(0)
        Y_batch_pred = torch.argmax(Y_batch_pred_weights,1)
        Y_pred += list(Y_batch_pred.numpy())
        

    return get_scores(Y_true, Y_pred)
        
NUM_EPOCHS = 5
for i in range(NUM_EPOCHS):
    lstm.train()
    #for i in tqdm(range(500)):
    for i in tqdm(range(len(train_labels))):
        batch_tokens = train_tokens_ids[i].unsqueeze(0)
        tags = train_labels[i].unsqueeze(1)
        
        
        predicted_tags = lstm(batch_tokens)

        
        optimizer.zero_grad()
        loss  = criterion(predicted_tags.squeeze(0),tags.squeeze(1))
        
        loss.backward()
        optimizer.step()
        
    lstm.eval()
    print(eval_model(validation_tokens_ids, validation_labels, lstm))
HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))
HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))
(0.5068524970963996, 0.5072649075903755, 0.5070586184860281)
HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))
HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))
(0.653649243957614, 0.6381494827385795, 0.6458063757205035)
HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))
HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))
(0.7140486069946651, 0.7001046146693014, 0.7070078647728607)
HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))
HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))
(0.756327964151629, 0.725909566430315, 0.7408066429418744)
HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))
HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))
(0.7963248522230789, 0.7203301174009067, 0.7564235581324383)
eval_model(validation_tokens_ids, validation_labels, lstm)
HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))
(0.7963248522230789, 0.7203301174009067, 0.7564235581324383)
eval_model(test_tokens_ids, test_labels, lstm)
HBox(children=(FloatProgress(value=0.0, max=3453.0), HTML(value='')))
(0.7450810185185185, 0.6348619329388561, 0.685569755058573)
len(train_tokens_ids)
14041

pytania

  • co zrobić z trenowaniem na batchach > 1 ?
  • co zrobić, żeby sieć uwzględniała następne tokeny, a nie tylko poprzednie?
  • w jaki sposób wykorzystać taką sieć do zadania zwykłej klasyfikacji?

Zadanie na zajęcia ( 20 minut)

zmodyfikować sieć tak, żeby była używała dwuwarstwowej, dwukierunkowej warstwy GRU oraz dropoutu. Dropout ma nałożony na embeddingi.


class GRU(torch.nn.Module):

    def __init__(self):
        super(GRU, self).__init__()
        self.emb = torch.nn.Embedding(len(vocab.itos),100)
        self.dropout = torch.nn.Dropout(0.2)
        self.rec = torch.nn.GRU(100, 256, 2, batch_first = True, bidirectional = True)
        self.fc1 = torch.nn.Linear(2* 256 , 9)
        
    def forward(self, x):
        emb = torch.relu(self.emb(x))
        emb = self.dropout(emb)
        
        gru_output, h_n = self.rec(emb)
        
        out_weights = self.fc1(gru_output)

        return out_weights
gru = GRU()
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(gru.parameters())
NUM_EPOCHS = 5
for i in range(NUM_EPOCHS):
    gru.train()
    #for i in tqdm(range(50)):
    for i in tqdm(range(len(train_labels))):
        batch_tokens = train_tokens_ids[i].unsqueeze(0)
        tags = train_labels[i].unsqueeze(1)
        
        
        predicted_tags = gru(batch_tokens)

        
        optimizer.zero_grad()
        loss  = criterion(predicted_tags.squeeze(0),tags.squeeze(1))
        
        loss.backward()
        optimizer.step()
        
        
    gru.eval()
    print(eval_model(validation_tokens_ids, validation_labels, gru))
HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))
HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))
(0.6431379891406104, 0.39927932116703474, 0.49268502581755597)
HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))
HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))
(0.6638910917261432, 0.5838660932232942, 0.6213123879027769)
HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))
HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))
(0.6697936210131332, 0.7054515866558178, 0.6871603260869565)
HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))
HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))
(0.7091097308488613, 0.7166104847146344, 0.7128403769439787)
HBox(children=(FloatProgress(value=0.0, max=14041.0), HTML(value='')))
HBox(children=(FloatProgress(value=0.0, max=3250.0), HTML(value='')))
(0.7708963348252087, 0.740788097175404, 0.755542382928275)

Zadanie domowe

  • sklonować repozytorium https://git.wmi.amu.edu.pl/kubapok/en-ner-conll-2003
  • stworzyć model seq labelling bazujący na sieci neuronowej opisanej w punkcie niżej (można bazować na tym jupyterze lub nie).
  • model sieci to GRU (o dowolnych parametrach) + CRF w pytorchu korzystając z modułu CRF z poprzednich zajęć- - stworzyć predykcje w plikach dev-0/out.tsv oraz test-A/out.tsv
  • wynik fscore sprawdzony za pomocą narzędzia geval (patrz poprzednie zadanie) powinien wynosić conajmniej 0.65
  • proszę umieścić predykcję oraz skrypty generujące (w postaci tekstowej a nie jupyter) w repo, a w MS TEAMS umieścić link do swojego repo termin 22.06, 60 punktów, za najlepszy wynik- 100 punktów