aitech-eks-pub/cw/09_sequence_labeling_ODPOWIEDZI.ipynb

Klasyfikacja wieloklasowa i sequence labelling

import numpy as np
import gensim
import torch
import pandas as pd
import seaborn as sns
from sklearn.model_selection import train_test_split

from datasets import load_dataset
from torchtext.vocab import Vocab
from collections import Counter

from sklearn.datasets import fetch_20newsgroups
# https://scikit-learn.org/0.19/datasets/twenty_newsgroups.html

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import accuracy_score

Klasyfikacja

Klasfikacja binarna- 2 klasy

CATEGORIES = ['soc.religion.christian', 'alt.atheism']

newsgroups = fetch_20newsgroups(categories=CATEGORIES)

X =  newsgroups['data']

Y = newsgroups['target']

Y_names = newsgroups['target_names']

X[0:1]

Y

Y_names

del CATEGORIES, newsgroups, X, Y, Y_names

klasyfikacja wieloklasowa

newsgroups_train_dev = fetch_20newsgroups(subset = 'train')
newsgroups_test = fetch_20newsgroups(subset = 'test')

newsgroups_train_dev_text = newsgroups_train_dev['data']
newsgroups_test_text = newsgroups_test['data']

Y_train_dev = newsgroups_train_dev['target']
Y_test = newsgroups_test['target']

newsgroups_train_text, newsgroups_dev_text, Y_train, Y_dev = train_test_split(newsgroups_train_dev_text, Y_train_dev, random_state=42)

Y_names = newsgroups_train_dev['target_names']

Y_train_dev

Y_names

Jaki baseline?

pd.value_counts(Y_train)

accuracy_score(Y_test, np.ones_like(Y_test) * 10)

Pytanie - w jaki sposób stworzyć taki klasyfikator na podstawie tylko wiedzy z poprzednich ćwiczeń?

Zadanie - stworzyć klasyfikator regresji logistycznej one vs rest na podstawie tfdif. TFIDF powinien mieć słownik o wielkości 10000

https://scikit-learn.org/stable/modules/generated/sklearn.multiclass.OneVsRestClassifier.html https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.TfidfVectorizer.html

from sklearn.multiclass import OneVsRestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.feature_extraction.text import TfidfVectorizer

FEAUTERES = 10_000

vectorizer = TfidfVectorizer(max_features=FEAUTERES)
X_train = vectorizer.fit_transform(newsgroups_train_text)

X_dev = vectorizer.transform(newsgroups_dev_text)
X_test = vectorizer.transform(newsgroups_test_text)

clf = OneVsRestClassifier(LogisticRegression()).fit(X_train, Y_train)

clf.predict(X_train[0:1])

clf.predict_proba(X_train[0:1])

np.max(clf.predict_proba(X_train[0]))

accuracy_score(clf.predict(X_train), Y_train)

accuracy_score(clf.predict(X_dev), Y_dev)

accuracy_score(clf.predict(X_test), Y_test)

Podejście softmax na tfidif

Zadanie Na podstawie poprzednich zajęć stworzyć sieć w pytorch bez warstw ukrytych, z jedną warstwą _output z funkcją softmax (bez trenowania i ewaluacji sieci)

Użyć https://pytorch.org/docs/stable/generated/torch.nn.Softmax.html?highlight=softmax

X_train

class NeuralNetworkModel(torch.nn.Module):

    def __init__(self,FEAUTERES, output_size):
        super(NeuralNetworkModel, self).__init__()
        self.fc1 = torch.nn.Linear(FEAUTERES,OUTPUT_SIZE)
        self.softmax = torch.nn.Softmax(dim=0)
        

    def forward(self, x):
        x = self.fc1(x)
        x = self.softmax(x)
        return x

OUTPUT_SIZE = len(Y_names)

nn_model = NeuralNetworkModel(FEAUTERES, OUTPUT_SIZE)

nn_model(torch.Tensor(X_train[0:3].astype(np.float32).todense()))

BATCH_SIZE = 5

criterion = torch.nn.NLLLoss()

optimizer = torch.optim.SGD(nn_model.parameters(), lr = 0.2)
#optimizer = torch.optim.Adam(nn_model.parameters())

def get_loss_acc(model, X_dataset, Y_dataset):
    loss_score = 0
    acc_score = 0
    items_total = 0
    model.eval()
    for i in range(0, Y_dataset.shape[0], BATCH_SIZE):
        X = X_dataset[i:i+BATCH_SIZE]
        X = torch.tensor(X.astype(np.float32).todense())
        Y = Y_dataset[i:i+BATCH_SIZE]
        Y = torch.tensor(Y)
        Y_predictions = model(X)
        acc_score += torch.sum(torch.argmax(Y_predictions,dim=1) == Y).item()
        items_total += Y.shape[0] 

        loss = criterion(Y_predictions, Y)

        loss_score += loss.item() * Y.shape[0] 
    return (loss_score / items_total), (acc_score / items_total)

for epoch in range(5):
    loss_score = 0
    acc_score = 0
    items_total = 0
    nn_model.train()
    for i in range(0, Y_train.shape[0], BATCH_SIZE):
        X = X_train[i:i+BATCH_SIZE]
        X = torch.tensor(X.astype(np.float32).todense())
        Y = Y_train[i:i+BATCH_SIZE]

        Y = torch.tensor(Y)
        Y_predictions = nn_model(X)
        acc_score += torch.sum(torch.argmax(Y_predictions,dim=1) == Y).item()
        items_total += Y.shape[0] 

        optimizer.zero_grad()
        loss = criterion(Y_predictions, Y)
        loss.backward()
        optimizer.step()


        loss_score += loss.item() * Y.shape[0]

    
    display(epoch)
    display(get_loss_acc(nn_model, X_train, Y_train))
    display(get_loss_acc(nn_model, X_dev, Y_dev))

X.shape

newsgroups_train_text

Podejście softmax z embeddingami na przykładzie NER

# !pip install torchtext
# !pip install datasets

https://www.aclweb.org/anthology/W03-0419.pdf

dataset = load_dataset("conll2003")

def build_vocab(dataset):
    counter = Counter()
    for document in dataset:
        counter.update(document)
    return Vocab(counter, specials=['<unk>', '<pad>', '<bos>', '<eos>'])

vocab = build_vocab(dataset['train']['tokens'])

dataset['train']['tokens']

len(vocab.itos)

vocab['on']

def data_process(dt):
    return [ torch.tensor([vocab['<bos>']] +[vocab[token]  for token in  document ] + [vocab['<eos>']], dtype = torch.long) for document in dt]

def labels_process(dt):
    return [ torch.tensor([0] + document + [0], dtype = torch.long) for document in dt]

train_tokens_ids = data_process(dataset['train']['tokens'])

test_tokens_ids = data_process(dataset['test']['tokens'])

train_labels = labels_process(dataset['train']['ner_tags'])

test_labels = labels_process(dataset['test']['ner_tags'])

train_tokens_ids[0]

max([max(x) for x in dataset['train']['ner_tags'] ])

class NERModel(torch.nn.Module):

    def __init__(self,):
        super(NERModel, self).__init__()
        self.emb = torch.nn.Embedding(23627,200)
        self.fc1 = torch.nn.Linear(600,9)
        #self.softmax = torch.nn.Softmax(dim=0)
        # nie trzeba, bo używamy https://pytorch.org/docs/stable/generated/torch.nn.CrossEntropyLoss.html
        # jako kryterium
        

    def forward(self, x):
        x = self.emb(x)
        x = x.reshape(600) 
        x = self.fc1(x)
        #x = self.softmax(x)
        return x

train_tokens_ids[0][1:4]

ner_model = NERModel()

ner_model(train_tokens_ids[0][1:4])

criterion = torch.nn.CrossEntropyLoss()

optimizer = torch.optim.Adam(ner_model.parameters())

len(train_labels)

for epoch in range(2):
    loss_score = 0
    acc_score = 0
    prec_score = 0
    selected_items = 0
    recall_score = 0
    relevant_items = 0
    items_total = 0
    nn_model.train()
    #for i in range(len(train_labels)):
    for i in range(100):
        for j in range(1, len(train_labels[i]) - 1):
    
            X = train_tokens_ids[i][j-1: j+2]
            Y = train_labels[i][j: j+1]

            Y_predictions = ner_model(X)
            
            
            acc_score += int(torch.argmax(Y_predictions) == Y)
            
            if torch.argmax(Y_predictions) != 0:
                selected_items +=1
            if  torch.argmax(Y_predictions) != 0 and torch.argmax(Y_predictions) == Y.item():
                prec_score += 1
            
            if  Y.item() != 0:
                relevant_items +=1
            if  Y.item() != 0 and torch.argmax(Y_predictions) == Y.item():
                recall_score += 1
            
            items_total += 1

            
            optimizer.zero_grad()
            loss = criterion(Y_predictions.unsqueeze(0), Y)
            loss.backward()
            optimizer.step()


            loss_score += loss.item() 
    
    precision = prec_score / selected_items
    recall = recall_score / relevant_items
    f1_score = (2*precision * recall) / (precision + recall)
    display('epoch: ', epoch)
    display('loss: ', loss_score / items_total)
    display('acc: ', acc_score / items_total)
    display('prec: ', precision)
    display('recall: : ', recall)
    display('f1: ', f1_score)

loss_score = 0
acc_score = 0
prec_score = 0
selected_items = 0
recall_score = 0
relevant_items = 0
items_total = 0
nn_model.eval()
for i in range(100):
#for i in range(len(test_labels)):
    for j in range(1, len(test_labels[i]) - 1):

        X = test_tokens_ids[i][j-1: j+2]
        Y = test_labels[i][j: j+1]

        Y_predictions = ner_model(X)


        acc_score += int(torch.argmax(Y_predictions) == Y)

        if torch.argmax(Y_predictions) != 0:
            selected_items +=1
        if  torch.argmax(Y_predictions) != 0 and torch.argmax(Y_predictions) == Y.item():
            prec_score += 1

        if  Y.item() != 0:
            relevant_items +=1
        if  Y.item() != 0 and torch.argmax(Y_predictions) == Y.item():
            recall_score += 1

        items_total += 1


        loss = criterion(Y_predictions.unsqueeze(0), Y)



        loss_score += loss.item() 

precision = prec_score / selected_items
recall = recall_score / relevant_items
f1_score = (2*precision * recall) / (precision + recall)
display('loss: ', loss_score / items_total)
display('acc: ', acc_score / items_total)
display('prec: ', precision)
display('recall: : ', recall)
display('f1: ', f1_score)

Zadanie domowe

• sklonować repozytorium https://git.wmi.amu.edu.pl/kubapok/en-ner-conll-2003
• stworzyć klasyfikator bazujący na sieci neuronowej feed forward w pytorchu (można bazować na tym jupyterze lub nie).
• klasyfikator powinien obejmować dodatkowe cechy (np. długość wyrazu, czy wyraz zaczyna się od wielkiej litery, stemmming słowa, czy zawiera cyfrę)
• stworzyć predykcje w plikach dev-0/out.tsv oraz test-A/out.tsv
• wynik fscore sprawdzony za pomocą narzędzia geval (patrz poprzednie zadanie) powinien wynosić conajmniej 0.60
• proszę umieścić predykcję oraz skrypty generujące (w postaci tekstowej a nie jupyter) w repo, a w MS TEAMS umieścić link do swojego repo termin 08.06, 80 punktów