forked from kubapok/en-ner-conll-2003
875 lines
24 KiB
Plaintext
875 lines
24 KiB
Plaintext
|
{
|
||
|
|
"nbformat": 4,
|
||
|
|
"nbformat_minor": 0,
|
||
|
|
"metadata": {
|
||
|
|
"accelerator": "GPU",
|
||
|
|
"colab": {
|
||
|
|
"name": "main.ipynb",
|
||
|
|
"provenance": []
|
||
|
|
},
|
||
|
|
"kernelspec": {
|
||
|
|
"display_name": "Python 3",
|
||
|
|
"name": "python3"
|
||
|
|
},
|
||
|
|
"language_info": {
|
||
|
|
"name": "python"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"cells": [
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"metadata": {
|
||
|
|
"id": "OY5VomOSCBez"
|
||
|
|
},
|
||
|
|
"source": [
|
||
|
|
"import numpy as np\n",
|
||
|
|
"import gensim\n",
|
||
|
|
"import torch\n",
|
||
|
|
"import pandas as pd\n",
|
||
|
|
"\n",
|
||
|
|
"from torchtext.vocab import Vocab\n",
|
||
|
|
"from collections import Counter\n",
|
||
|
|
"\n",
|
||
|
|
"import lzma\n",
|
||
|
|
"import re\n",
|
||
|
|
"import itertools"
|
||
|
|
],
|
||
|
|
"execution_count": 1,
|
||
|
|
"outputs": []
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"metadata": {
|
||
|
|
"id": "VXcowLY6HlNC"
|
||
|
|
},
|
||
|
|
"source": [
|
||
|
|
"class NeuralNetworkModel(torch.nn.Module):\n",
|
||
|
|
"\n",
|
||
|
|
" def __init__(self, output_size):\n",
|
||
|
|
" super(NeuralNetworkModel, self).__init__()\n",
|
||
|
|
" self.fc1 = torch.nn.Linear(10_000,len(train_tokens_ids))\n",
|
||
|
|
" self.softmax = torch.nn.Softmax(dim=0)\n",
|
||
|
|
" \n",
|
||
|
|
"\n",
|
||
|
|
" def forward(self, x):\n",
|
||
|
|
" x = self.fc1(x)\n",
|
||
|
|
" x = self.softmax(x)\n",
|
||
|
|
" return x"
|
||
|
|
],
|
||
|
|
"execution_count": 2,
|
||
|
|
"outputs": []
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"metadata": {
|
||
|
|
"id": "OXX_vPpTHhOq"
|
||
|
|
},
|
||
|
|
"source": [
|
||
|
|
"class NERModel(torch.nn.Module):\n",
|
||
|
|
"\n",
|
||
|
|
" def __init__(self,):\n",
|
||
|
|
" super(NERModel, self).__init__()\n",
|
||
|
|
" self.emb = torch.nn.Embedding(23627,200)\n",
|
||
|
|
" self.fc1 = torch.nn.Linear(600,9)\n",
|
||
|
|
"\n",
|
||
|
|
" def forward(self, x):\n",
|
||
|
|
" x = self.emb(x)\n",
|
||
|
|
" x = x.reshape(600) \n",
|
||
|
|
" x = self.fc1(x)\n",
|
||
|
|
" return x"
|
||
|
|
],
|
||
|
|
"execution_count": 3,
|
||
|
|
"outputs": []
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"metadata": {
|
||
|
|
"id": "NNpGPta9C4TI"
|
||
|
|
},
|
||
|
|
"source": [
|
||
|
|
"def get_dataset(path):\n",
|
||
|
|
" data = lzma.open(path).read().decode('UTF-8').split('\\n')\n",
|
||
|
|
" return [line.split('\\t') for line in data][:-1]\n",
|
||
|
|
"\n",
|
||
|
|
"train_data = get_dataset('train.tsv.xz')\n",
|
||
|
|
"\n",
|
||
|
|
"tokens = []\n",
|
||
|
|
"ner_tags = []\n",
|
||
|
|
"\n",
|
||
|
|
"for i in train_data:\n",
|
||
|
|
" ner_tags.append(i[0].split())\n",
|
||
|
|
" tokens.append(i[1].split())\n",
|
||
|
|
"\n",
|
||
|
|
"ner_tags_set = list(set(itertools.chain(*ner_tags)))\n",
|
||
|
|
"\n",
|
||
|
|
"ner_tags_dictionary = {}\n",
|
||
|
|
"\n",
|
||
|
|
"for i in range(len(ner_tags_set)):\n",
|
||
|
|
" ner_tags_dictionary[ner_tags_set[i]] = i"
|
||
|
|
],
|
||
|
|
"execution_count": 4,
|
||
|
|
"outputs": []
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"metadata": {
|
||
|
|
"id": "vvOF0opUGEMN"
|
||
|
|
},
|
||
|
|
"source": [
|
||
|
|
"for i in range(len(ner_tags)):\n",
|
||
|
|
" for j in range(len(ner_tags[i])):\n",
|
||
|
|
" ner_tags[i][j] = ner_tags_dictionary[ner_tags[i][j]]"
|
||
|
|
],
|
||
|
|
"execution_count": 5,
|
||
|
|
"outputs": []
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"metadata": {
|
||
|
|
"id": "kcGENe59us2p"
|
||
|
|
},
|
||
|
|
"source": [
|
||
|
|
"def data_preprocessing(data):\n",
|
||
|
|
" return [ torch.tensor([vocab['<bos>']] +[vocab[token] for token in document ] + [vocab['<eos>']], dtype = torch.long) for document in data ]\n",
|
||
|
|
"\n",
|
||
|
|
"def labels_preprocessing(data):\n",
|
||
|
|
" return [ torch.tensor([0] + document + [0], dtype = torch.long) for document in data ]\n",
|
||
|
|
"\n",
|
||
|
|
"def build_vocab(dataset):\n",
|
||
|
|
" counter = Counter()\n",
|
||
|
|
" for document in dataset:\n",
|
||
|
|
" counter.update(document)\n",
|
||
|
|
" return Vocab(counter)\n",
|
||
|
|
"\n",
|
||
|
|
"\n",
|
||
|
|
"vocab = build_vocab(tokens)\n",
|
||
|
|
"train_tokens_ids = data_preprocessing(tokens)\n",
|
||
|
|
"train_labels = labels_preprocessing(ner_tags)"
|
||
|
|
],
|
||
|
|
"execution_count": 6,
|
||
|
|
"outputs": []
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"metadata": {
|
||
|
|
"colab": {
|
||
|
|
"base_uri": "https://localhost:8080/",
|
||
|
|
"height": 437
|
||
|
|
},
|
||
|
|
"id": "yoCYSZNeHJeT",
|
||
|
|
"outputId": "91cc012b-4437-41a3-e312-1aaae3542c91"
|
||
|
|
},
|
||
|
|
"source": [
|
||
|
|
"nn_model = NeuralNetworkModel(len(train_tokens_ids))\n",
|
||
|
|
"train_tokens_ids[0][1:4]\n",
|
||
|
|
"\n",
|
||
|
|
"ner_model = NERModel()\n",
|
||
|
|
"ner_model(train_tokens_ids[0][1:4])\n",
|
||
|
|
"\n",
|
||
|
|
"criterion = torch.nn.CrossEntropyLoss()\n",
|
||
|
|
"optimizer = torch.optim.Adam(ner_model.parameters())\n",
|
||
|
|
"\n",
|
||
|
|
"for epoch in range(2):\n",
|
||
|
|
" loss_score = 0\n",
|
||
|
|
" acc_score = 0\n",
|
||
|
|
" prec_score = 0\n",
|
||
|
|
" selected_items = 0\n",
|
||
|
|
" recall_score = 0\n",
|
||
|
|
" relevant_items = 0\n",
|
||
|
|
" items_total = 0\n",
|
||
|
|
" nn_model.train()\n",
|
||
|
|
" for i in range(100):\n",
|
||
|
|
" for j in range(1, len(train_labels[i]) - 1):\n",
|
||
|
|
" \n",
|
||
|
|
" X = train_tokens_ids[i][j-1: j+2]\n",
|
||
|
|
" Y = train_labels[i][j: j+1]\n",
|
||
|
|
"\n",
|
||
|
|
" Y_predictions = ner_model(X)\n",
|
||
|
|
" \n",
|
||
|
|
" \n",
|
||
|
|
" acc_score += int(torch.argmax(Y_predictions) == Y)\n",
|
||
|
|
" \n",
|
||
|
|
" if torch.argmax(Y_predictions) != 0:\n",
|
||
|
|
" selected_items +=1\n",
|
||
|
|
" if torch.argmax(Y_predictions) != 0 and torch.argmax(Y_predictions) == Y.item():\n",
|
||
|
|
" prec_score += 1\n",
|
||
|
|
" \n",
|
||
|
|
" if Y.item() != 0:\n",
|
||
|
|
" relevant_items +=1\n",
|
||
|
|
" if Y.item() != 0 and torch.argmax(Y_predictions) == Y.item():\n",
|
||
|
|
" recall_score += 1\n",
|
||
|
|
" \n",
|
||
|
|
" items_total += 1\n",
|
||
|
|
"\n",
|
||
|
|
" \n",
|
||
|
|
" optimizer.zero_grad()\n",
|
||
|
|
" loss = criterion(Y_predictions.unsqueeze(0), Y)\n",
|
||
|
|
" loss.backward()\n",
|
||
|
|
" optimizer.step()\n",
|
||
|
|
"\n",
|
||
|
|
"\n",
|
||
|
|
" loss_score += loss.item() \n",
|
||
|
|
" \n",
|
||
|
|
" precision = prec_score / selected_items\n",
|
||
|
|
" recall = recall_score / relevant_items\n",
|
||
|
|
" f1_score = (2*precision * recall) / (precision + recall)\n",
|
||
|
|
" display('epoch: ', epoch)\n",
|
||
|
|
" display('loss: ', loss_score / items_total)\n",
|
||
|
|
" display('acc: ', acc_score / items_total)\n",
|
||
|
|
" display('prec: ', precision)\n",
|
||
|
|
" display('recall: : ', recall)\n",
|
||
|
|
" display('f1: ', f1_score)"
|
||
|
|
],
|
||
|
|
"execution_count": 7,
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"application/vnd.google.colaboratory.intrinsic+json": {
|
||
|
|
"type": "string"
|
||
|
|
},
|
||
|
|
"text/plain": [
|
||
|
|
"'epoch: '"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"0"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"application/vnd.google.colaboratory.intrinsic+json": {
|
||
|
|
"type": "string"
|
||
|
|
},
|
||
|
|
"text/plain": [
|
||
|
|
"'loss: '"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"0.5260595670613091"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"application/vnd.google.colaboratory.intrinsic+json": {
|
||
|
|
"type": "string"
|
||
|
|
},
|
||
|
|
"text/plain": [
|
||
|
|
"'acc: '"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"0.8669271431854746"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"application/vnd.google.colaboratory.intrinsic+json": {
|
||
|
|
"type": "string"
|
||
|
|
},
|
||
|
|
"text/plain": [
|
||
|
|
"'prec: '"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"0.868387037208014"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"application/vnd.google.colaboratory.intrinsic+json": {
|
||
|
|
"type": "string"
|
||
|
|
},
|
||
|
|
"text/plain": [
|
||
|
|
"'recall: : '"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"0.8694707649641784"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"application/vnd.google.colaboratory.intrinsic+json": {
|
||
|
|
"type": "string"
|
||
|
|
},
|
||
|
|
"text/plain": [
|
||
|
|
"'f1: '"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"0.868928563179897"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"application/vnd.google.colaboratory.intrinsic+json": {
|
||
|
|
"type": "string"
|
||
|
|
},
|
||
|
|
"text/plain": [
|
||
|
|
"'epoch: '"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"1"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"application/vnd.google.colaboratory.intrinsic+json": {
|
||
|
|
"type": "string"
|
||
|
|
},
|
||
|
|
"text/plain": [
|
||
|
|
"'loss: '"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"0.46469578519580995"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"application/vnd.google.colaboratory.intrinsic+json": {
|
||
|
|
"type": "string"
|
||
|
|
},
|
||
|
|
"text/plain": [
|
||
|
|
"'acc: '"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"0.8826936336474374"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"application/vnd.google.colaboratory.intrinsic+json": {
|
||
|
|
"type": "string"
|
||
|
|
},
|
||
|
|
"text/plain": [
|
||
|
|
"'prec: '"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"0.8842406612180819"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"application/vnd.google.colaboratory.intrinsic+json": {
|
||
|
|
"type": "string"
|
||
|
|
},
|
||
|
|
"text/plain": [
|
||
|
|
"'recall: : '"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"0.885139819736538"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"application/vnd.google.colaboratory.intrinsic+json": {
|
||
|
|
"type": "string"
|
||
|
|
},
|
||
|
|
"text/plain": [
|
||
|
|
"'f1: '"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"0.884690012011457"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"metadata": {
|
||
|
|
"id": "UPGojltN-9_o"
|
||
|
|
},
|
||
|
|
"source": [
|
||
|
|
"def process_output(lines):\n",
|
||
|
|
" result = []\n",
|
||
|
|
" for line in lines:\n",
|
||
|
|
" last_label = None\n",
|
||
|
|
" new_line = []\n",
|
||
|
|
" for label in line:\n",
|
||
|
|
" if(label != \"O\" and label[0:2] == \"I-\"):\n",
|
||
|
|
" if last_label == None or last_label == \"O\":\n",
|
||
|
|
" label = label.replace('I-', 'B-')\n",
|
||
|
|
" else:\n",
|
||
|
|
" label = \"I-\" + last_label[2:]\n",
|
||
|
|
" last_label = label\n",
|
||
|
|
" new_line.append(label)\n",
|
||
|
|
" x = (\" \".join(new_line))\n",
|
||
|
|
" result.append(\" \".join(new_line))\n",
|
||
|
|
" return result"
|
||
|
|
],
|
||
|
|
"execution_count": 8,
|
||
|
|
"outputs": []
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"metadata": {
|
||
|
|
"id": "KOVSTjGWVuq9"
|
||
|
|
},
|
||
|
|
"source": [
|
||
|
|
"with open('dev-0/in.tsv', \"r\", encoding=\"utf-8\") as f:\n",
|
||
|
|
" dev_0_data = [line.rstrip() for line in f]\n",
|
||
|
|
" \n",
|
||
|
|
"dev_0_data = [i.split() for i in dev_0_data]\n",
|
||
|
|
"\n",
|
||
|
|
"with open('dev-0/expected.tsv', \"r\", encoding=\"utf-8\") as f:\n",
|
||
|
|
" dev_0_tags = [line.rstrip() for line in f]\n",
|
||
|
|
" \n",
|
||
|
|
"dev_0_tags = [i.split() for i in dev_0_tags]\n",
|
||
|
|
"\n",
|
||
|
|
"for i in range(len(dev_0_tags)):\n",
|
||
|
|
" for j in range(len(dev_0_tags[i])):\n",
|
||
|
|
" dev_0_tags[i][j] = ner_tags_dictionary[dev_0_tags[i][j]]\n",
|
||
|
|
" \n",
|
||
|
|
"test_tokens_ids = data_preprocessing(dev_0_data)\n",
|
||
|
|
"test_labels = labels_preprocessing(dev_0_tags)\n"
|
||
|
|
],
|
||
|
|
"execution_count": 9,
|
||
|
|
"outputs": []
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"metadata": {
|
||
|
|
"colab": {
|
||
|
|
"base_uri": "https://localhost:8080/",
|
||
|
|
"height": 192
|
||
|
|
},
|
||
|
|
"id": "Pt7sVRdhWCqC",
|
||
|
|
"outputId": "cf1ffab1-e043-4875-9514-c2713d3c5393"
|
||
|
|
},
|
||
|
|
"source": [
|
||
|
|
"result = []\n",
|
||
|
|
"\n",
|
||
|
|
"loss_score = 0\n",
|
||
|
|
"acc_score = 0\n",
|
||
|
|
"prec_score = 0\n",
|
||
|
|
"selected_items = 0\n",
|
||
|
|
"recall_score = 0\n",
|
||
|
|
"relevant_items = 0\n",
|
||
|
|
"items_total = 0\n",
|
||
|
|
"nn_model.eval()\n",
|
||
|
|
"\n",
|
||
|
|
"for i in range(len(test_tokens_ids)):\n",
|
||
|
|
" result.append([])\n",
|
||
|
|
" for j in range(1, len(test_labels[i]) - 1):\n",
|
||
|
|
"\n",
|
||
|
|
" X = test_tokens_ids[i][j-1: j+2]\n",
|
||
|
|
" Y = test_labels[i][j: j+1]\n",
|
||
|
|
"\n",
|
||
|
|
" Y_predictions = ner_model(X)\n",
|
||
|
|
"\n",
|
||
|
|
"\n",
|
||
|
|
" acc_score += int(torch.argmax(Y_predictions) == Y)\n",
|
||
|
|
"\n",
|
||
|
|
" if torch.argmax(Y_predictions) != 0:\n",
|
||
|
|
" selected_items +=1\n",
|
||
|
|
" if torch.argmax(Y_predictions) != 0 and torch.argmax(Y_predictions) == Y.item():\n",
|
||
|
|
" prec_score += 1\n",
|
||
|
|
"\n",
|
||
|
|
" if Y.item() != 0:\n",
|
||
|
|
" relevant_items +=1\n",
|
||
|
|
" if Y.item() != 0 and torch.argmax(Y_predictions) == Y.item():\n",
|
||
|
|
" recall_score += 1\n",
|
||
|
|
"\n",
|
||
|
|
" items_total += 1\n",
|
||
|
|
" loss = criterion(Y_predictions.unsqueeze(0), Y)\n",
|
||
|
|
" loss_score += loss.item() \n",
|
||
|
|
" \n",
|
||
|
|
" result[i].append(int(torch.argmax(Y_predictions)))\n",
|
||
|
|
"\n",
|
||
|
|
"precision = prec_score / selected_items\n",
|
||
|
|
"recall = recall_score / relevant_items\n",
|
||
|
|
"f1_score = (2*precision * recall) / (precision + recall)\n",
|
||
|
|
"display('loss: ', loss_score / items_total)\n",
|
||
|
|
"display('acc: ', acc_score / items_total)\n",
|
||
|
|
"display('prec: ', precision)\n",
|
||
|
|
"display('recall: : ', recall)\n",
|
||
|
|
"display('f1: ', f1_score)"
|
||
|
|
],
|
||
|
|
"execution_count": 10,
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"application/vnd.google.colaboratory.intrinsic+json": {
|
||
|
|
"type": "string"
|
||
|
|
},
|
||
|
|
"text/plain": [
|
||
|
|
"'loss: '"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"0.9558752998502382"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"application/vnd.google.colaboratory.intrinsic+json": {
|
||
|
|
"type": "string"
|
||
|
|
},
|
||
|
|
"text/plain": [
|
||
|
|
"'acc: '"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"0.8221458628103122"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"application/vnd.google.colaboratory.intrinsic+json": {
|
||
|
|
"type": "string"
|
||
|
|
},
|
||
|
|
"text/plain": [
|
||
|
|
"'prec: '"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"0.8261110597800979"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"application/vnd.google.colaboratory.intrinsic+json": {
|
||
|
|
"type": "string"
|
||
|
|
},
|
||
|
|
"text/plain": [
|
||
|
|
"'recall: : '"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"0.8253637102134259"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"application/vnd.google.colaboratory.intrinsic+json": {
|
||
|
|
"type": "string"
|
||
|
|
},
|
||
|
|
"text/plain": [
|
||
|
|
"'f1: '"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"0.8257372158959724"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
}
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"metadata": {
|
||
|
|
"id": "Bx-pxGsjlVLJ"
|
||
|
|
},
|
||
|
|
"source": [
|
||
|
|
"def save_to_file(path, results):\n",
|
||
|
|
" with open(path, \"w\") as f:\n",
|
||
|
|
" for line in results:\n",
|
||
|
|
" print(line)\n",
|
||
|
|
" f.write(line + \"\\n\")"
|
||
|
|
],
|
||
|
|
"execution_count": 21,
|
||
|
|
"outputs": []
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"metadata": {
|
||
|
|
"id": "20zgcUx4sMCe"
|
||
|
|
},
|
||
|
|
"source": [
|
||
|
|
"tags = []\n",
|
||
|
|
"tmp = []\n",
|
||
|
|
"for i in ner_tags_dictionary:\n",
|
||
|
|
" tmp.append(i)\n",
|
||
|
|
"\n",
|
||
|
|
"for i in range(len(result)):\n",
|
||
|
|
" tags.append([])\n",
|
||
|
|
" for j in range(len(result[i])):\n",
|
||
|
|
" tags[i].append(tmp[result[i][j]])\n",
|
||
|
|
"\n",
|
||
|
|
"processed_tags = process_output(tags)\n",
|
||
|
|
"\n",
|
||
|
|
"with open(\"dev-0/out.tsv\", \"w\") as f:\n",
|
||
|
|
" for line in processed_tags:\n",
|
||
|
|
" f.write(line + \"\\n\")\n",
|
||
|
|
"\n",
|
||
|
|
"with open('dev-0/expected.tsv', \"r\", encoding=\"utf-8\") as f:\n",
|
||
|
|
" dev_0_tags = [line.rstrip() for line in f]\n",
|
||
|
|
" \n",
|
||
|
|
"dev_0_tags = [i.split() for i in dev_0_tags]"
|
||
|
|
],
|
||
|
|
"execution_count": 24,
|
||
|
|
"outputs": []
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"metadata": {
|
||
|
|
"id": "d-QCHMrycKwH"
|
||
|
|
},
|
||
|
|
"source": [
|
||
|
|
"with open('test-A/in.tsv', \"r\", encoding=\"utf-8\") as file:\n",
|
||
|
|
" test_data = [line.rstrip() for line in file]\n",
|
||
|
|
" \n",
|
||
|
|
"test_data = [i.split() for i in test_data]\n",
|
||
|
|
"test_tokens_ids = data_preprocessing(test_data)\n",
|
||
|
|
"result = []\n",
|
||
|
|
"\n",
|
||
|
|
"loss_score = 0\n",
|
||
|
|
"acc_score = 0\n",
|
||
|
|
"prec_score = 0\n",
|
||
|
|
"selected_items = 0\n",
|
||
|
|
"recall_score = 0\n",
|
||
|
|
"relevant_items = 0\n",
|
||
|
|
"items_total = 0\n",
|
||
|
|
"nn_model.eval()\n",
|
||
|
|
"\n",
|
||
|
|
"test_tokens_length = len(test_tokens_ids)\n",
|
||
|
|
"\n",
|
||
|
|
"for i in range(test_tokens_length):\n",
|
||
|
|
" result.append([])\n",
|
||
|
|
" for j in range(1, len(test_tokens_ids[i]) - 1):\n",
|
||
|
|
" X = test_tokens_ids[i][j-1: j + 2]\n",
|
||
|
|
" Y_predictions = ner_model(X)\n",
|
||
|
|
" result[i].append(int(torch.argmax(Y_predictions)))"
|
||
|
|
],
|
||
|
|
"execution_count": 26,
|
||
|
|
"outputs": []
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"metadata": {
|
||
|
|
"id": "862mKzAMkbx_"
|
||
|
|
},
|
||
|
|
"source": [
|
||
|
|
"tags = []\n",
|
||
|
|
"tmp = []\n",
|
||
|
|
"\n",
|
||
|
|
"for i in ner_tags_dictionary:\n",
|
||
|
|
" tmp.append(i)\n",
|
||
|
|
"\n",
|
||
|
|
"result_length = len(result)\n",
|
||
|
|
"\n",
|
||
|
|
"for i in range(result_length):\n",
|
||
|
|
" tags.append([])\n",
|
||
|
|
" for j in range(len(result[i])):\n",
|
||
|
|
" tags[i].append(tmp[result[i][j]])\n",
|
||
|
|
"\n",
|
||
|
|
"processed_tags = process_output(tags)\n",
|
||
|
|
"\n",
|
||
|
|
"with open(\"test-A/out.tsv\", \"w\") as f:\n",
|
||
|
|
" for line in processed_tags:\n",
|
||
|
|
" f.write(line + \"\\n\")"
|
||
|
|
],
|
||
|
|
"execution_count": 27,
|
||
|
|
"outputs": []
|
||
|
|
}
|
||
|
|
]
|
||
|
|
}
|