Merge branch 'master' of git.wmi.amu.edu.pl:filipg/aitech-eks

cw/10_CRF.ipynb

@@ -0,0 +1,427 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Podejście softmax z embeddingami na przykładzie NER"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "scrolled": true
+   },
+   "source": [
+    "https://pytorch-crf.readthedocs.io/en/stable/"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "https://www.aclweb.org/anthology/W03-0419.pdf"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import gensim\n",
+    "import torch\n",
+    "import pandas as pd\n",
+    "import seaborn as sns\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "\n",
+    "from datasets import load_dataset\n",
+    "from torchtext.vocab import Vocab\n",
+    "from collections import Counter\n",
+    "\n",
+    "from sklearn.datasets import fetch_20newsgroups\n",
+    "# https://scikit-learn.org/0.19/datasets/twenty_newsgroups.html\n",
+    "\n",
+    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
+    "from sklearn.metrics import accuracy_score\n",
+    "\n",
+    "from tqdm.notebook import tqdm\n",
+    "\n",
+    "import torch\n",
+    "from torchcrf import CRF"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "dataset = load_dataset(\"conll2003\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def build_vocab(dataset):\n",
+    "    counter = Counter()\n",
+    "    for document in dataset:\n",
+    "        counter.update(document)\n",
+    "    return Vocab(counter, specials=['<unk>', '<pad>', '<bos>', '<eos>'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vocab = build_vocab(dataset['train']['tokens'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "len(vocab.itos)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vocab['on']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def data_process(dt):\n",
+    "    return [ torch.tensor([vocab['<bos>']] +[vocab[token]  for token in  document ] + [vocab['<eos>']], dtype = torch.long) for document in dt]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def labels_process(dt):\n",
+    "    return [ torch.tensor([0] + document + [0], dtype = torch.long) for document in dt]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_tokens_ids = data_process(dataset['train']['tokens'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_tokens_ids = data_process(dataset['test']['tokens'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "validation_tokens_ids =  data_process(dataset['validation']['tokens'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "train_labels = labels_process(dataset['train']['ner_tags'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "validation_labels = labels_process(dataset['validation']['ner_tags'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_labels = labels_process(dataset['test']['ner_tags'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_tokens_ids[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_scores(y_true, y_pred):\n",
+    "    acc_score = 0\n",
+    "    tp = 0\n",
+    "    fp = 0\n",
+    "    selected_items = 0\n",
+    "    relevant_items = 0 \n",
+    "\n",
+    "    for p,t in zip(y_pred, y_true):\n",
+    "        if p == t:\n",
+    "            acc_score +=1\n",
+    "\n",
+    "        if p > 0 and p == t:\n",
+    "            tp +=1\n",
+    "\n",
+    "        if p > 0:\n",
+    "            selected_items += 1\n",
+    "\n",
+    "        if t > 0 :\n",
+    "            relevant_items +=1\n",
+    "\n",
+    "            \n",
+    "            \n",
+    "    if selected_items == 0:\n",
+    "        precision = 1.0\n",
+    "    else:\n",
+    "        precision = tp / selected_items\n",
+    "        \n",
+    "            \n",
+    "    if relevant_items == 0:\n",
+    "        recall = 1.0\n",
+    "    else:\n",
+    "        recall = tp / relevant_items\n",
+    "    \n",
+    "    \n",
+    "    if precision + recall == 0.0 :\n",
+    "        f1 = 0.0\n",
+    "    else:\n",
+    "        f1 = 2* precision * recall  / (precision + recall)\n",
+    "\n",
+    "    return precision, recall, f1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "num_tags = max([max(x) for x in dataset['train']['ner_tags'] ]) + 1 "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class FF(torch.nn.Module):\n",
+    "\n",
+    "    def __init__(self,):\n",
+    "        super(FF, self).__init__()\n",
+    "        self.emb = torch.nn.Embedding(23627,200)\n",
+    "        self.fc1 = torch.nn.Linear(200,num_tags)\n",
+    "       \n",
+    "\n",
+    "    def forward(self, x):\n",
+    "        x = self.emb(x)\n",
+    "        x = self.fc1(x)\n",
+    "        return x"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ff = FF()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "crf = CRF(num_tags)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "params = list(ff.parameters()) + list(crf.parameters())\n",
+    "\n",
+    "optimizer = torch.optim.Adam(params)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def eval_model(dataset_tokens, dataset_labels):\n",
+    "    Y_true = []\n",
+    "    Y_pred = []\n",
+    "    ff.eval()\n",
+    "    crf.eval()\n",
+    "    for i in tqdm(range(len(dataset_labels))):\n",
+    "        batch_tokens = dataset_tokens[i]\n",
+    "        tags = list(dataset_labels[i].numpy())\n",
+    "        emissions = ff(batch_tokens).unsqueeze(1)\n",
+    "        Y_pred += crf.decode(emissions)[0]\n",
+    "        Y_true += tags\n",
+    "\n",
+    "    return get_scores(Y_true, Y_pred)\n",
+    "        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "NUM_EPOCHS = 4"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "for i in range(NUM_EPOCHS):\n",
+    "    ff.train()\n",
+    "    crf.train()\n",
+    "    for i in tqdm(range(len(train_labels))):\n",
+    "        batch_tokens = train_tokens_ids[i]\n",
+    "        tags = train_labels[i].unsqueeze(1)\n",
+    "        emissions = ff(batch_tokens).unsqueeze(1)\n",
+    "\n",
+    "        optimizer.zero_grad()\n",
+    "        loss  = -crf(emissions,tags)\n",
+    "        loss.backward()\n",
+    "        optimizer.step()\n",
+    "        \n",
+    "    ff.eval()\n",
+    "    crf.eval()\n",
+    "    print(eval_model(validation_tokens_ids, validation_labels))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "eval_model(validation_tokens_ids, validation_labels)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "eval_model(test_tokens_ids, test_labels)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "len(train_tokens_ids)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Zadanie domowe\n",
+    "\n",
+    "- sklonować repozytorium https://git.wmi.amu.edu.pl/kubapok/en-ner-conll-2003\n",
+    "- stworzyć klasyfikator bazujący na sieci neuronowej feed forward w pytorchu  + CRF (można bazować na tym jupyterze lub nie).\n",
+    "- sieć feedforward powinna obejmować aktualne słowo, poprzednie i następne + dodatkowe cechy (np. długość wyrazu, czy wyraz zaczyna się od wielkiej litery, stemmming słowa, czy zawiera cyfrę)\n",
+    "- stworzyć predykcje w plikach dev-0/out.tsv oraz test-A/out.tsv\n",
+    "- wynik fscore sprawdzony za pomocą narzędzia geval (patrz poprzednie zadanie) powinien wynosić conajmniej 0.65\n",
+    "- proszę umieścić predykcję oraz skrypty generujące (w postaci tekstowej a nie jupyter) w repo, a w MS TEAMS umieścić link do swojego repo\n",
+    "termin 15.06, 60 punktów, za najlepszy wynik- 100 punktów\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}