Final model

.gitignore

@@ -1,8 +1,8 @@
-
-*~
-*.swp
-*.bak
-*.pyc
-*.o
-.DS_Store
-.token
+*~
+*.swp
+*.bak
+*.pyc
+*.o
+.DS_Store
+.token
+model.pkl

README.md

@@ -1,9 +1,15 @@
-Challenging America word-gap prediction
-===================================
-
-Guess a word in a gap.
-
-Evaluation metric
------------------
-
-LikelihoodHashed is the metric
+Challenging America word-gap prediction
+===================================
+
+This task is to predict the word-gap between two sentences.
+
+Evaluation
+-----------------
+
+PerplexityHashed is the metric so to check the performance of the model. The lower the perplexity, the better the model. To run evaluation run the following command:
+
+```bash
+./geval --metric PerplexityHashed --test-name dev-0
+```
+
+Perplexity calculated on `dev-0` is equal `981.69`

in-header.tsv

@@ -1 +1 @@
-FileId	Year	LeftContext	RightContext
+FileId	Year	LeftContext	RightContext

out-header.tsv

@@ -1 +1 @@
-Word
+Word

src/04_statystyczny_model_językowy.ipynb

@@ -0,0 +1,769 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "<h1> Modelowanie języka </h1>\n",
+    "<h2> 4. <i>Statystyczny model językowy</i>  [ćwiczenia]</h2> "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "NR_INDEKSU = 452629"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "https://web.stanford.edu/~jurafsky/slp3/3.pdf"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from collections import Counter, defaultdict\n",
+    "from tqdm import tqdm\n",
+    "import re\n",
+    "import nltk\n",
+    "import math\n",
+    "import random"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class Model():\n",
+    "    \n",
+    "    def __init__(self, vocab_size = 30_000, UNK_token = '<UNK>', n = 2):\n",
+    "        self.n = n\n",
+    "        self.vocab_size = vocab_size\n",
+    "        self.UNK_token = UNK_token\n",
+    "        self.ngrams = defaultdict(lambda: defaultdict(int))\n",
+    "        self.contexts = defaultdict(int)\n",
+    "        self.vocab = set()\n",
+    "    \n",
+    "    def train(self, corpus: list) -> None:\n",
+    "        self.vocab = set()\n",
+    "        self.vocab.add(self.UNK_token)\n",
+    "\n",
+    "        counts = Counter(corpus)\n",
+    "        most_common = counts.most_common(self.vocab_size - 1)\n",
+    "        for word, _ in most_common:\n",
+    "            self.vocab.add(word)\n",
+    "\n",
+    "        corpus = [word if word in self.vocab else self.UNK_token for word in corpus]\n",
+    "\n",
+    "        n_grams = list(nltk.ngrams(corpus, self.n))\n",
+    "        for gram in tqdm(n_grams):\n",
+    "            context = gram[:-1]\n",
+    "            word = gram[-1]\n",
+    "\n",
+    "            if word == self.UNK_token:\n",
+    "                continue\n",
+    "\n",
+    "            self.ngrams[context][word] += 1\n",
+    "            self.contexts[context] += 1\n",
+    "    \n",
+    "    def get_conditional_prob_for_word(self, text: list, word: str) -> float:\n",
+    "        if len(text) < self.n - 1:\n",
+    "            raise ValueError(\"Text is too short for the given n-gram order.\")\n",
+    "        \n",
+    "        context = tuple(text[-self.n + 1:])\n",
+    "        if context not in self.ngrams:\n",
+    "            return 0.0\n",
+    "        \n",
+    "        total_count = sum(self.ngrams[context].values())\n",
+    "        word_count = self.ngrams[context][word]\n",
+    "        \n",
+    "        if total_count == 0:\n",
+    "            return 0.0\n",
+    "        else:\n",
+    "            return word_count / total_count\n",
+    "    \n",
+    "    def get_prob_for_text(self, text: list) -> float:\n",
+    "        if len(text) < self.n - 1:\n",
+    "            raise ValueError(\"Text is too short for the given n-gram order.\")\n",
+    "        \n",
+    "        prob = 1.0\n",
+    "        n_grams = list(nltk.ngrams(text, self.n))\n",
+    "        for gram in n_grams:\n",
+    "            context = gram[:-1]\n",
+    "            word = gram[-1]\n",
+    "            prob *= self.get_conditional_prob_for_word(context, word)\n",
+    "        \n",
+    "        return prob\n",
+    "    \n",
+    "    def most_probable_next_word(self, text: list) -> str:\n",
+    "        '''nie powinien zwracań nigdy <UNK>'''\n",
+    "        if len(text) < self.n - 1:\n",
+    "            raise ValueError(\"Text is too short for the given n-gram order.\")\n",
+    "        \n",
+    "        context = tuple(text[-self.n+1:])\n",
+    "        if context not in self.ngrams:\n",
+    "            return \"\"\n",
+    "        \n",
+    "        most_probable_word = max(self.ngrams[context], key=self.ngrams[context].get)\n",
+    "        return most_probable_word\n",
+    "    \n",
+    "    def generate_text(self, text_beginning: list, length: int, greedy: bool) -> list:\n",
+    "        '''nie powinien zwracań nigdy <UNK>'''\n",
+    "        if len(text_beginning) < self.n - 1:\n",
+    "            raise ValueError(\"Text beginning is too short for the given n-gram order.\")\n",
+    "        \n",
+    "        text_beginning = [word if word in self.vocab else self.UNK_token for word in text_beginning]\n",
+    "        \n",
+    "        generated_text = text_beginning[:]\n",
+    "        while len(generated_text) < length:\n",
+    "            if self.n == 1:\n",
+    "                context = ()\n",
+    "            else:\n",
+    "                context = tuple(generated_text[-self.n+1:])\n",
+    "            if greedy:\n",
+    "                next_word = self.most_probable_next_word(context)\n",
+    "            else:\n",
+    "                candidate_words = list(self.ngrams[context].keys())\n",
+    "                probabilities = [self.get_prob_for_text(generated_text + [word]) for word in candidate_words]\n",
+    "                next_word = random.choices(candidate_words, weights=probabilities)[0]\n",
+    "                \n",
+    "            if next_word == self.UNK_token:\n",
+    "                break\n",
+    "            generated_text.append(next_word)\n",
+    "        \n",
+    "        return generated_text\n",
+    "\n",
+    "    def get_perplexity(self, text: list) -> float:\n",
+    "        if len(text) < self.n - 1:\n",
+    "            raise ValueError(\"Text is too short for the given n-gram order.\")\n",
+    "        \n",
+    "        log_prob = 0.0\n",
+    "        N = 0\n",
+    "        for i in range(len(text) - self.n + 1):\n",
+    "            context = text[i:i + self.n - 1]\n",
+    "            word = text[i + self.n - 1]\n",
+    "            prob = self.get_prob_for_text(context + [word])\n",
+    "            if prob == 0.0:\n",
+    "                return float('inf')\n",
+    "            else:\n",
+    "                log_prob += math.log2(self.get_prob_for_text(context + [word]))\n",
+    "            N += 1\n",
+    "        \n",
+    "        if N == 0:\n",
+    "            return float('inf')\n",
+    "\n",
+    "        avg_log_prob = log_prob / N\n",
+    "        perplexity = 2 ** (-avg_log_prob)\n",
+    "        return perplexity"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Zadanie (60 punktów)\n",
+    "\n",
+    "- Wybierz tekst w dowolnym języku (10 000 000 słów).\n",
+    "- Podziel zbiór na train/test w proporcji 9:1.\n",
+    "- Stwórz unigramowy model językowy.\n",
+    "- Stwórz bigramowy model językowy.\n",
+    "- Stwórz trigramowy model językowy.\n",
+    "- Wymyśl 5 krótkich zdań. Dla każdego oblicz jego prawdopodobieństwo.\n",
+    "- Napisz włąsnoręcznie funkcję, która liczy perplexity na korpusie i policz perplexity na każdym z modeli dla podzbiorów train i test.\n",
+    "- Wygeneruj tekst, zaczynając od wymyślonych 5 początków. Postaraj się, żeby dla obu funkcji, a przynajmniej dla `high_probable_next_word`, teksty były orginalne.\n",
+    "- Stwórz model dla korpusu z ZADANIE 1 i policz perplexity dla każdego z tekstów (zrób split 9:1) dla train i test.\n",
+    "\n",
+    "Dodatkowo:\n",
+    "- Dokonaj klasyfikacji za pomocą modelu językowego.\n",
+    "    - Znajdź duży zbiór danych dla klasyfikacji binarnej, wytrenuj osobne modele dla każdej z klas i użyj dla klasyfikacji.\n",
+    "- Zastosuj wygładzanie metodą Laplace'a."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### START ZADANIA"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Podział korpusu na train/test"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Train size: 90.004%\n",
+      "Test size: 9.996%\n"
+     ]
+    }
+   ],
+   "source": [
+    "corpus = re.split(r'\\s+', open(\"04_materialy/pan-tadeusz.txt\", encoding=\"UTF-8\").read())\n",
+    "\n",
+    "split_index = int(len(corpus) * 0.9)\n",
+    "\n",
+    "while corpus[split_index].endswith(('.', '?', '!')) == False:\n",
+    "    split_index += 1\n",
+    "split_index += 1\n",
+    "\n",
+    "train = corpus[:split_index]\n",
+    "test = corpus[split_index:]\n",
+    "\n",
+    "print(f\"Train size: {len(train)/len(corpus)*100:.3f}%\")\n",
+    "print(f\"Test size: {len(test)/len(corpus)*100:.3f}%\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Unigramowy model języka"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Training unigram model...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 62189/62189 [00:00<00:00, 1066841.60it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"Training unigram model...\")\n",
+    "unigram_model = Model(vocab_size = 300_000, n = 1)\n",
+    "unigram_model.train(train)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Generating text with unigram model... (greedy)\n",
+      "Śród takich pól przed laty, w w w w w w w w w w w w w w w\n",
+      "Generating text with unigram model... (non-greedy)\n",
+      "Śród takich pól przed laty, dworskiej od\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"Generating text with unigram model... (greedy)\")\n",
+    "text = unigram_model.generate_text(re.split(r'\\s+', 'Śród takich pól przed laty,'), 20, greedy = True)\n",
+    "print(' '.join(text))\n",
+    "\n",
+    "print(\"Generating text with unigram model... (non-greedy)\")\n",
+    "text = unigram_model.generate_text(re.split(r'\\s+', 'Śród takich pól przed laty,'), 7, greedy = False)\n",
+    "print(' '.join(text))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Bigramowy model języka"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Training bigram model...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "  0%|          | 0/62188 [00:00<?, ?it/s]"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 62188/62188 [00:00<00:00, 714486.32it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"Training bigram model...\")\n",
+    "bigram_model = Model(vocab_size = 300_000, n = 2)\n",
+    "bigram_model.train(train)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Generating text with bigram model... (greedy)\n",
+      "Śród takich pól przed laty, nad nim się w tym łacniej w tym łacniej w tym łacniej w tym łacniej\n",
+      "Generating text with bigram model... (non-greedy)\n",
+      "Śród takich pól przed laty, nad Woźnego lepiej niedźwiedź kości; Pójdź, księże, w sądy podkomorskie. Dotąd mej Birbante-rokka: Oby ten\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"Generating text with bigram model... (greedy)\")\n",
+    "text = bigram_model.generate_text(re.split(r'\\s+', 'Śród takich pól przed laty,'), 20, greedy = True)\n",
+    "print(' '.join(text))\n",
+    "\n",
+    "print(\"Generating text with bigram model... (non-greedy)\")\n",
+    "text = bigram_model.generate_text(re.split(r'\\s+', 'Śród takich pól przed laty,'), 20, greedy = False)\n",
+    "print(' '.join(text))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Trigramowy model języka"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Training trigram model...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 62187/62187 [00:00<00:00, 295370.31it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"Training trigram model...\")\n",
+    "trigram_model = Model(vocab_size = 300_000, n = 3)\n",
+    "trigram_model.train(train)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Generating text with trigram model... (greedy)\n",
+      "Śród takich pól przed laty, nad brzegiem ruczaju, Na pagórku niewielkim, we brzozowym gaju, Stał dwór szlachecki, z drzewa, lecz podmurowany; Świeciły się z nim na miejscu pustym oczy swe osadzał. Dziwna rzecz! miejsca wkoło są siedzeniem dziewic, Na które\n",
+      "Generating text with trigram model... (non-greedy)\n",
+      "Śród takich pól przed laty, nad brzegiem ruczaju, Na pagórku niewielkim, we brzozowym gaju, Stał dwór szlachecki, z drzewa, gotyckiej naśladowstwo sztuki. Z wierzchu ozdoby sztuczne, nie rylcem, nie dłutem, Ale zręcznie ciesielskim wyrzezane sklutem, Krzywe jak szabasowych ramiona świeczników;\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"Generating text with trigram model... (greedy)\")\n",
+    "text = trigram_model.generate_text(re.split(r'\\s+', 'Śród takich pól przed laty,'), 40, greedy = True)\n",
+    "print(' '.join(text))\n",
+    "\n",
+    "print(\"Generating text with trigram model... (non-greedy)\")\n",
+    "text = trigram_model.generate_text(re.split(r'\\s+', 'Śród takich pól przed laty,'), 40, greedy = False)\n",
+    "print(' '.join(text))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Wymyśl 5 krótkich zdań. Dla każdego oblicz jego prawdopodobieństwo."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Sentence: Nikt go na polowanie\n",
+      "Unigram model: 0.0000000000\n",
+      "Bigram model: 0.0000027142\n",
+      "Trigram model: 0.2000000000\n",
+      "\n",
+      "Sentence: Podróżny długo w oknie stał\n",
+      "Unigram model: 0.0000000000\n",
+      "Bigram model: 0.0000124784\n",
+      "Trigram model: 0.3333333333\n",
+      "\n",
+      "Sentence: Rzekł z uśmiechem,\n",
+      "Unigram model: 0.0000000001\n",
+      "Bigram model: 0.0000521023\n",
+      "Trigram model: 1.0000000000\n",
+      "\n",
+      "Sentence: Pod płotem wąskie, długie, wypukłe pagórki,\n",
+      "Unigram model: 0.0000000000\n",
+      "Bigram model: 0.0192307692\n",
+      "Trigram model: 1.0000000000\n",
+      "\n",
+      "Sentence: Hrabia oczy roztworzył.\n",
+      "Unigram model: 0.0000000000\n",
+      "Bigram model: 0.0004479283\n",
+      "Trigram model: 0.5000000000\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "sentences = [\n",
+    "    \"Nikt go na polowanie\",\n",
+    "    \"Podróżny długo w oknie stał\",\n",
+    "    \"Rzekł z uśmiechem,\",\n",
+    "    \"Pod płotem wąskie, długie, wypukłe pagórki,\",\n",
+    "    \"Hrabia oczy roztworzył.\"\n",
+    "]\n",
+    "\n",
+    "for sentence in sentences:\n",
+    "    sentence = re.split(r'\\s+', sentence)\n",
+    "    print(f\"Sentence: {' '.join(sentence)}\")\n",
+    "    print(f\"Unigram model: {unigram_model.get_prob_for_text(sentence):.10f}\")\n",
+    "    print(f\"Bigram model: {bigram_model.get_prob_for_text(sentence):.10f}\")\n",
+    "    print(f\"Trigram model: {trigram_model.get_prob_for_text(sentence):.10f}\")\n",
+    "    print()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Napisz włąsnoręcznie funkcję, która liczy perplexity na korpusie i policz perplexity na każdym z modeli dla podzbiorów train i test."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Calculating perplexity for unigram model...\n",
+      "Train perplexity: 5666.4901484896\n",
+      "Test perplexity: inf\n",
+      "\n",
+      "\n",
+      "Calculating perplexity for bigram model...\n",
+      "Train perplexity: 9.1369500910\n",
+      "Test perplexity: inf\n",
+      "\n",
+      "\n",
+      "Calculating perplexity for trigram model...\n",
+      "Train perplexity: 1.1857475475\n",
+      "Test perplexity: inf\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"Calculating perplexity for unigram model...\")\n",
+    "train_perplexity = unigram_model.get_perplexity(train)\n",
+    "test_perplexity = unigram_model.get_perplexity(test)\n",
+    "print(f\"Train perplexity: {train_perplexity:.10f}\")\n",
+    "print(f\"Test perplexity: {test_perplexity:.10f}\")\n",
+    "\n",
+    "print(\"\\n\")\n",
+    "\n",
+    "print(\"Calculating perplexity for bigram model...\")\n",
+    "train_perplexity = bigram_model.get_perplexity(train)\n",
+    "test_perplexity = bigram_model.get_perplexity(test)\n",
+    "print(f\"Train perplexity: {train_perplexity:.10f}\")\n",
+    "print(f\"Test perplexity: {test_perplexity:.10f}\")\n",
+    "\n",
+    "print(\"\\n\")\n",
+    "\n",
+    "print(\"Calculating perplexity for trigram model...\")\n",
+    "train_perplexity = trigram_model.get_perplexity(train)\n",
+    "test_perplexity = trigram_model.get_perplexity(test)\n",
+    "print(f\"Train perplexity: {train_perplexity:.10f}\")\n",
+    "print(f\"Test perplexity: {test_perplexity:.10f}\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Wygeneruj tekst, zaczynając od wymyślonych 5 początków. Postaraj się, żeby dla obu funkcji, a przynajmniej dla `high_probable_next_word`, teksty były orginalne."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Sentence: Nikt go na polowanie\n",
+      "Unigram model: Nikt go na polowanie w w w w w w w w w w w w w w w w\n",
+      "Bigram model: Nikt go na polowanie uprosić nie jest w tym łacniej w tym łacniej w tym łacniej w tym łacniej w\n",
+      "Trigram model: Nikt go na polowanie uprosić nie może, Białopiotrowiczowi samemu odmówił! Bo cóż by on na waszych polowaniach łowił? Piękna byłaby\n",
+      "\n",
+      "Sentence: Podróżny długo w oknie stał\n",
+      "Unigram model: Podróżny długo w oknie stał w w w w w w w w w w w w w w w\n",
+      "Bigram model: Podróżny długo w oknie stał w tym łacniej w tym łacniej w tym łacniej w tym łacniej w tym łacniej\n",
+      "Trigram model: Podróżny długo w oknie stał patrząc, dumając, Wonnymi powiewami kwiatów oddychając. Oblicze aż na krzaki fijołkowe skłonił, Oczyma ciekawymi po\n",
+      "\n",
+      "Sentence: Rzekł z uśmiechem,\n",
+      "Unigram model: Rzekł z uśmiechem, w w w w w w w w w w w w w w w w w\n",
+      "Bigram model: Rzekł z uśmiechem, a na kształt ogromnego gmachu, Słońce ostatnich kresów nieba dochodziło, Mniej silnie, ale nie jest w tym\n",
+      "Trigram model: Rzekł z uśmiechem, a był to pan kapitan Ryków, Stary żołnierz, stał w bliskiej wiosce na kwaterze, Pan Sędzia nagłym\n",
+      "\n",
+      "Sentence: Pod płotem wąskie, długie, wypukłe pagórki,\n",
+      "Unigram model: Pod płotem wąskie, długie, wypukłe pagórki, w w w w w w w w w w w w w w\n",
+      "Bigram model: Pod płotem wąskie, długie, wypukłe pagórki, Bez Suwarowa to nie jest w tym łacniej w tym łacniej w tym łacniej\n",
+      "Trigram model: Pod płotem wąskie, długie, wypukłe pagórki, Bez drzew, krzewów i kwiatów: ogród na ogórki. Pięknie wyrosły; liściem wielkim, rozłożystym, Okryły\n",
+      "\n",
+      "Sentence: Hrabia oczy roztworzył.\n",
+      "Unigram model: Hrabia oczy roztworzył. w w w w w w w w w w w w w w w w w\n",
+      "Bigram model: Hrabia oczy roztworzył. Zmieszany, zdziwiony, Milczał; bo w tym łacniej w tym łacniej w tym łacniej w tym łacniej w\n",
+      "Trigram model: Hrabia oczy roztworzył. Zmieszany, zdziwiony, Milczał; wreszcie, zniżając swej rozmowy tony: «Przepraszam — rzekł — mój Rejencie, prawda bez wątpienia,\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "sentences = [\n",
+    "    \"Nikt go na polowanie\",\n",
+    "    \"Podróżny długo w oknie stał\",\n",
+    "    \"Rzekł z uśmiechem,\",\n",
+    "    \"Pod płotem wąskie, długie, wypukłe pagórki,\",\n",
+    "    \"Hrabia oczy roztworzył.\"\n",
+    "]\n",
+    "\n",
+    "for sentence in sentences:\n",
+    "    sentence = re.split(r'\\s+', sentence)\n",
+    "    print(f\"Sentence: {' '.join(sentence)}\")\n",
+    "    print(f\"Unigram model: {' '.join(unigram_model.generate_text(sentence, 20, greedy = True))}\")\n",
+    "    print(f\"Bigram model: {' '.join(bigram_model.generate_text(sentence, 20, greedy = True))}\")\n",
+    "    print(f\"Trigram model: {' '.join(trigram_model.generate_text(sentence, 20, greedy = True))}\")\n",
+    "    print()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Dokonaj klasyfikacji za pomocą modelu językowego.\n",
+    "- Znajdź duży zbiór danych dla klasyfikacji binarnej, wytrenuj osobne modele dla każdej z klas i użyj dla klasyfikacji."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 136592/136592 [00:00<00:00, 357332.74it/s]\n",
+      "100%|██████████| 126878/126878 [00:00<00:00, 299366.46it/s]\n",
+      "100%|██████████| 498/498 [00:00<00:00, 71213.51it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Accuracy: 0.645\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "from datasets import load_dataset\n",
+    "\n",
+    "# Load dataset as sentiment140\n",
+    "dataset = load_dataset(\"sentiment140\")\n",
+    "train = zip(dataset[\"train\"][\"text\"], dataset[\"train\"][\"sentiment\"])\n",
+    "test = zip(dataset[\"test\"][\"text\"], dataset[\"test\"][\"sentiment\"])\n",
+    "\n",
+    "train = list(train)\n",
+    "random.shuffle(train)\n",
+    "train = list(train)[:20_000]\n",
+    "\n",
+    "test = list(test)\n",
+    "random.shuffle(test)\n",
+    "test = list(test)[:1_000]\n",
+    "\n",
+    "pos = [text.split() for text, label in train if label == 0]\n",
+    "neg = [text.split() for text, label in train if label > 0]\n",
+    "\n",
+    "pos_model = Model(vocab_size = 6_000_000, n = 3)\n",
+    "neg_model = Model(vocab_size = 6_000_000, n = 3)\n",
+    "\n",
+    "pos_model.train(sum(pos, []))\n",
+    "neg_model.train(sum(neg, []))\n",
+    "\n",
+    "correct = 0\n",
+    "for text, label in tqdm(test):\n",
+    "    text = text.split()\n",
+    "    pos_perplexity = pos_model.get_perplexity(text)\n",
+    "    neg_perplexity = neg_model.get_perplexity(text)\n",
+    "    result = \"pos\" if pos_perplexity < neg_perplexity else \"neg\"\n",
+    "    if result == \"pos\" and label == 0:\n",
+    "        correct += 1\n",
+    "    elif result == \"neg\" and label > 0:\n",
+    "        correct += 1\n",
+    "\n",
+    "accuracy = correct / len(test)\n",
+    "print(f\"Accuracy: {accuracy:.3f}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Zastosuj wygładzanie metodą Laplace'a."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "on , gdy tańczyłem , krzyknął : » precz za drzwi złodzieja ! « że wtenczas za pułkowej okradzenie kasy\n"
+     ]
+    }
+   ],
+   "source": [
+    "from nltk.lm import Laplace\n",
+    "from nltk.lm.preprocessing import padded_everygram_pipeline\n",
+    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
+    "\n",
+    "n = 5\n",
+    "tokenized_text = [list(map(str.lower, word_tokenize(sent))) for sent in sent_tokenize(open(\"04_materialy/pan-tadeusz.txt\", encoding=\"UTF-8\").read())]\n",
+    "train, vocab = padded_everygram_pipeline(n, tokenized_text)\n",
+    "\n",
+    "model = Laplace(n)\n",
+    "model.fit(train, vocab)\n",
+    "\n",
+    "print(' '.join(model.generate(20, random_seed=42)))\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### KONIEC ZADANIA"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## WYKONANIE ZADAŃ\n",
+    "Zgodnie z instrukcją 01_Kodowanie_tekstu.ipynb"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Teoria informacji"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Wygładzanie modeli językowych"
+   ]
+  }
+ ],
+ "metadata": {
+  "author": "Jakub Pokrywka",
+  "email": "kubapok@wmi.amu.edu.pl",
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "lang": "pl",
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.1"
+  },
+  "subtitle": "0.Informacje na temat przedmiotu[ćwiczenia]",
+  "title": "Ekstrakcja informacji",
+  "year": "2021"
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

src/evaluate.py

@@ -0,0 +1,48 @@
+import sys
+import os
+import pandas as pd
+import csv
+from model import Model
+from tqdm import tqdm
+import re
+import numpy as np
+import math
+
+print("Loading model")
+dataset_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'train', 'in.tsv.xz'))
+model = Model.load(os.path.abspath(os.path.join(os.path.dirname(dataset_dir), 'model.pkl')))
+
+print("Evaluating")
+dataset_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', sys.argv[1], 'in.tsv.xz'))
+output_dir = os.path.abspath(os.path.join(os.path.dirname(dataset_dir), 'out.tsv'))
+
+df = pd.read_csv(dataset_dir, sep='\t', header=None, names=['FileId', 'Year', 'LeftContext', 'RightContext'], quoting=csv.QUOTE_NONE)
+df = df.replace(r'\\r+|\\n+|\\t+', ' ', regex=True)
+
+final = ""
+
+for i, (_, row) in tqdm(enumerate(df.iterrows()), total=len(df)):
+  text = ""
+  prob_sum = 0.0
+  
+  probs = model.fill_gap(re.split(r"\s+", row['LeftContext']), re.split(r"\s+", row['RightContext']))
+  
+  if len(probs) == 0:
+    text = ":1"
+  else:
+    prob_sum = sum([prob for _, prob in probs])
+    
+    for word, prob in probs:
+      new_prob = math.floor(prob / prob_sum * 100) / 100
+      
+      if new_prob == 1.0:
+        new_prob = 0.99
+      
+      text += f"{word}:{new_prob} "
+      
+    text += ":0.01"
+  
+  final += text + "\n"
+
+with open(output_dir, 'w', encoding="UTF-8") as f:
+  f.write(final)

src/model.py

@@ -0,0 +1,111 @@
+from collections import defaultdict, Counter
+from tqdm import tqdm
+import nltk
+import random
+import pickle
+from multiprocessing import Pool
+import math
+from bidict import bidict
+
+class Model():
+    
+    def __init__(self, UNK_token = '<UNK>', n = 3):
+        self.n = n
+        self.UNK_token = UNK_token
+        self.ngrams = defaultdict(defaultdict(int).copy)
+        self.contexts = defaultdict(int)
+        self.tokenizer = bidict({ UNK_token: 0 })
+        self._tokenizer_index = 1
+        self.vocab = set()
+        
+        self.n_split = self.n // 2
+        
+    def train_tokenizer(self, corpus: list) -> list[int]:
+        for word in tqdm(corpus):
+            if word not in self.vocab:
+                self.vocab.add(word)
+                self.tokenizer[word] = self._tokenizer_index
+                
+                self._tokenizer_index += 1
+        
+    def tokenize(self, corpus: list, verbose = False) -> list[int]:
+        result = []
+        
+        for word in tqdm(corpus) if verbose else corpus:
+            if word not in self.vocab:
+                result.append(self.tokenizer[self.UNK_token])
+            else:
+                result.append(self.tokenizer[word])
+        
+        return result
+    
+    def process_gram(self, gram: tuple) -> tuple:
+        left_context = gram[:self.n_split]
+        right_context = gram[self.n_split + 1:]
+        word = gram[self.n_split]
+
+        if word == self.UNK_token:
+            return
+
+        self.ngrams[(left_context, right_context)][word] += 1
+        self.contexts[(left_context, right_context)] += 1
+    
+    def train(self, corpus: list) -> None:
+        
+        print("Training tokenizer")
+        self.train_tokenizer(corpus)
+        
+        print("Tokenizing corpus")
+        corpus = self.tokenize(corpus, verbose = True)
+
+        print("Saving n-grams")
+        n_grams = list(nltk.ngrams(corpus, self.n))
+        for gram in tqdm(n_grams):
+            self.process_gram(gram)
+    
+    def get_conditional_probability_for_word(self, left_context: list, right_context: list, word: str) -> float:
+        left_context = tuple(left_context[-self.n_split:])
+        right_context = tuple(right_context[:self.n_split])
+        
+        total_count = self.contexts[(left_context, right_context)]
+        
+        if total_count == 0:
+            return 0.0
+        else:
+            word_count = self.ngrams[(left_context, right_context)][word]
+
+            return word_count / total_count
+    
+    def get_probabilities(self, left_context: list, right_context: list) -> float:
+        left_context = tuple(left_context[-self.n_split:])
+        right_context = tuple(right_context[:self.n_split])
+
+        words = list(self.ngrams[(left_context, right_context)].keys())
+        probs = []
+
+        for word in words:
+            prob = self.get_conditional_probability_for_word(left_context, right_context, word)
+            probs.append((word, prob))
+        
+        return sorted(probs, reverse = True, key = lambda x: x[1])[:10]
+    
+    def fill_gap(self, left_context: list, right_context: list) -> list:
+        left_context = self.tokenize(left_context)
+        right_context = self.tokenize(right_context)
+        
+        result = []
+        probabilities = self.get_probabilities(left_context, right_context)
+        for token, probability in probabilities:
+            word = self.tokenizer.inverse[token]
+            result.append((word, probability))
+            
+        return result
+    
+    def save(self, output_dir: str) -> None:
+        with open(output_dir, 'wb') as f:
+            pickle.dump(self, f)
+            
+    @staticmethod
+    def load(model_path: str) -> 'Model':
+        with open(model_path, 'rb') as f:
+            return pickle.load(f)

src/train.py

@@ -0,0 +1,32 @@
+from collections import Counter, defaultdict
+from tqdm import tqdm
+import re
+import nltk
+import random
+import os
+import sys
+import pickle
+import csv
+import pandas as pd
+from model import Model
+
+dataset_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'train', 'in.tsv.xz'))
+expected_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'train', 'expected.tsv'))
+
+model = Model(n = 3)
+
+df = pd.read_csv(dataset_dir, sep='\t', header=None, names=['FileId', 'Year', 'LeftContext', 'RightContext'], quoting=csv.QUOTE_NONE, dtype=str, chunksize=1000)
+expected_df = pd.read_csv(expected_dir, sep='\t', header=None, names=['Word'], quoting=csv.QUOTE_NONE, dtype=str, chunksize=1000)
+
+print('Loading training corpus...')
+corpus = []
+for j, (df, expected_df) in tqdm(enumerate(zip(df, expected_df)), total=432):
+    df = df.replace(r'\\r+|\\n+|\\t+', ' ', regex=True)
+    
+    for left_context, word, right_context in zip(df['LeftContext'].to_list(), expected_df['Word'].to_list(), df['LeftContext'].to_list()):
+        corpus.extend(re.split(r"\s+", left_context.strip()) + [str(word).strip()] + re.split(r"\s+", right_context.strip()))
+
+print('Training model...')
+model.train(corpus)
+print('Saving model...')
+model.save(os.path.abspath(os.path.join(os.path.dirname(dataset_dir), 'model.pkl')))