test

challenging-america-word-gap-prediction/README.md

@@ -1,9 +0,0 @@
-Challenging America word-gap prediction
-===================================
-
-Guess a word in a gap.
-
-Evaluation metric
------------------
-
-LikelihoodHashed is the metric

notebook.ipynb

@@ -1,163 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "unxz: challenging-america-word-gap-prediction/train/in.tsv.xz: No such file or directory\n",
-      "unxz: challenging-america-word-gap-prediction/test-A/in.tsv.xz: No such file or directory\n",
-      "unxz: challenging-america-word-gap-prediction/dev-0/in.tsv.xz: No such file or directory\n"
-     ]
-    }
-   ],
-   "source": [
-    "!unxz challenging-america-word-gap-prediction/train/in.tsv.xz --keep\n",
-    "!unxz challenging-america-word-gap-prediction/test-A/in.tsv.xz --keep\n",
-    "!unxz challenging-america-word-gap-prediction/dev-0/in.tsv.xz --keep"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "expected.tsv  in.tsv\n"
-     ]
-    }
-   ],
-   "source": [
-    "!ls challenging-america-word-gap-prediction/train"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 50,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "All texts: 10\n",
-      "All labels: 10\n"
-     ]
-    }
-   ],
-   "source": [
-    "import nltk\n",
-    "def get_texts():\n",
-    "    with open(\"challenging-america-word-gap-prediction/train/in.tsv\", \"r\", encoding=\"UTF-8\") as f:\n",
-    "        i = 0\n",
-    "        while True:\n",
-    "            i+=1\n",
-    "            text = f.readline()\n",
-    "            if(text == None or i > 10):\n",
-    "                break\n",
-    "            text = text.split('\\t')[6]\n",
-    "            text = text.replace(\"-\\n\", \"\").replace(\"\\n\", \" \")\n",
-    "            yield \n",
-    "\n",
-    "# def get_words():\n",
-    "#     for text in get_texts():\n",
-    "#         for word in nltk.word_tokenize(text):\n",
-    "#             yield word\n",
-    "\n",
-    "def get_labels():\n",
-    "    with open(\"challenging-america-word-gap-prediction/train/expected.tsv\", \"r\", encoding=\"UTF-8\") as f:\n",
-    "        yield from f.readlines()[0:10]\n",
-    "\n",
-    "texts_sum = sum(1 for text in get_texts())\n",
-    "labels_sum = sum(1 for label in get_labels())\n",
-    "# words_sum = sum(1 for word in get_words())\n",
-    "print(f\"All texts: {texts_sum}\")\n",
-    "print(f\"All labels: {labels_sum}\")\n",
-    "# print(f\"All words: {words_sum}\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 51,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "None\n",
-      "None\n",
-      "None\n",
-      "None\n",
-      "None\n",
-      "None\n",
-      "None\n",
-      "None\n",
-      "None\n",
-      "None\n"
-     ]
-    }
-   ],
-   "source": [
-    "for text in get_texts():\n",
-    "    print(text)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Model bigramowy odwrotny"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "class Model():\n",
-    "    def __init__(self, vocab_size, UNK_token= '<UNK>'):\n",
-    "        pass\n",
-    "    \n",
-    "    def train(corpus:list) -> None:\n",
-    "        pass\n",
-    "    \n",
-    "    def predict(text: list, probs: str) -> float:\n",
-    "        pass"
-   ]
-  }
- ],
- "metadata": {
-  "interpreter": {
-   "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
-  },
-  "kernelspec": {
-   "display_name": "Python 3.8.5 64-bit",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.8.5"
-  },
-  "orig_nbformat": 4
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

run.py

@@ -0,0 +1,81 @@
+#%%
+import pandas as pd
+from collections import defaultdict, Counter
+
+from sqlalchemy import true
+from nltk import trigrams, word_tokenize, bigrams
+import csv
+
+#%%
+class Model:
+    def __init__(self):
+        self.model = defaultdict(lambda: defaultdict(lambda: 0))
+        self.model_bi = defaultdict(lambda: defaultdict(lambda: 0))
+        train_in = pd.read_csv("train/in.tsv.xz", sep='\t', header=None, encoding="UTF-8", on_bad_lines="skip", quoting=csv.QUOTE_NONE)[[6, 7]]
+        train_expected = pd.read_csv("train/expected.tsv", sep='\t', header=None, encoding="UTF-8", on_bad_lines="skip", quoting=csv.QUOTE_NONE)
+        data = pd.concat([train_in, train_expected], axis=1)
+        self.data = data[6] + data[0] + data[7]
+        self.data = self.data.apply(self.clean)
+
+    def clean(self, text):
+        text = str(text).lower().strip().replace("’", "'").replace('\\n', " ").replace("'t", " not").replace("'s", " is").replace("'ll", " will").replace("'m", " am").replace("'ve", " have").replace(",", "").replace("-", "")
+        return text
+
+    def train(self):
+        alpha = 0.7
+        vocab = set()
+        for text in model.data:
+            words = word_tokenize(text)
+            for w1, w2, w3 in trigrams(words, pad_left=True, pad_right=True):
+                self.model[w1, w2][w3] += 1
+                vocab.add(w1)
+                vocab.add(w2)
+                vocab.add(w3)
+            for w1, w2 in bigrams(words, pad_left=True, pad_right=True):
+                self.model_bi[w1][w2] +=1
+        for w1, w2 in self.model:
+            total_count = float(sum(self.model[w1, w2].values()))
+            denominator = total_count * len(vocab)
+            for w in self.model[w1, w2]:
+                self.model[w1, w2][w] = self.model[w1, w2][w] / denominator * alpha
+        for w1 in self.model_bi:
+            total_count = float(sum(self.model_bi[w1].values()))
+            denominator = total_count * len(vocab)
+            for w in self.model_bi[w1]:
+                self.model_bi[w1][w] = self.model_bi[w1][w] / denominator * (1-alpha)
+
+    def predict(self, words):
+        trigrams = Counter(dict(self.model[words]))
+        bigrams = Counter(dict(self.model_bi[words[-1]]))
+        predictions = dict((trigrams + bigrams).most_common(6))
+        total_prob = 0
+        
+        result = ""
+        for word, prob in predictions.items():
+            total_prob += prob
+            result += f"{word}:{prob} "
+
+        if len(result) == 0:
+            return "a:0.2 the:0.2 to:0.2 of:0.1 and:0.1 of:0.1 :0.1"
+        return result + f":{max(1-total_prob, 0.01)}"
+
+model = Model()
+
+#%%
+model.data
+model.train()
+
+#%%
+def predict(model, path, result_path):
+    data = pd.read_csv(path, sep='\t', header=None, encoding="UTF-8", on_bad_lines="skip", quoting=csv.QUOTE_NONE)[7]
+    with open(result_path, "w+", encoding="UTF-8") as f:
+        for text in data:
+            words = word_tokenize(model.clean(text))
+            if len(words) < 2:
+                prediction = "a:0.2 the:0.2 to:0.2 of:0.1 and:0.1 of:0.1 :0.1"
+            else:
+                prediction = model.predict((words[-2], words[-1]))
+            f.write(prediction + "\n")
+
+predict(model, "dev-0/in.tsv.xz", "dev-0/out.tsv")
+predict(model, "test-A/in.tsv.xz", "test-A/out.tsv")