init solution trigram. score 580

.gitignore

@@ -0,0 +1,8 @@
+
+*~
+*.swp
+*.bak
+*.pyc
+*.o
+.DS_Store
+.token

README.md

@@ -0,0 +1,3 @@
+## challenging-america-word-gap-prediction
+### using simple trigram nn
+calculated perplexity: 583.35

config.txt

@@ -0,0 +1 @@
+--metric PerplexityHashed --precision 2  --in-header in-header.tsv  --out-header out-header.tsv

generate_train_txt.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[9]:
+
+
+import regex as re
+from tqdm.notebook import tqdm
+
+def _clean(text):
+	text = text.replace('-\\n', '').replace('\\n', ' ').replace('\\t', ' ')#.replace('<s>','s')
+	while '  ' in text:
+		text = text.replace('  ',' ')
+
+	return re.sub(r'\p{P}', '', text)
+
+def clean(text):
+    text = text.replace('-\\n', '').replace('\\n', ' ').replace('\\t', ' ')
+    text = re.sub(r'\n', ' ', text)
+    text = re.sub(r'(?<=\w)[,-](?=\w)', '', text)
+    text = re.sub(r'\s+', ' ', text)
+    text = re.sub(r'\p{P}', '', text)
+    text = text.strip()
+    return text
+
+def generate_file(input_path, expected_path, output_path):
+	with open(input_path, encoding='utf8') as input_file, open(expected_path, encoding='utf8') as expected_file, open(output_path, 'w', encoding='utf-8') as output_file:
+		for line, word in tqdm(zip(input_file, expected_file), total=432022):
+			columns = line.split('\t')
+			prefix = clean(columns[6])
+			suffix = clean(columns[7])
+
+			train_line = f"{prefix.strip()} {word.strip()} {suffix.strip()}\n"
+
+			output_file.write(train_line)
+
+generate_file('train/in.tsv', 'train/expected.tsv', 'train/train.txt')
+
+
+# In[ ]:
+
+
+
+

in-header.tsv

@@ -0,0 +1 @@
+FileId	Year	LeftContext	RightContext

out-header.tsv

@@ -0,0 +1 @@
+Word

run.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+from torch.utils.data import IterableDataset, DataLoader
+from torchtext.vocab import build_vocab_from_iterator
+
+import regex as re
+import sys
+import itertools
+from itertools import islice
+
+from torch import nn
+import torch
+
+from tqdm.notebook import tqdm
+
+embed_size = 100
+vocab_size = 25_000
+num_epochs = 1
+device = 'cuda'
+batch_size = 2048
+train_file_path = 'train/train.txt'
+
+with open(train_file_path, 'r', encoding='utf-8') as file:
+    total = len(file.readlines())
+
+
+# In[2]:
+
+
+# Function to extract words from a line of text
+def get_words_from_line(line):
+    line = line.rstrip()
+    yield '<s>'
+    for m in re.finditer(r'[\p{L}0-9\*]+|\p{P}+', line):
+        yield m.group(0).lower()
+    yield '</s>'
+
+# Generator to read lines from a file
+def get_word_lines_from_file(file_name):
+    limit = total * 2
+    with open(file_name, 'r', encoding='utf8') as fh:
+        for line in tqdm(fh, total=total):
+            limit -= 1
+            if not limit:
+                break
+            yield get_words_from_line(line)
+
+# Function to create trigrams from a sequence
+def look_ahead_iterator(gen):
+    prev1, prev2 = None, None
+    for item in gen:
+        if prev1 is not None and prev2 is not None:
+            yield (prev2, prev1, item)
+        prev2 = prev1
+        prev1 = item
+
+# Dataset class for trigrams
+class Trigrams(IterableDataset):
+    def __init__(self, text_file, vocabulary_size):
+        self.vocab = build_vocab_from_iterator(
+            get_word_lines_from_file(text_file),
+            max_tokens=vocabulary_size,
+            specials=['<unk>']
+        )
+        self.vocab.set_default_index(self.vocab['<unk>'])
+        self.vocabulary_size = vocabulary_size
+        self.text_file = text_file
+
+    def __iter__(self):
+        return look_ahead_iterator(
+            (self.vocab[t] for t in itertools.chain.from_iterable(get_word_lines_from_file(self.text_file)))
+        )
+
+# Instantiate the dataset
+train_dataset = Trigrams(train_file_path, vocab_size)
+
+
+# In[3]:
+
+
+# Neural network model for trigram language modeling
+class SimpleTrigramNeuralLanguageModel(nn.Module):
+    def __init__(self, vocabulary_size, embedding_size):
+        super(SimpleTrigramNeuralLanguageModel, self).__init__()
+        self.embedding = nn.Embedding(vocabulary_size, embedding_size)
+        self.linear1 = nn.Linear(embedding_size * 2, embedding_size)
+        self.linear2 = nn.Linear(embedding_size, vocabulary_size)
+        self.softmax = nn.Softmax(dim=1)
+        self.embedding_size = embedding_size
+
+    def forward(self, x):
+        embeds = self.embedding(x).view(-1, self.embedding_size * 2)
+        out = torch.relu(self.linear1(embeds))
+        out = self.linear2(out)
+        return self.softmax(out)
+
+# Instantiate the model
+model = SimpleTrigramNeuralLanguageModel(vocab_size, embed_size)
+
+
+# In[4]:
+
+
+model = SimpleTrigramNeuralLanguageModel(vocab_size, embed_size).to(device)
+data = DataLoader(train_dataset, batch_size=batch_size)
+optimizer = torch.optim.Adam(model.parameters())
+criterion = torch.nn.NLLLoss()
+
+model.train()
+step = 0
+for _ in range(num_epochs):
+    for x1,x2,y in tqdm(data, total=total):
+       x = torch.cat((x1,x2), dim=0).to(device)
+       y = y.to(device)
+       optimizer.zero_grad()
+       ypredicted = model(x)
+       loss = criterion(torch.log(ypredicted), y)
+       if step % 5000 == 0:
+          print(step, loss)
+       step += 1
+       loss.backward()
+       optimizer.step()
+
+model.eval()
+
+
+# In[10]:
+
+
+def get_gap_candidates(words, n=10, vocab=train_dataset.vocab):
+    ixs = vocab.forward(words)
+    ixs = torch.tensor(ixs)
+    ixs = torch.cat(tuple([ixs]), dim=0).to(device)
+
+    out = model(ixs)
+    top = torch.topk(out[0], n)
+    top_indices = top.indices.tolist()
+    top_probs = top.values.tolist()
+    top_words = vocab.lookup_tokens(top_indices)
+    return list(zip(top_words, top_probs))
+
+
+# In[11]:
+
+
+def clean(text):
+    text = text.replace('-\\n', '').replace('\\n', ' ').replace('\\t', ' ')
+    text = re.sub(r'\n', ' ', text)
+    text = re.sub(r'(?<=\w)[,-](?=\w)', '', text)
+    text = re.sub(r'\s+', ' ', text)
+    text = re.sub(r'\p{P}', '', text)
+    text = text.strip()
+    return text
+    
+def predictor(prefix):
+    words = clean(prefix)
+    candidates = get_gap_candidates(words.strip().split(' ')[-2:])
+
+    probs_sum = 0
+    output = ''
+    for word,prob in candidates:
+        if word == "<unk>":
+            continue
+        probs_sum += prob
+        output += f"{word}:{prob} "
+    output += f":{1-probs_sum}"
+
+    return output
+
+
+# In[12]:
+
+
+predictor("I really bug")
+
+
+# In[13]:
+
+
+def generate_result(input_path, output_path='out.tsv'):
+    with open(input_path, encoding='utf-8') as f:
+        lines = f.readlines()
+
+    with open(output_path, 'w', encoding='utf-8') as output_file:
+        for line in lines:
+            result = predictor(line)
+            output_file.write(result + '\n')
+
+
+# In[14]:
+
+
+generate_result('dev-0/in.tsv', output_path='dev-0/out.tsv')
+
+
+# In[ ]:
+
+
+
+

trigram.ipynb

@@ -0,0 +1,370 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "dfd117bd-5d6f-46e6-979c-092a8065fa0b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "S:\\WENV\\Lib\\site-packages\\torchtext\\vocab\\__init__.py:4: UserWarning: \n",
+      "/!\\ IMPORTANT WARNING ABOUT TORCHTEXT STATUS /!\\ \n",
+      "Torchtext is deprecated and the last released version will be 0.18 (this one). You can silence this warning by calling the following at the beginnign of your scripts: `import torchtext; torchtext.disable_torchtext_deprecation_warning()`\n",
+      "  warnings.warn(torchtext._TORCHTEXT_DEPRECATION_MSG)\n",
+      "S:\\WENV\\Lib\\site-packages\\torchtext\\utils.py:4: UserWarning: \n",
+      "/!\\ IMPORTANT WARNING ABOUT TORCHTEXT STATUS /!\\ \n",
+      "Torchtext is deprecated and the last released version will be 0.18 (this one). You can silence this warning by calling the following at the beginnign of your scripts: `import torchtext; torchtext.disable_torchtext_deprecation_warning()`\n",
+      "  warnings.warn(torchtext._TORCHTEXT_DEPRECATION_MSG)\n"
+     ]
+    }
+   ],
+   "source": [
+    "from torch.utils.data import IterableDataset, DataLoader\n",
+    "from torchtext.vocab import build_vocab_from_iterator\n",
+    "\n",
+    "import regex as re\n",
+    "import sys\n",
+    "import itertools\n",
+    "from itertools import islice\n",
+    "\n",
+    "from torch import nn\n",
+    "import torch\n",
+    "\n",
+    "from tqdm.notebook import tqdm\n",
+    "\n",
+    "embed_size = 100\n",
+    "vocab_size = 25_000\n",
+    "num_epochs = 1\n",
+    "device = 'cuda'\n",
+    "batch_size = 2048\n",
+    "train_file_path = 'train/train.txt'\n",
+    "\n",
+    "with open(train_file_path, 'r', encoding='utf-8') as file:\n",
+    "    total = len(file.readlines())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "40392665-79bc-4032-a5de-9d189545c9f7",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5049d1e295954b7baf71ac05a793071a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/432022 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# Function to extract words from a line of text\n",
+    "def get_words_from_line(line):\n",
+    "    line = line.rstrip()\n",
+    "    yield '<s>'\n",
+    "    for m in re.finditer(r'[\\p{L}0-9\\*]+|\\p{P}+', line):\n",
+    "        yield m.group(0).lower()\n",
+    "    yield '</s>'\n",
+    "\n",
+    "# Generator to read lines from a file\n",
+    "def get_word_lines_from_file(file_name):\n",
+    "    limit = total * 2\n",
+    "    with open(file_name, 'r', encoding='utf8') as fh:\n",
+    "        for line in tqdm(fh, total=total):\n",
+    "            limit -= 1\n",
+    "            if not limit:\n",
+    "                break\n",
+    "            yield get_words_from_line(line)\n",
+    "\n",
+    "# Function to create trigrams from a sequence\n",
+    "def look_ahead_iterator(gen):\n",
+    "    prev1, prev2 = None, None\n",
+    "    for item in gen:\n",
+    "        if prev1 is not None and prev2 is not None:\n",
+    "            yield (prev2, prev1, item)\n",
+    "        prev2 = prev1\n",
+    "        prev1 = item\n",
+    "\n",
+    "# Dataset class for trigrams\n",
+    "class Trigrams(IterableDataset):\n",
+    "    def __init__(self, text_file, vocabulary_size):\n",
+    "        self.vocab = build_vocab_from_iterator(\n",
+    "            get_word_lines_from_file(text_file),\n",
+    "            max_tokens=vocabulary_size,\n",
+    "            specials=['<unk>']\n",
+    "        )\n",
+    "        self.vocab.set_default_index(self.vocab['<unk>'])\n",
+    "        self.vocabulary_size = vocabulary_size\n",
+    "        self.text_file = text_file\n",
+    "\n",
+    "    def __iter__(self):\n",
+    "        return look_ahead_iterator(\n",
+    "            (self.vocab[t] for t in itertools.chain.from_iterable(get_word_lines_from_file(self.text_file)))\n",
+    "        )\n",
+    "\n",
+    "# Instantiate the dataset\n",
+    "train_dataset = Trigrams(train_file_path, vocab_size)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "0cf7aa68-37aa-48a4-b647-e0e5002ca5c9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Neural network model for trigram language modeling\n",
+    "class SimpleTrigramNeuralLanguageModel(nn.Module):\n",
+    "    def __init__(self, vocabulary_size, embedding_size):\n",
+    "        super(SimpleTrigramNeuralLanguageModel, self).__init__()\n",
+    "        self.embedding = nn.Embedding(vocabulary_size, embedding_size)\n",
+    "        self.linear1 = nn.Linear(embedding_size * 2, embedding_size)\n",
+    "        self.linear2 = nn.Linear(embedding_size, vocabulary_size)\n",
+    "        self.softmax = nn.Softmax(dim=1)\n",
+    "        self.embedding_size = embedding_size\n",
+    "\n",
+    "    def forward(self, x):\n",
+    "        embeds = self.embedding(x).view(-1, self.embedding_size * 2)\n",
+    "        out = torch.relu(self.linear1(embeds))\n",
+    "        out = self.linear2(out)\n",
+    "        return self.softmax(out)\n",
+    "\n",
+    "# Instantiate the model\n",
+    "model = SimpleTrigramNeuralLanguageModel(vocab_size, embed_size)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "0858967e-5143-4253-921d-a009dbbdca27",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c422bb888518406e9f6a4a8f10f2b473",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/432022 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4020a53c31544ef3b4017c43798aa305",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/432022 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0 tensor(10.1654, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
+      "5000 tensor(6.5147, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
+      "10000 tensor(6.6747, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
+      "15000 tensor(6.9061, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
+      "20000 tensor(6.8899, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
+      "25000 tensor(6.8373, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
+      "30000 tensor(6.8942, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
+      "35000 tensor(6.9564, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
+      "40000 tensor(6.9709, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
+      "45000 tensor(6.9592, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
+      "50000 tensor(6.8195, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
+      "55000 tensor(6.7074, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
+      "60000 tensor(6.8755, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
+      "65000 tensor(6.9605, device='cuda:0', grad_fn=<NllLossBackward0>)\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "SimpleTrigramNeuralLanguageModel(\n",
+       "  (embedding): Embedding(25000, 100)\n",
+       "  (linear1): Linear(in_features=200, out_features=100, bias=True)\n",
+       "  (linear2): Linear(in_features=100, out_features=25000, bias=True)\n",
+       "  (softmax): Softmax(dim=1)\n",
+       ")"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model = SimpleTrigramNeuralLanguageModel(vocab_size, embed_size).to(device)\n",
+    "data = DataLoader(train_dataset, batch_size=batch_size)\n",
+    "optimizer = torch.optim.Adam(model.parameters())\n",
+    "criterion = torch.nn.NLLLoss()\n",
+    "\n",
+    "model.train()\n",
+    "step = 0\n",
+    "for _ in range(num_epochs):\n",
+    "    for x1,x2,y in tqdm(data, total=total):\n",
+    "       x = torch.cat((x1,x2), dim=0).to(device)\n",
+    "       y = y.to(device)\n",
+    "       optimizer.zero_grad()\n",
+    "       ypredicted = model(x)\n",
+    "       loss = criterion(torch.log(ypredicted), y)\n",
+    "       if step % 5000 == 0:\n",
+    "          print(step, loss)\n",
+    "       step += 1\n",
+    "       loss.backward()\n",
+    "       optimizer.step()\n",
+    "\n",
+    "model.eval()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "da4d116c-beec-436d-84d8-577282507226",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_gap_candidates(words, n=10, vocab=train_dataset.vocab):\n",
+    "    ixs = vocab.forward(words)\n",
+    "    ixs = torch.tensor(ixs)\n",
+    "    ixs = torch.cat(tuple([ixs]), dim=0).to(device)\n",
+    "\n",
+    "    out = model(ixs)\n",
+    "    top = torch.topk(out[0], n)\n",
+    "    top_indices = top.indices.tolist()\n",
+    "    top_probs = top.values.tolist()\n",
+    "    top_words = vocab.lookup_tokens(top_indices)\n",
+    "    return list(zip(top_words, top_probs))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "0cafd70a-29b3-4a49-b40f-b8ce3143084a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def clean(text):\n",
+    "    text = text.replace('-\\\\n', '').replace('\\\\n', ' ').replace('\\\\t', ' ')\n",
+    "    text = re.sub(r'\\n', ' ', text)\n",
+    "    text = re.sub(r'(?<=\\w)[,-](?=\\w)', '', text)\n",
+    "    text = re.sub(r'\\s+', ' ', text)\n",
+    "    text = re.sub(r'\\p{P}', '', text)\n",
+    "    text = text.strip()\n",
+    "    return text\n",
+    "    \n",
+    "def predictor(prefix):\n",
+    "    words = clean(prefix)\n",
+    "    candidates = get_gap_candidates(words.strip().split(' ')[-2:])\n",
+    "\n",
+    "    probs_sum = 0\n",
+    "    output = ''\n",
+    "    for word,prob in candidates:\n",
+    "        if word == \"<unk>\":\n",
+    "            continue\n",
+    "        probs_sum += prob\n",
+    "        output += f\"{word}:{prob} \"\n",
+    "    output += f\":{1-probs_sum}\"\n",
+    "\n",
+    "    return output"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "2084fb5f-6405-4e44-a06f-953db852e526",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'the:0.07267232984304428 of:0.043321046978235245 and:0.032147664576768875 to:0.02692588046193123 a:0.020654045045375824 in:0.020213929936289787 that:0.010836434550583363 is:0.00959325022995472 it:0.008407277055084705 :0.755228141322732'"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "predictor(\"I really bug\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "965ebaf3-4c0b-4462-8ac5-4746ec9489ab",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def generate_result(input_path, output_path='out.tsv'):\n",
+    "    with open(input_path, encoding='utf-8') as f:\n",
+    "        lines = f.readlines()\n",
+    "\n",
+    "    with open(output_path, 'w', encoding='utf-8') as output_file:\n",
+    "        for line in lines:\n",
+    "            result = predictor(line)\n",
+    "            output_file.write(result + '\\n')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "80547ba7-9d01-4d2b-9e83-269919513de9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "generate_result('dev-0/in.tsv', output_path='dev-0/out.tsv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1d7d72b0-d629-487e-bcec-4756fa88ae49",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}