add old solution with better out scores to new branch
This commit is contained in:
Kacper
parent
380ef29e71
commit
71785832be
21038
dev-0/out.tsv
21038
dev-0/out.tsv
File diff suppressed because it is too large
Load Diff
@ -1,9 +0,0 @@
|
||||
According to recent news and a half of the north side or the other must be a man of great force of character to the country. He was a member of a family in the United States to the full amount of the principal and interest of this section shall be subject to the
|
||||
|
||||
Recent studies have shown that the present condition of things in which we have been in a very short time after the war and the war was over and that every man who has been in the hands of the United States of the North and the South American States, and those States who had
|
||||
|
||||
Today I was taking a stroll in the park when suddenly and that the said estate has by no Mr. ii him, but he was too young to be a very good reason that the above named de- - . . . They are able six e of a good deal of time and money to the amount of the tax
|
||||
|
||||
The most unbelievable story ever told goes like this to be the most important of these are the only two men who were at the time of the year when they tried an 1 the said sum of money to be paid in case of your South and the West may have to do the work of the committee
|
||||
|
||||
he war between natural and the few who are not in the interest of the said William H. and acres of them, more o the State from the control of the state of New York and New York and New York are the looked for food in the greatest number of the most
|
||||
84
simple_bigram.py
Normal file
84
simple_bigram.py
Normal file
@ -0,0 +1,84 @@
|
||||
from collections import Counter
|
||||
import lzma
|
||||
import os
|
||||
|
||||
class BigramModel:
|
||||
def __init__(self):
|
||||
self.vocab = None
|
||||
self.unigram_counts = None
|
||||
self.bigram_counts = None
|
||||
|
||||
def train(self, filename, vocab_size=5000):
|
||||
def get_vocab(filename, vocab_size):
|
||||
print('Generating vocab')
|
||||
file_vocab = Counter()
|
||||
with lzma.open(filename, 'r') as f:
|
||||
for line in f:
|
||||
line = ' '.join(line.decode('utf-8').strip().split('\t')[-2:]).replace(r'\n', ' ').split()
|
||||
line_vocab = Counter(line)
|
||||
file_vocab.update(line_vocab)
|
||||
if len(file_vocab) > vocab_size:
|
||||
file_vocab = [tup[0] for tup in file_vocab.most_common(vocab_size)]
|
||||
else:
|
||||
file_vocab = file_vocab.keys()
|
||||
return file_vocab
|
||||
|
||||
def get_gram_counts(filename):
|
||||
print('Generating unigram and bigram counts')
|
||||
file_unigram_counts = Counter()
|
||||
file_bigram_counts = Counter()
|
||||
with lzma.open(filename, 'r') as f:
|
||||
for line in f:
|
||||
line = line.decode('utf-8').strip().replace(r'\n', ' ').split('\t')[-2:]
|
||||
line_unigram_counts = Counter(' '.join(line).split())
|
||||
file_unigram_counts.update(line_unigram_counts)
|
||||
line_left, line_right = line[0].split(), line[1].split()
|
||||
line_bigram_counts_left = Counter(
|
||||
[tuple(line_left[i: i + 2]) for i in range(len(line_left) - 2 + 1)])
|
||||
line_bigram_counts_right = Counter(
|
||||
[tuple(line_right[i: i + 2]) for i in range(len(line_right) - 2 + 1)])
|
||||
file_bigram_counts.update(line_bigram_counts_left)
|
||||
file_bigram_counts.update(line_bigram_counts_right)
|
||||
return file_unigram_counts, file_bigram_counts
|
||||
|
||||
self.vocab = get_vocab(filename, vocab_size)
|
||||
self.unigram_counts, self.bigram_counts = get_gram_counts(filename)
|
||||
|
||||
def get_bigram_prob(self, bigram, smoothing):
|
||||
if smoothing:
|
||||
return (self.bigram_counts.get(bigram, 0) + 1) / (
|
||||
self.unigram_counts.get(bigram[0], 0) + len(self.vocab) + 1)
|
||||
else:
|
||||
return self.bigram_counts.get(bigram, 0) / self.unigram_counts.get(bigram[0], 1)
|
||||
|
||||
def predict_gaps(self, path, smoothing=True, topk=5):
|
||||
print('Making predictions')
|
||||
with lzma.open(path + '/in.tsv.xz', 'r') as f, open(path + '/out.tsv', 'w', encoding='utf-8') as out:
|
||||
for line in f:
|
||||
line = line.decode('utf-8').replace(r'\n', ' ').split('\t')[-2:]
|
||||
left_context, right_context = line[0].strip().split()[-1], line[1].strip().split()[0]
|
||||
context_probs = dict()
|
||||
for word in self.vocab:
|
||||
left_context_prob = self.get_bigram_prob((left_context, word), smoothing)
|
||||
right_context_prob = self.get_bigram_prob((word, right_context), smoothing)
|
||||
context_probs[word] = left_context_prob * right_context_prob
|
||||
if len(set(context_probs.values())) == 1:
|
||||
out.write('the:0.2 be:0.2 of:0.2\n')
|
||||
else:
|
||||
top_context_probs = sorted(context_probs.items(), key=lambda x: x[1], reverse=True)[:topk]
|
||||
topk_prob_sum = sum([prob for word, prob in top_context_probs])
|
||||
top_context_probs = [(word, (prob / topk_prob_sum)) for word, prob in top_context_probs]
|
||||
probs_string = '\t'.join([f'{word}:{prob}' for word, prob in top_context_probs[-2:] if prob > 0]) # Sadly simply removing last two entries gives way better results...
|
||||
out.write(probs_string + '\n')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
for vocab_size in [5000]:
|
||||
model = BigramModel()
|
||||
model.train('challenging-america-word-gap-prediction/train/in.tsv.xz', vocab_size=vocab_size)
|
||||
for topk in [5]:
|
||||
model.predict_gaps('challenging-america-word-gap-prediction/dev-0', smoothing=False, topk=topk)
|
||||
os.chdir('challenging-america-word-gap-prediction/')
|
||||
print(f'topk:{topk} vocab:{vocab_size}')
|
||||
print(os.system('./geval --test-name dev-0'))
|
||||
os.chdir('../')
|
||||
286
solution.ipynb
286
solution.ipynb
@ -1,286 +0,0 @@
|
||||
{
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"provenance": []
|
||||
},
|
||||
"kernelspec": {
|
||||
"name": "python3",
|
||||
"display_name": "Python 3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
},
|
||||
"accelerator": "GPU",
|
||||
"gpuClass": "standard"
|
||||
},
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"from torchtext.vocab import build_vocab_from_iterator\n",
|
||||
"import pickle\n",
|
||||
"from torch.utils.data import IterableDataset\n",
|
||||
"import itertools\n",
|
||||
"from torch import nn\n",
|
||||
"import torch\n",
|
||||
"import lzma\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"from tqdm import tqdm"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "WnglOFA8gGJl"
|
||||
},
|
||||
"execution_count": 2,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"def simple_preprocess(line):\n",
|
||||
" return line.replace(r'\\n', ' ')\n",
|
||||
"\n",
|
||||
"def get_words_from_line(line):\n",
|
||||
" line = line.strip()\n",
|
||||
" line = simple_preprocess(line)\n",
|
||||
" yield '<s>'\n",
|
||||
" for t in line.split():\n",
|
||||
" yield t\n",
|
||||
" yield '</s>'\n",
|
||||
"\n",
|
||||
"def get_word_lines_from_file(file_name, n_size=-1):\n",
|
||||
" with lzma.open(file_name, 'r') as fh:\n",
|
||||
" n = 0\n",
|
||||
" for line in fh:\n",
|
||||
" n += 1\n",
|
||||
" yield get_words_from_line(line.decode('utf-8'))\n",
|
||||
" if n == n_size:\n",
|
||||
" break\n",
|
||||
"\n",
|
||||
"def look_ahead_iterator(gen):\n",
|
||||
" prev = None\n",
|
||||
" for item in gen:\n",
|
||||
" if prev is not None:\n",
|
||||
" yield prev, item\n",
|
||||
" prev = item\n",
|
||||
"\n",
|
||||
"def build_vocab(file, vocab_size):\n",
|
||||
" try:\n",
|
||||
" with open(f'bigram_nn_vocab_{vocab_size}.pickle', 'rb') as handle:\n",
|
||||
" vocab = pickle.load(handle)\n",
|
||||
" except:\n",
|
||||
" vocab = build_vocab_from_iterator(\n",
|
||||
" get_word_lines_from_file(file),\n",
|
||||
" max_tokens = vocab_size,\n",
|
||||
" specials = ['<unk>'])\n",
|
||||
" with open(f'bigram_nn_vocab_{vocab_size}.pickle', 'wb') as handle:\n",
|
||||
" pickle.dump(vocab, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
|
||||
" return vocab\n",
|
||||
"\n",
|
||||
"class Bigrams(IterableDataset):\n",
|
||||
" def __init__(self, text_file, vocabulary_size):\n",
|
||||
" self.vocab = vocab\n",
|
||||
" self.vocab.set_default_index(self.vocab['<unk>'])\n",
|
||||
" self.vocabulary_size = vocabulary_size\n",
|
||||
" self.text_file = text_file\n",
|
||||
"\n",
|
||||
" def __iter__(self):\n",
|
||||
" return look_ahead_iterator(\n",
|
||||
" (self.vocab[t] for t in itertools.chain.from_iterable(get_word_lines_from_file(self.text_file))))\n",
|
||||
"\n",
|
||||
"class SimpleBigramNeuralLanguageModel(nn.Module):\n",
|
||||
" def __init__(self, vocabulary_size, embedding_size):\n",
|
||||
" super(SimpleBigramNeuralLanguageModel, self).__init__()\n",
|
||||
" self.model = nn.Sequential(\n",
|
||||
" nn.Embedding(vocabulary_size, embedding_size),\n",
|
||||
" nn.Linear(embedding_size, vocabulary_size),\n",
|
||||
" nn.Softmax(dim=1)\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" def forward(self, x):\n",
|
||||
" return self.model(x)"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "aW_3JqSNgLLr"
|
||||
},
|
||||
"execution_count": 3,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"max_steps= -1\n",
|
||||
"vocab_size = 20000\n",
|
||||
"embed_size = 150\n",
|
||||
"batch_size = 5000\n",
|
||||
"learning_rate = 0.001\n",
|
||||
"vocab = build_vocab('challenging-america-word-gap-prediction/train/in.tsv.xz', vocab_size)\n",
|
||||
"train_dataset = Bigrams('challenging-america-word-gap-prediction/train/in.tsv.xz', vocab_size)\n",
|
||||
"if torch.cuda.is_available():\n",
|
||||
" device = 'cuda'\n",
|
||||
"else:\n",
|
||||
" raise Exception()\n",
|
||||
"model = SimpleBigramNeuralLanguageModel(vocab_size, embed_size).to(device)\n",
|
||||
"data = DataLoader(train_dataset, batch_size=batch_size)\n",
|
||||
"optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
|
||||
"criterion = torch.nn.NLLLoss()\n",
|
||||
"\n",
|
||||
"model.train()\n",
|
||||
"step = 0\n",
|
||||
"for x, y in data:\n",
|
||||
" x = x.to(device)\n",
|
||||
" y = y.to(device)\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
" y_predicted = model(x)\n",
|
||||
" loss = criterion(torch.log(y_predicted), y)\n",
|
||||
" if step % 1000 == 0:\n",
|
||||
" print(f'steps: {step}, loss: {loss.item()}')\n",
|
||||
" if step != 0:\n",
|
||||
" torch.save(model.state_dict(), f'bigram_nn_model_steps-{step}_vocab-{vocab_size}_embed-{embed_size}_batch-{batch_size}.bin')\n",
|
||||
" if step == max_steps:\n",
|
||||
" break\n",
|
||||
" step += 1\n",
|
||||
" loss.backward()\n",
|
||||
" optimizer.step()"
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "QQw_E7Ku4h0a",
|
||||
"outputId": "4a37d9ba-1abd-46ae-b157-cd6d52b951a2"
|
||||
},
|
||||
"execution_count": 4,
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/home/ked/PycharmProjects/mj9/venv/lib/python3.10/site-packages/torch/nn/modules/container.py:217: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
|
||||
" input = module(input)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"steps: 0, loss: 10.091094017028809\n",
|
||||
"steps: 1000, loss: 5.73332405090332\n",
|
||||
"steps: 2000, loss: 5.655370712280273\n",
|
||||
"steps: 3000, loss: 5.457630634307861\n",
|
||||
"steps: 4000, loss: 5.38517427444458\n",
|
||||
"steps: 5000, loss: 5.467936992645264\n",
|
||||
"steps: 6000, loss: 5.372152328491211\n",
|
||||
"steps: 7000, loss: 5.272013187408447\n",
|
||||
"steps: 8000, loss: 5.439966201782227\n",
|
||||
"steps: 9000, loss: 5.268238544464111\n",
|
||||
"steps: 10000, loss: 5.1395182609558105\n",
|
||||
"steps: 11000, loss: 5.2558159828186035\n",
|
||||
"steps: 12000, loss: 5.263617515563965\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ename": "KeyboardInterrupt",
|
||||
"evalue": "",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
|
||||
"\u001B[0;31mKeyboardInterrupt\u001B[0m Traceback (most recent call last)",
|
||||
"Cell \u001B[0;32mIn[4], line 31\u001B[0m\n\u001B[1;32m 29\u001B[0m \u001B[38;5;28;01mbreak\u001B[39;00m\n\u001B[1;32m 30\u001B[0m step \u001B[38;5;241m+\u001B[39m\u001B[38;5;241m=\u001B[39m \u001B[38;5;241m1\u001B[39m\n\u001B[0;32m---> 31\u001B[0m \u001B[43mloss\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mbackward\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 32\u001B[0m optimizer\u001B[38;5;241m.\u001B[39mstep()\n",
|
||||
"File \u001B[0;32m~/PycharmProjects/mj9/venv/lib/python3.10/site-packages/torch/_tensor.py:487\u001B[0m, in \u001B[0;36mTensor.backward\u001B[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001B[0m\n\u001B[1;32m 477\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m has_torch_function_unary(\u001B[38;5;28mself\u001B[39m):\n\u001B[1;32m 478\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m handle_torch_function(\n\u001B[1;32m 479\u001B[0m Tensor\u001B[38;5;241m.\u001B[39mbackward,\n\u001B[1;32m 480\u001B[0m (\u001B[38;5;28mself\u001B[39m,),\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 485\u001B[0m inputs\u001B[38;5;241m=\u001B[39minputs,\n\u001B[1;32m 486\u001B[0m )\n\u001B[0;32m--> 487\u001B[0m \u001B[43mtorch\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mautograd\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mbackward\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 488\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mgradient\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mretain_graph\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mcreate_graph\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43minputs\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43minputs\u001B[49m\n\u001B[1;32m 489\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n",
|
||||
"File \u001B[0;32m~/PycharmProjects/mj9/venv/lib/python3.10/site-packages/torch/autograd/__init__.py:200\u001B[0m, in \u001B[0;36mbackward\u001B[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001B[0m\n\u001B[1;32m 195\u001B[0m retain_graph \u001B[38;5;241m=\u001B[39m create_graph\n\u001B[1;32m 197\u001B[0m \u001B[38;5;66;03m# The reason we repeat same the comment below is that\u001B[39;00m\n\u001B[1;32m 198\u001B[0m \u001B[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001B[39;00m\n\u001B[1;32m 199\u001B[0m \u001B[38;5;66;03m# calls in the traceback and some print out the last line\u001B[39;00m\n\u001B[0;32m--> 200\u001B[0m \u001B[43mVariable\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_execution_engine\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrun_backward\u001B[49m\u001B[43m(\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;66;43;03m# Calls into the C++ engine to run the backward pass\u001B[39;49;00m\n\u001B[1;32m 201\u001B[0m \u001B[43m \u001B[49m\u001B[43mtensors\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mgrad_tensors_\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mretain_graph\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mcreate_graph\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43minputs\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 202\u001B[0m \u001B[43m \u001B[49m\u001B[43mallow_unreachable\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mTrue\u001B[39;49;00m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43maccumulate_grad\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mTrue\u001B[39;49;00m\u001B[43m)\u001B[49m\n",
|
||||
"\u001B[0;31mKeyboardInterrupt\u001B[0m: "
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"vocab_size = 20000\n",
|
||||
"embed_size = 150\n",
|
||||
"batch_size = 5000\n",
|
||||
"vocab = build_vocab('challenging-america-word-gap-prediction/train/in.tsv.xz', vocab_size)\n",
|
||||
"vocab.set_default_index(vocab['<unk>'])"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "N9-wmLOEZ2aV"
|
||||
},
|
||||
"execution_count": 5,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"topk = 5\n",
|
||||
"preds = []\n",
|
||||
"device = 'cuda'\n",
|
||||
"model = SimpleBigramNeuralLanguageModel(vocab_size, embed_size).to(device)\n",
|
||||
"model.load_state_dict(torch.load('bigram_nn_model_steps-10000_vocab-20000_embed-150_batch-5000.bin'))\n",
|
||||
"model.eval()\n",
|
||||
"for path in ['challenging-america-word-gap-prediction/dev-0', 'challenging-america-word-gap-prediction/test-A']:\n",
|
||||
" with lzma.open(f'{path}/in.tsv.xz', 'r') as fh, open(f'{path}/out.tsv', 'w', encoding='utf-8') as f_out:\n",
|
||||
" for line in fh:\n",
|
||||
" previous_word = simple_preprocess(line.decode('utf-8').split('\\t')[-2].strip()).split()[-1]\n",
|
||||
" ixs = torch.tensor(vocab.forward([previous_word])).to(device)\n",
|
||||
" out = model(ixs)\n",
|
||||
" top = torch.topk(out[0], topk)\n",
|
||||
" top_indices = top.indices.tolist()\n",
|
||||
" top_probs = top.values.tolist()\n",
|
||||
" top_words = vocab.lookup_tokens(top_indices)\n",
|
||||
" top_zipped = zip(top_words, top_probs)\n",
|
||||
" pred = ''\n",
|
||||
" total_prob = 0\n",
|
||||
" for word, prob in top_zipped:\n",
|
||||
" if word != '<unk>':\n",
|
||||
" pred += f'{word}:{prob} '\n",
|
||||
" total_prob += prob\n",
|
||||
" unk_prob = 1 - total_prob\n",
|
||||
" pred += f':{unk_prob}'\n",
|
||||
" f_out.write(pred + '\\n')"
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "99uioFpVCJL8",
|
||||
"outputId": "d4267cb1-e557-478a-8cf7-91a90db07698"
|
||||
},
|
||||
"execution_count": 24,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 25,
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/home/ked/PycharmProjects/mj9/challenging-america-word-gap-prediction\n",
|
||||
"394.97\r\n",
|
||||
"/home/ked/PycharmProjects/mj9\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%cd challenging-america-word-gap-prediction/\n",
|
||||
"!./geval --test-name dev-0\n",
|
||||
"%cd ../"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"outputs": [],
|
||||
"source": [],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
14828
test-A/out.tsv
14828
test-A/out.tsv
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user