From bfd5df1fb6dce709c79d69c44087ea9479da63e7 Mon Sep 17 00:00:00 2001 From: wangobango Date: Wed, 27 Apr 2022 15:40:26 +0200 Subject: [PATCH] init --- main.ipynb | 198 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 main.ipynb diff --git a/main.ipynb b/main.ipynb new file mode 100644 index 0000000..a3c52ab --- /dev/null +++ b/main.ipynb @@ -0,0 +1,198 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from torchtext.vocab import build_vocab_from_iterator\n", + "from nltk.tokenize import word_tokenize\n", + "\n", + "\n", + "def get_words_from_line(line):\n", + " line = line.split('\\t')\n", + " left_context = line[2]\n", + " right_context = line[3]\n", + " line = ' '.join([left_context, right_context])\n", + " tokenized = word_tokenize(line)\n", + " for word in tokenized:\n", + " yield word.lower()\n", + "\n", + "def get_context_from_line(line):\n", + " line = line.split('\\t')\n", + " left_context = line[2]\n", + " right_context = line[3]\n", + " left_context = [cleanString(t) for t in word_tokenize(left_context)]\n", + " right_context = [cleanString(t) for t in word_tokenize(right_context)]\n", + " yield left_context[-2:] + right_context[:2]\n", + "\n", + "def cleanString(value):\n", + " value = value.strip('')\n", + " value = value.strip('\\n')\n", + " value = value.strip('\\t')\n", + " value = value.lower()\n", + " return value\n", + "\n", + "def get_word_lines_from_file(file_name):\n", + " with open(file_name, 'r') as fh:\n", + " for line in fh:\n", + " yield get_words_from_line(line)\n", + "\n", + "vocab_size = 20000\n", + "\n", + "vocab = build_vocab_from_iterator(\n", + " get_word_lines_from_file('dev-0/in.tsv'),\n", + " max_tokens = vocab_size,\n", + " specials = [''])\n", + "\n", + "vocab.set_default_index(vocab[''])" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "from torch import nn\n", + "import torch\n", + "\n", + "embed_size = 100\n", + "context_size = 4\n", + "\n", + "class SimpleNgramNeuralLanguageModel(nn.Module):\n", + " def __init__(self, vocabulary_size, embedding_size, context_size):\n", + " super(SimpleNgramNeuralLanguageModel, self).__init__()\n", + " self.embedding = nn.Embedding(vocabulary_size, embedding_size)\n", + " self.fn1 = nn.Linear(embedding_size * context_size, vocabulary_size)\n", + " self.fn2 = nn.Linear(vocabulary_size, vocabulary_size)\n", + " self.out = nn.Softmax(dim=0)\n", + "\n", + " def forward(self, x):\n", + " x = torch.as_tensor([torch.cat((self.embedding(val[0]), self.embedding(val[1]), self.embedding(val[2]), self.embedding(val[3]))) for val in x])\n", + " x = self.fn1(x)\n", + " x = self.fn2(x)\n", + " x = self.out(x)\n", + " return x\n", + "\n", + "model = SimpleNgramNeuralLanguageModel(vocab_size, embed_size, context_size)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "from torch.utils.data import IterableDataset, DataLoader\n", + "\n", + "class Ngrams(IterableDataset):\n", + " def __init__(self, text_file, vocabulary_size, expected_file):\n", + " self.vocab = build_vocab_from_iterator(\n", + " get_word_lines_from_file(text_file),\n", + " max_tokens = vocabulary_size,\n", + " specials = [''])\n", + " self.vocab.set_default_index(self.vocab[''])\n", + " self.vocabulary_size = vocabulary_size\n", + " self.text_file = text_file\n", + " self.expected_file = expected_file\n", + "\n", + " def __iter__(self):\n", + " with open(self.text_file, 'r') as inData:\n", + " with open(self.expected_file, 'r') as expData:\n", + " for lineIn, lineExp in zip(inData, expData):\n", + " yield torch.as_tensor([self.vocab[t] for chunk in get_context_from_line(lineIn) for t in chunk]), torch.as_tensor(self.vocab[cleanString(lineExp)])\n", + "\n", + "train_dataset = Ngrams('dev-0/in.tsv', vocab_size, 'dev-0/expected.tsv')" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[239, 466, 232, 1]])\n", + "tensor([2])\n" + ] + }, + { + "ename": "RuntimeError", + "evalue": "CUDA error: no kernel image is available for execution on the device\nCUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m/home/ramon/projects/retro-gap/main.ipynb Cell 4'\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 13\u001b[0m y \u001b[39m=\u001b[39m y\u001b[39m.\u001b[39mto(device)\n\u001b[1;32m 14\u001b[0m optimizer\u001b[39m.\u001b[39mzero_grad()\n\u001b[0;32m---> 15\u001b[0m ypredicted \u001b[39m=\u001b[39m model(x)\n\u001b[1;32m 16\u001b[0m \u001b[39mprint\u001b[39m(ypredicted)\n\u001b[1;32m 17\u001b[0m loss \u001b[39m=\u001b[39m criterion(torch\u001b[39m.\u001b[39mlog(ypredicted), y)\n", + "File \u001b[0;32m~/.local/lib/python3.10/site-packages/torch/nn/modules/module.py:1110\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1106\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1107\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1108\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1109\u001b[0m \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1110\u001b[0m \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 1111\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1112\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n", + "\u001b[1;32m/home/ramon/projects/retro-gap/main.ipynb Cell 2'\u001b[0m in \u001b[0;36mSimpleNgramNeuralLanguageModel.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, x):\n\u001b[0;32m---> 16\u001b[0m x \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mas_tensor([torch\u001b[39m.\u001b[39mcat((\u001b[39mself\u001b[39m\u001b[39m.\u001b[39membedding(val[\u001b[39m0\u001b[39m]), \u001b[39mself\u001b[39m\u001b[39m.\u001b[39membedding(val[\u001b[39m1\u001b[39m]), \u001b[39mself\u001b[39m\u001b[39m.\u001b[39membedding(val[\u001b[39m2\u001b[39m]), \u001b[39mself\u001b[39m\u001b[39m.\u001b[39membedding(val[\u001b[39m3\u001b[39m]))) \u001b[39mfor\u001b[39;00m val \u001b[39min\u001b[39;00m x])\n\u001b[1;32m 17\u001b[0m x \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfn1(x)\n\u001b[1;32m 18\u001b[0m x \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfn2(x)\n", + "\u001b[1;32m/home/ramon/projects/retro-gap/main.ipynb Cell 2'\u001b[0m in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, x):\n\u001b[0;32m---> 16\u001b[0m x \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mas_tensor([torch\u001b[39m.\u001b[39mcat((\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49membedding(val[\u001b[39m0\u001b[39;49m]), \u001b[39mself\u001b[39m\u001b[39m.\u001b[39membedding(val[\u001b[39m1\u001b[39m]), \u001b[39mself\u001b[39m\u001b[39m.\u001b[39membedding(val[\u001b[39m2\u001b[39m]), \u001b[39mself\u001b[39m\u001b[39m.\u001b[39membedding(val[\u001b[39m3\u001b[39m]))) \u001b[39mfor\u001b[39;00m val \u001b[39min\u001b[39;00m x])\n\u001b[1;32m 17\u001b[0m x \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfn1(x)\n\u001b[1;32m 18\u001b[0m x \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfn2(x)\n", + "File \u001b[0;32m~/.local/lib/python3.10/site-packages/torch/nn/modules/module.py:1110\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1106\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1107\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1108\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1109\u001b[0m \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1110\u001b[0m \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 1111\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1112\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n", + "File \u001b[0;32m~/.local/lib/python3.10/site-packages/torch/nn/modules/sparse.py:158\u001b[0m, in \u001b[0;36mEmbedding.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 157\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39minput\u001b[39m: Tensor) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m Tensor:\n\u001b[0;32m--> 158\u001b[0m \u001b[39mreturn\u001b[39;00m F\u001b[39m.\u001b[39;49membedding(\n\u001b[1;32m 159\u001b[0m \u001b[39minput\u001b[39;49m, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mweight, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mpadding_idx, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmax_norm,\n\u001b[1;32m 160\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mnorm_type, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mscale_grad_by_freq, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msparse)\n", + "File \u001b[0;32m~/.local/lib/python3.10/site-packages/torch/nn/functional.py:2183\u001b[0m, in \u001b[0;36membedding\u001b[0;34m(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)\u001b[0m\n\u001b[1;32m 2177\u001b[0m \u001b[39m# Note [embedding_renorm set_grad_enabled]\u001b[39;00m\n\u001b[1;32m 2178\u001b[0m \u001b[39m# XXX: equivalent to\u001b[39;00m\n\u001b[1;32m 2179\u001b[0m \u001b[39m# with torch.no_grad():\u001b[39;00m\n\u001b[1;32m 2180\u001b[0m \u001b[39m# torch.embedding_renorm_\u001b[39;00m\n\u001b[1;32m 2181\u001b[0m \u001b[39m# remove once script supports set_grad_enabled\u001b[39;00m\n\u001b[1;32m 2182\u001b[0m _no_grad_embedding_renorm_(weight, \u001b[39minput\u001b[39m, max_norm, norm_type)\n\u001b[0;32m-> 2183\u001b[0m \u001b[39mreturn\u001b[39;00m torch\u001b[39m.\u001b[39;49membedding(weight, \u001b[39minput\u001b[39;49m, padding_idx, scale_grad_by_freq, sparse)\n", + "\u001b[0;31mRuntimeError\u001b[0m: CUDA error: no kernel image is available for execution on the device\nCUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1." + ] + } + ], + "source": [ + "device = 'cuda'\n", + "model = SimpleNgramNeuralLanguageModel(vocab_size, embed_size, context_size).to(device)\n", + "data = DataLoader(train_dataset, batch_size=1)\n", + "optimizer = torch.optim.Adam(model.parameters())\n", + "criterion = torch.nn.NLLLoss()\n", + " \n", + "model.train()\n", + "step = 0\n", + "for x, y in data:\n", + " print(x)\n", + " print(y)\n", + " x = x.to(device)\n", + " y = y.to(device)\n", + " optimizer.zero_grad()\n", + " ypredicted = model(x)\n", + " print(ypredicted)\n", + " loss = criterion(torch.log(ypredicted), y)\n", + " if step % 100 == 0:\n", + " print(step, loss)\n", + " step += 1\n", + " loss.backward()\n", + " optimizer.step()\n", + " break\n", + " \n", + "# torch.save(model.state_dict(), 'model1.bin')\n", + "\n", + "#Policzmy najbardziej prawdopodobne kontynuację dla zadanego słowa:\n", + "\n" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a" + }, + "kernelspec": { + "display_name": "Python 3.10.2 64-bit", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.2" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +}