This commit is contained in:
wangobango 2022-04-27 15:40:26 +02:00
parent c8996883e3
commit bfd5df1fb6

198
main.ipynb Normal file
View File

@ -0,0 +1,198 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"from torchtext.vocab import build_vocab_from_iterator\n",
"from nltk.tokenize import word_tokenize\n",
"\n",
"\n",
"def get_words_from_line(line):\n",
" line = line.split('\\t')\n",
" left_context = line[2]\n",
" right_context = line[3]\n",
" line = ' '.join([left_context, right_context])\n",
" tokenized = word_tokenize(line)\n",
" for word in tokenized:\n",
" yield word.lower()\n",
"\n",
"def get_context_from_line(line):\n",
" line = line.split('\\t')\n",
" left_context = line[2]\n",
" right_context = line[3]\n",
" left_context = [cleanString(t) for t in word_tokenize(left_context)]\n",
" right_context = [cleanString(t) for t in word_tokenize(right_context)]\n",
" yield left_context[-2:] + right_context[:2]\n",
"\n",
"def cleanString(value):\n",
" value = value.strip('')\n",
" value = value.strip('\\n')\n",
" value = value.strip('\\t')\n",
" value = value.lower()\n",
" return value\n",
"\n",
"def get_word_lines_from_file(file_name):\n",
" with open(file_name, 'r') as fh:\n",
" for line in fh:\n",
" yield get_words_from_line(line)\n",
"\n",
"vocab_size = 20000\n",
"\n",
"vocab = build_vocab_from_iterator(\n",
" get_word_lines_from_file('dev-0/in.tsv'),\n",
" max_tokens = vocab_size,\n",
" specials = ['<unk>'])\n",
"\n",
"vocab.set_default_index(vocab['<unk>'])"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"from torch import nn\n",
"import torch\n",
"\n",
"embed_size = 100\n",
"context_size = 4\n",
"\n",
"class SimpleNgramNeuralLanguageModel(nn.Module):\n",
" def __init__(self, vocabulary_size, embedding_size, context_size):\n",
" super(SimpleNgramNeuralLanguageModel, self).__init__()\n",
" self.embedding = nn.Embedding(vocabulary_size, embedding_size)\n",
" self.fn1 = nn.Linear(embedding_size * context_size, vocabulary_size)\n",
" self.fn2 = nn.Linear(vocabulary_size, vocabulary_size)\n",
" self.out = nn.Softmax(dim=0)\n",
"\n",
" def forward(self, x):\n",
" x = torch.as_tensor([torch.cat((self.embedding(val[0]), self.embedding(val[1]), self.embedding(val[2]), self.embedding(val[3]))) for val in x])\n",
" x = self.fn1(x)\n",
" x = self.fn2(x)\n",
" x = self.out(x)\n",
" return x\n",
"\n",
"model = SimpleNgramNeuralLanguageModel(vocab_size, embed_size, context_size)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"from torch.utils.data import IterableDataset, DataLoader\n",
"\n",
"class Ngrams(IterableDataset):\n",
" def __init__(self, text_file, vocabulary_size, expected_file):\n",
" self.vocab = build_vocab_from_iterator(\n",
" get_word_lines_from_file(text_file),\n",
" max_tokens = vocabulary_size,\n",
" specials = ['<unk>'])\n",
" self.vocab.set_default_index(self.vocab['<unk>'])\n",
" self.vocabulary_size = vocabulary_size\n",
" self.text_file = text_file\n",
" self.expected_file = expected_file\n",
"\n",
" def __iter__(self):\n",
" with open(self.text_file, 'r') as inData:\n",
" with open(self.expected_file, 'r') as expData:\n",
" for lineIn, lineExp in zip(inData, expData):\n",
" yield torch.as_tensor([self.vocab[t] for chunk in get_context_from_line(lineIn) for t in chunk]), torch.as_tensor(self.vocab[cleanString(lineExp)])\n",
"\n",
"train_dataset = Ngrams('dev-0/in.tsv', vocab_size, 'dev-0/expected.tsv')"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"tensor([[239, 466, 232, 1]])\n",
"tensor([2])\n"
]
},
{
"ename": "RuntimeError",
"evalue": "CUDA error: no kernel image is available for execution on the device\nCUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m/home/ramon/projects/retro-gap/main.ipynb Cell 4'\u001b[0m in \u001b[0;36m<cell line: 9>\u001b[0;34m()\u001b[0m\n\u001b[1;32m <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000003?line=12'>13</a>\u001b[0m y \u001b[39m=\u001b[39m y\u001b[39m.\u001b[39mto(device)\n\u001b[1;32m <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000003?line=13'>14</a>\u001b[0m optimizer\u001b[39m.\u001b[39mzero_grad()\n\u001b[0;32m---> <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000003?line=14'>15</a>\u001b[0m ypredicted \u001b[39m=\u001b[39m model(x)\n\u001b[1;32m <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000003?line=15'>16</a>\u001b[0m \u001b[39mprint\u001b[39m(ypredicted)\n\u001b[1;32m <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000003?line=16'>17</a>\u001b[0m loss \u001b[39m=\u001b[39m criterion(torch\u001b[39m.\u001b[39mlog(ypredicted), y)\n",
"File \u001b[0;32m~/.local/lib/python3.10/site-packages/torch/nn/modules/module.py:1110\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1105'>1106</a>\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1106'>1107</a>\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1107'>1108</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1108'>1109</a>\u001b[0m \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1109'>1110</a>\u001b[0m \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1110'>1111</a>\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1111'>1112</a>\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
"\u001b[1;32m/home/ramon/projects/retro-gap/main.ipynb Cell 2'\u001b[0m in \u001b[0;36mSimpleNgramNeuralLanguageModel.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000001?line=14'>15</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, x):\n\u001b[0;32m---> <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000001?line=15'>16</a>\u001b[0m x \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mas_tensor([torch\u001b[39m.\u001b[39mcat((\u001b[39mself\u001b[39m\u001b[39m.\u001b[39membedding(val[\u001b[39m0\u001b[39m]), \u001b[39mself\u001b[39m\u001b[39m.\u001b[39membedding(val[\u001b[39m1\u001b[39m]), \u001b[39mself\u001b[39m\u001b[39m.\u001b[39membedding(val[\u001b[39m2\u001b[39m]), \u001b[39mself\u001b[39m\u001b[39m.\u001b[39membedding(val[\u001b[39m3\u001b[39m]))) \u001b[39mfor\u001b[39;00m val \u001b[39min\u001b[39;00m x])\n\u001b[1;32m <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000001?line=16'>17</a>\u001b[0m x \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfn1(x)\n\u001b[1;32m <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000001?line=17'>18</a>\u001b[0m x \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfn2(x)\n",
"\u001b[1;32m/home/ramon/projects/retro-gap/main.ipynb Cell 2'\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000001?line=14'>15</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, x):\n\u001b[0;32m---> <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000001?line=15'>16</a>\u001b[0m x \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mas_tensor([torch\u001b[39m.\u001b[39mcat((\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49membedding(val[\u001b[39m0\u001b[39;49m]), \u001b[39mself\u001b[39m\u001b[39m.\u001b[39membedding(val[\u001b[39m1\u001b[39m]), \u001b[39mself\u001b[39m\u001b[39m.\u001b[39membedding(val[\u001b[39m2\u001b[39m]), \u001b[39mself\u001b[39m\u001b[39m.\u001b[39membedding(val[\u001b[39m3\u001b[39m]))) \u001b[39mfor\u001b[39;00m val \u001b[39min\u001b[39;00m x])\n\u001b[1;32m <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000001?line=16'>17</a>\u001b[0m x \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfn1(x)\n\u001b[1;32m <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000001?line=17'>18</a>\u001b[0m x \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfn2(x)\n",
"File \u001b[0;32m~/.local/lib/python3.10/site-packages/torch/nn/modules/module.py:1110\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1105'>1106</a>\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1106'>1107</a>\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1107'>1108</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1108'>1109</a>\u001b[0m \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1109'>1110</a>\u001b[0m \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1110'>1111</a>\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1111'>1112</a>\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
"File \u001b[0;32m~/.local/lib/python3.10/site-packages/torch/nn/modules/sparse.py:158\u001b[0m, in \u001b[0;36mEmbedding.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/sparse.py?line=156'>157</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39minput\u001b[39m: Tensor) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m Tensor:\n\u001b[0;32m--> <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/sparse.py?line=157'>158</a>\u001b[0m \u001b[39mreturn\u001b[39;00m F\u001b[39m.\u001b[39;49membedding(\n\u001b[1;32m <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/sparse.py?line=158'>159</a>\u001b[0m \u001b[39minput\u001b[39;49m, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mweight, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mpadding_idx, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmax_norm,\n\u001b[1;32m <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/sparse.py?line=159'>160</a>\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mnorm_type, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mscale_grad_by_freq, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msparse)\n",
"File \u001b[0;32m~/.local/lib/python3.10/site-packages/torch/nn/functional.py:2183\u001b[0m, in \u001b[0;36membedding\u001b[0;34m(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)\u001b[0m\n\u001b[1;32m <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/functional.py?line=2176'>2177</a>\u001b[0m \u001b[39m# Note [embedding_renorm set_grad_enabled]\u001b[39;00m\n\u001b[1;32m <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/functional.py?line=2177'>2178</a>\u001b[0m \u001b[39m# XXX: equivalent to\u001b[39;00m\n\u001b[1;32m <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/functional.py?line=2178'>2179</a>\u001b[0m \u001b[39m# with torch.no_grad():\u001b[39;00m\n\u001b[1;32m <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/functional.py?line=2179'>2180</a>\u001b[0m \u001b[39m# torch.embedding_renorm_\u001b[39;00m\n\u001b[1;32m <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/functional.py?line=2180'>2181</a>\u001b[0m \u001b[39m# remove once script supports set_grad_enabled\u001b[39;00m\n\u001b[1;32m <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/functional.py?line=2181'>2182</a>\u001b[0m _no_grad_embedding_renorm_(weight, \u001b[39minput\u001b[39m, max_norm, norm_type)\n\u001b[0;32m-> <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/functional.py?line=2182'>2183</a>\u001b[0m \u001b[39mreturn\u001b[39;00m torch\u001b[39m.\u001b[39;49membedding(weight, \u001b[39minput\u001b[39;49m, padding_idx, scale_grad_by_freq, sparse)\n",
"\u001b[0;31mRuntimeError\u001b[0m: CUDA error: no kernel image is available for execution on the device\nCUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1."
]
}
],
"source": [
"device = 'cuda'\n",
"model = SimpleNgramNeuralLanguageModel(vocab_size, embed_size, context_size).to(device)\n",
"data = DataLoader(train_dataset, batch_size=1)\n",
"optimizer = torch.optim.Adam(model.parameters())\n",
"criterion = torch.nn.NLLLoss()\n",
" \n",
"model.train()\n",
"step = 0\n",
"for x, y in data:\n",
" print(x)\n",
" print(y)\n",
" x = x.to(device)\n",
" y = y.to(device)\n",
" optimizer.zero_grad()\n",
" ypredicted = model(x)\n",
" print(ypredicted)\n",
" loss = criterion(torch.log(ypredicted), y)\n",
" if step % 100 == 0:\n",
" print(step, loss)\n",
" step += 1\n",
" loss.backward()\n",
" optimizer.step()\n",
" break\n",
" \n",
"# torch.save(model.state_dict(), 'model1.bin')\n",
"\n",
"#Policzmy najbardziej prawdopodobne kontynuację dla zadanego słowa:\n",
"\n"
]
}
],
"metadata": {
"interpreter": {
"hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a"
},
"kernelspec": {
"display_name": "Python 3.10.2 64-bit",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.2"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}