retro_gap/main.ipynb

from torchtext.vocab import build_vocab_from_iterator
from nltk.tokenize import word_tokenize


def get_words_from_line(line):
  line = line.split('\t')
  left_context = line[2]
  right_context = line[3]
  line = ' '.join([left_context, right_context])
  tokenized = word_tokenize(line)
  for word in tokenized:
     yield word.lower()

def get_context_from_line(line):
  line = line.split('\t')
  left_context = line[2]
  right_context = line[3]
  left_context = [cleanString(t) for t in word_tokenize(left_context)]
  right_context = [cleanString(t) for t in word_tokenize(right_context)]
  yield left_context[-2:] + right_context[:2]

def cleanString(value):
  value = value.strip('')
  value = value.strip('\n')
  value = value.strip('\t')
  value = value.lower()
  return value

def get_word_lines_from_file(file_name):
  with open(file_name, 'r') as fh:
    for line in fh:
       yield get_words_from_line(line)

vocab_size = 20000

vocab = build_vocab_from_iterator(
    get_word_lines_from_file('dev-0/in.tsv'),
    max_tokens = vocab_size,
    specials = ['<unk>'])

vocab.set_default_index(vocab['<unk>'])

from torch import nn
import torch

embed_size = 100
context_size = 4

class SimpleNgramNeuralLanguageModel(nn.Module):
  def __init__(self, vocabulary_size, embedding_size, context_size):
      super(SimpleNgramNeuralLanguageModel, self).__init__()
      self.embedding = nn.Embedding(vocabulary_size, embedding_size)
      self.fn1 = nn.Linear(embedding_size * context_size, vocabulary_size)
      self.fn2 = nn.Linear(vocabulary_size, vocabulary_size)
      self.out = nn.Softmax(dim=0)

  def forward(self, x):
      x = torch.as_tensor([torch.cat((self.embedding(val[0]), self.embedding(val[1]), self.embedding(val[2]), self.embedding(val[3]))) for val in x])
      x = self.fn1(x)
      x = self.fn2(x)
      x = self.out(x)
      return x

model = SimpleNgramNeuralLanguageModel(vocab_size, embed_size, context_size)

from torch.utils.data import IterableDataset, DataLoader

class Ngrams(IterableDataset):
  def __init__(self, text_file, vocabulary_size, expected_file):
      self.vocab = build_vocab_from_iterator(
         get_word_lines_from_file(text_file),
         max_tokens = vocabulary_size,
         specials = ['<unk>'])
      self.vocab.set_default_index(self.vocab['<unk>'])
      self.vocabulary_size = vocabulary_size
      self.text_file = text_file
      self.expected_file = expected_file

  def __iter__(self):
     with open(self.text_file, 'r') as inData:
        with open(self.expected_file, 'r') as expData:
         for lineIn, lineExp in zip(inData, expData):
            yield torch.as_tensor([self.vocab[t] for chunk in get_context_from_line(lineIn) for t in chunk]), torch.as_tensor(self.vocab[cleanString(lineExp)])

train_dataset = Ngrams('dev-0/in.tsv', vocab_size, 'dev-0/expected.tsv')

device = 'cuda'
model = SimpleNgramNeuralLanguageModel(vocab_size, embed_size, context_size).to(device)
data = DataLoader(train_dataset, batch_size=1)
optimizer = torch.optim.Adam(model.parameters())
criterion = torch.nn.NLLLoss()
   
model.train()
step = 0
for x, y in data:
   print(x)
   print(y)
   x = x.to(device)
   y = y.to(device)
   optimizer.zero_grad()
   ypredicted = model(x)
   print(ypredicted)
   loss = criterion(torch.log(ypredicted), y)
   if step % 100 == 0:
      print(step, loss)
   step += 1
   loss.backward()
   optimizer.step()
   break
   
#  torch.save(model.state_dict(), 'model1.bin')

#Policzmy najbardziej prawdopodobne kontynuację dla zadanego słowa:

tensor([[239, 466, 232,   1]])
tensor([2])

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
/home/ramon/projects/retro-gap/main.ipynb Cell 4' in <cell line: 9>()
     <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000003?line=12'>13</a> y = y.to(device)
     <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000003?line=13'>14</a> optimizer.zero_grad()
---> <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000003?line=14'>15</a> ypredicted = model(x)
     <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000003?line=15'>16</a> print(ypredicted)
     <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000003?line=16'>17</a> loss = criterion(torch.log(ypredicted), y)

File ~/.local/lib/python3.10/site-packages/torch/nn/modules/module.py:1110, in Module._call_impl(self, *input, **kwargs)
   <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1105'>1106</a> # If we don't have any hooks, we want to skip the rest of the logic in
   <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1106'>1107</a> # this function, and just call forward.
   <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1107'>1108</a> if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1108'>1109</a>         or _global_forward_hooks or _global_forward_pre_hooks):
-> <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1109'>1110</a>     return forward_call(*input, **kwargs)
   <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1110'>1111</a> # Do not call functions when jit is used
   <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1111'>1112</a> full_backward_hooks, non_full_backward_hooks = [], []

/home/ramon/projects/retro-gap/main.ipynb Cell 2' in SimpleNgramNeuralLanguageModel.forward(self, x)
     <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000001?line=14'>15</a> def forward(self, x):
---> <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000001?line=15'>16</a>     x = torch.as_tensor([torch.cat((self.embedding(val[0]), self.embedding(val[1]), self.embedding(val[2]), self.embedding(val[3]))) for val in x])
     <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000001?line=16'>17</a>     x = self.fn1(x)
     <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000001?line=17'>18</a>     x = self.fn2(x)

/home/ramon/projects/retro-gap/main.ipynb Cell 2' in <listcomp>(.0)
     <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000001?line=14'>15</a> def forward(self, x):
---> <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000001?line=15'>16</a>     x = torch.as_tensor([torch.cat((self.embedding(val[0]), self.embedding(val[1]), self.embedding(val[2]), self.embedding(val[3]))) for val in x])
     <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000001?line=16'>17</a>     x = self.fn1(x)
     <a href='vscode-notebook-cell:/home/ramon/projects/retro-gap/main.ipynb#ch0000001?line=17'>18</a>     x = self.fn2(x)

File ~/.local/lib/python3.10/site-packages/torch/nn/modules/module.py:1110, in Module._call_impl(self, *input, **kwargs)
   <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1105'>1106</a> # If we don't have any hooks, we want to skip the rest of the logic in
   <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1106'>1107</a> # this function, and just call forward.
   <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1107'>1108</a> if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1108'>1109</a>         or _global_forward_hooks or _global_forward_pre_hooks):
-> <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1109'>1110</a>     return forward_call(*input, **kwargs)
   <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1110'>1111</a> # Do not call functions when jit is used
   <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/module.py?line=1111'>1112</a> full_backward_hooks, non_full_backward_hooks = [], []

File ~/.local/lib/python3.10/site-packages/torch/nn/modules/sparse.py:158, in Embedding.forward(self, input)
    <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/sparse.py?line=156'>157</a> def forward(self, input: Tensor) -> Tensor:
--> <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/sparse.py?line=157'>158</a>     return F.embedding(
    <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/sparse.py?line=158'>159</a>         input, self.weight, self.padding_idx, self.max_norm,
    <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/modules/sparse.py?line=159'>160</a>         self.norm_type, self.scale_grad_by_freq, self.sparse)

File ~/.local/lib/python3.10/site-packages/torch/nn/functional.py:2183, in embedding(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)
   <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/functional.py?line=2176'>2177</a>     # Note [embedding_renorm set_grad_enabled]
   <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/functional.py?line=2177'>2178</a>     # XXX: equivalent to
   <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/functional.py?line=2178'>2179</a>     # with torch.no_grad():
   <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/functional.py?line=2179'>2180</a>     #   torch.embedding_renorm_
   <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/functional.py?line=2180'>2181</a>     # remove once script supports set_grad_enabled
   <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/functional.py?line=2181'>2182</a>     _no_grad_embedding_renorm_(weight, input, max_norm, norm_type)
-> <a href='file:///home/ramon/.local/lib/python3.10/site-packages/torch/nn/functional.py?line=2182'>2183</a> return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)

RuntimeError: CUDA error: no kernel image is available for execution on the device
CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.