nn trigram

nn.py

@@ -0,0 +1,40 @@
+import torch
+from utils import get_word_lines_from_data
+from torchtext.vocab import build_vocab_from_iterator
+import itertools
+
+
+class Trigrams(torch.utils.data.IterableDataset):
+    def __init__(self, data, vocabulary_size):
+        self.vocab = build_vocab_from_iterator(
+           get_word_lines_from_data(data),
+           max_tokens = vocabulary_size,
+           specials = ['<unk>'])
+        self.vocab.set_default_index(self.vocab['<unk>'])
+        self.vocabulary_size = vocabulary_size
+        self.data = data
+
+    @staticmethod
+    def look_ahead_iterator(gen):
+        w1 = None
+        for item in gen:
+            if w1 is not None:
+                yield (w1, item)
+            w1 = item
+
+    def __iter__(self):
+       return self.look_ahead_iterator(
+           (self.vocab[t] for t in itertools.chain.from_iterable(get_word_lines_from_data(self.data))))
+
+class Model(torch.nn.Module):
+    def __init__(self, vocabulary_size, embedding_size):
+        super(Model, self).__init__()
+        self.model = torch.nn.Sequential(
+            torch.nn.Embedding(vocabulary_size, embedding_size),
+            torch.nn.Linear(embedding_size, vocabulary_size),
+            torch.nn.Linear(embedding_size, vocabulary_size),
+            torch.nn.Softmax()
+        )
+
+    def forward(self, x):
+        return self.model(x)

tri_nn.py

@@ -0,0 +1,97 @@
+import torch
+import csv
+torch.cuda.empty_cache()
+from torch.utils.data import DataLoader
+import pandas as pd
+from os.path import exists
+
+from utils import read_csv, clean_text, get_words_from_line
+from nn import Trigrams, Model
+
+data = read_csv("train/in.tsv.xz")
+train_words = read_csv("train/expected.tsv")
+
+train_data = data[[6, 7]]
+train_data = pd.concat([train_data, train_words], axis=1)
+train_data = train_data[6] + train_data[0] + train_data[7]
+train_data = train_data.apply(clean_text)
+
+vocab_size = 30000
+embed_size = 150
+
+train_dataset = Trigrams(train_data, vocab_size)
+
+##################################################################################
+
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+model = Model(vocab_size, embed_size).to(device)
+print(device)
+if(not exists('model1.bin')):
+    data = DataLoader(train_dataset, batch_size=8000)
+    optimizer = torch.optim.Adam(model.parameters())
+    criterion = torch.nn.NLLLoss()
+
+    model.train()
+    step = 0
+    for i in range(2):
+      print(f"EPOCH {i}=========================")
+      for x, y in data:
+          x = x.to(device)
+          y = y.to(device)
+          optimizer.zero_grad()
+          ypredicted = model(x)
+          loss = criterion(torch.log(ypredicted), y)
+          if step % 100 == 0:
+              print(step, loss)
+          step += 1
+          loss.backward()
+          optimizer.step()
+
+    torch.save(model.state_dict(), 'model1.bin')
+else:
+    print("Loading model1")
+    model.load_state_dict(torch.load('model1.bin'))
+
+###################################################################
+
+vocab = train_dataset.vocab
+
+def predict(tokens):
+    ixs = torch.tensor(vocab.forward(tokens)).to(device)
+    out = model(ixs)
+    top = torch.topk(out[0], 8)
+    top_indices = top.indices.tolist()
+    top_probs = top.values.tolist()
+    top_words = vocab.lookup_tokens(top_indices)
+    result = ""
+    for word, prob in list(zip(top_words, top_probs)):
+                result += f"{word}:{prob} "
+    # result  += f':0.01'
+    return result
+
+DEFAULT_PREDICTION = "a:0.2 the:0.2 to:0.2 of:0.1 and:0.1 of:0.1 :0.1"
+
+def predict_file(result_path, data):
+    with open(result_path, "w+", encoding="UTF-8") as f:
+        for row in data:
+            result = {}
+            before = None
+            for before in get_words_from_line(clean_text(str(row)), False):
+              pass
+            before = [before]
+            print(before)
+            if(len(before) < 1):
+                result = DEFAULT_PREDICTION
+            else:
+                result = predict(before)
+            result = result.strip()
+            f.write(result + "\n")
+            print(result)
+
+dev_data = pd.read_csv("dev-0/in.tsv.xz", sep='\t', header=None, quoting=csv.QUOTE_NONE)[6]
+dev_data = dev_data.apply(clean_text)
+predict_file("dev-0/out.tsv", dev_data)
+
+test_data = pd.read_csv("test-A/in.tsv.xz", sep='\t', header=None, quoting=csv.QUOTE_NONE)[6]
+test_data = test_data.apply(clean_text)
+predict_file("test-A/out.tsv", test_data)

utils.py

@@ -22,6 +22,7 @@ def clean_text(text):
     res = res.replace("’", "'")
     res = REM.sub("", res)
     res = REP.sub(" ", res)
+    res = res.replace("'t", " not")
     res = res.replace("'s", " is")
     res = res.replace("'ll", " will")
     res = res.replace("won't", "will not")
@@ -30,4 +31,17 @@ def clean_text(text):
     res = res.replace("'ve'", "have")
     return res.replace("'m", " am")
 
+def get_words_from_line(line, specials = True):
+    line = line.rstrip()
+    if specials:
+      yield '<s>'
+    for m in re.finditer(r'[\p{L}0-9\*]+|\p{P}+', line):
+        yield m.group(0).lower()
+    if specials:
+      yield '</s>'
+
+
+def get_word_lines_from_data(d):
+    for line in d:
+        yield get_words_from_line(line)