final nn

run.py

@@ -19,10 +19,6 @@ import regex as re
 import csv
 import itertools
 from os.path import exists
-from nltk import word_tokenize
-
-import nltk
-nltk.download('punkt')
 
 vocab_size = 30000
 embed_size = 150
@@ -35,15 +31,20 @@ def clean(text):
     text = text.replace("'", "")
     return text
 
-def get_words_from_line(line):
+def get_words_from_line(line, specials = True):
-  return word_tokenize(line)
+    line = line.rstrip()
+    if specials:
+      yield '<s>'
+    for m in re.finditer(r'[\p{L}0-9\*]+|\p{P}+', line):
+        yield m.group(0).lower()
+    if specials:
+      yield '</s>'
 
 
 def get_word_lines_from_data(d):
     for line in d:
         yield get_words_from_line(line)
 
-
 class Model(torch.nn.Module):
     def __init__(self, vocabulary_size, embedding_size):
         super(Model, self).__init__()
@@ -115,6 +116,7 @@ if(not exists('model1.bin')):
 
     torch.save(model.state_dict(), 'model1.bin')
 else:
+    print("Loading model1")
     model.load_state_dict(torch.load('model1.bin'))
 
 
@@ -123,35 +125,44 @@ vocab = train_dataset.vocab
 def predict(tokens):
     ixs = torch.tensor(vocab.forward(tokens)).to(device)
     out = model(ixs)
-    top = torch.topk(out[0], 10)
+    top = torch.topk(out[0], 8)
     top_indices = top.indices.tolist()
     top_probs = top.values.tolist()
     top_words = vocab.lookup_tokens(top_indices)
     result = ""
     for word, prob in list(zip(top_words, top_probs)):
                 result += f"{word}:{prob} "
-    result  += f':0.001'
+    # result  += f':0.01'
     return result
 
+from nltk import word_tokenize
 def predict_file(result_path, data):
     with open(result_path, "w+", encoding="UTF-8") as f:
         for row in data:
             result = {}
-            before = word_tokenize(clean(str(row)))[-1:]
+            before = None
+            for before in get_words_from_line(clean(str(row)), False):
+              pass
+            before = [before]
+            print(before)
             if(len(before) < 1):
-                result = "the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1"
+                result = "a:0.2 the:0.2 to:0.2 of:0.1 and:0.1 of:0.1 :0.1"
             else:
                 result = predict(before)
+            result = result.strip()
             f.write(result + "\n")
             print(result)
 
 
 dev_data = pd.read_csv("gdrive/MyDrive/dev-0/in.tsv.xz", sep='\t', header=None, quoting=csv.QUOTE_NONE)[6]
 dev_data = dev_data.apply(clean)
-predict_file("dev-0-out.tsv", dev_data)
+predict_file("gdrive/MyDrive/dev-0/out.tsv", dev_data)
 
 test_data = pd.read_csv("gdrive/MyDrive/test-A/in.tsv.xz", sep='\t', header=None, quoting=csv.QUOTE_NONE)[6]
 test_data = test_data.apply(clean)
-predict_file("test-A-out.tsv", test_data)
+predict_file("gdrive/MyDrive/test-A/out.tsv", test_data)
 
-!cp -r "model1.bin" "gdrive/MyDrive/model1.bin"
+!cp -r gdrive/MyDrive/dev-0 dev-0
+!./geval -t dev-0 --metric PerplexityHashed
+
+!rm -r dev-0