v3

run.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-"""Untitled0.ipynb
+"""run
 
 Automatically generated by Colaboratory.
 
@@ -19,6 +19,10 @@ import regex as re
 import csv
 import itertools
 from os.path import exists
+from nltk import word_tokenize
+
+import nltk
+nltk.download('punkt')
 
 vocab_size = 30000
 embed_size = 150
@@ -32,11 +36,7 @@ def clean(text):
     return text
 
 def get_words_from_line(line):
-    line = line.rstrip()
-    yield '<s>'
-    for m in re.finditer(r'[\p{L}0-9\*]+|\p{P}+', line):
-        yield m.group(0).lower()
-    yield '</s>'
+  return word_tokenize(line)
 
 
 def get_word_lines_from_data(d):
@@ -120,8 +120,6 @@ else:
 
 vocab = train_dataset.vocab
 
-import nltk
-nltk.download('punkt')
 def predict(tokens):
     ixs = torch.tensor(vocab.forward(tokens)).to(device)
     out = model(ixs)
@@ -132,17 +130,16 @@ def predict(tokens):
     result = ""
     for word, prob in list(zip(top_words, top_probs)):
                 result += f"{word}:{prob} "
-    result  += f':0.01'
+    result  += f':0.001'
     return result
 
-from nltk import word_tokenize
 def predict_file(result_path, data):
     with open(result_path, "w+", encoding="UTF-8") as f:
         for row in data:
             result = {}
             before = word_tokenize(clean(str(row)))[-1:]
             if(len(before) < 1):
-                result = "a:0.2 the:0.2 to:0.2 of:0.1 and:0.1 of:0.1 :0.1"
+                result = "the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1"
             else:
                 result = predict(before)
             f.write(result + "\n")
@@ -151,8 +148,10 @@ def predict_file(result_path, data):
 
 dev_data = pd.read_csv("gdrive/MyDrive/dev-0/in.tsv.xz", sep='\t', header=None, quoting=csv.QUOTE_NONE)[6]
 dev_data = dev_data.apply(clean)
-predict_file("dev-0/out.tsv", dev_data)
+predict_file("dev-0-out.tsv", dev_data)
 
 test_data = pd.read_csv("gdrive/MyDrive/test-A/in.tsv.xz", sep='\t', header=None, quoting=csv.QUOTE_NONE)[6]
 test_data = test_data.apply(clean)
-predict_file("test-A/out.tsv", test_data)
+predict_file("test-A-out.tsv", test_data)
+
+!cp -r "model1.bin" "gdrive/MyDrive/model1.bin"