kenml

run2.py

@@ -4,9 +4,9 @@ import regex as re
 import kenlm
 from english_words import english_words_alpha_set
 from nltk import  word_tokenize
-from math import log10
 from pathlib import  Path
 import os
+import numpy as np
 
 
 KENLM_BUILD_PATH = Path("/home/bartek/Pulpit/challenging-america-word-gap-prediction/kenlm/build")
@@ -28,7 +28,7 @@ def create_train_data():
         error_bad_lines=False,
         header=None,
         quoting=csv.QUOTE_NONE,
-        nrows=10000
+        nrows=50000
     )
     train_labels = pd.read_csv(
         "train/expected.tsv",
@@ -36,7 +36,7 @@ def create_train_data():
         error_bad_lines=False,
         header=None,
         quoting=csv.QUOTE_NONE,
-        nrows=10000
+        nrows=50000
     )
 
     train_data = data[[6, 7]]
@@ -58,6 +58,10 @@ def train_model():
     os.system('echo %s|sudo -S %s' % (SUDO_PASSWORD, build_binary_command))
     
 
+def softmax(x):
+    e_x = np.exp(x - np.max(x))
+    return e_x / e_x.sum(axis=0)
+
 def predict(model, before, after):
     prob = 0.0
     best = []
@@ -77,11 +81,13 @@ def predict(model, before, after):
             if worst_score[1] < text_score:
                 best.remove(worst_score)
                 best.append((word, text_score))
-    probs = sorted(best, key=lambda tup: tup[1], reverse=True)
+    words = [word[0] for word in best]
+    probs = [prob[1] for prob in best]
+    probs = softmax(probs)
+    bests = sorted(zip(words, probs), key=lambda x:x[1], reverse=True)
     pred_str = ''
-    for word, prob in probs:
+    for word, prob in bests:
         pred_str += f'{word}:{prob} '
-    pred_str += f':{log10(0.99)}'
     return pred_str
 
 def make_prediction(model, path, result_path):
@@ -99,6 +105,6 @@ def make_prediction(model, path, result_path):
 if __name__ == "__main__":
     create_train_file()
     train_model()
-    model = kenlm.Model('model.arpa')
+    model = kenlm.Model('model.binary')
     make_prediction(model, "dev-0/in.tsv.xz", "dev-0/out.tsv")
     make_prediction(model, "test-A/in.tsv.xz", "test-A/out.tsv")