s470611/challenging-america-word-gap-prediction
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

kenml

Browse Source
This commit is contained in:
Bartosz Karwacki 2022-04-25 15:06:34 +02:00
parent 77852bcc1e
commit a7cd8979d3
3 changed files with 17946 additions and 17940 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21038
dev-0/out.tsv
View File

File diff suppressed because it is too large Load Diff

20
run2.py
View File

@ -4,9 +4,9 @@ import regex as re
import kenlm import kenlm
from english_words import english_words_alpha_set from english_words import english_words_alpha_set
from nltk import word_tokenize from nltk import word_tokenize
from math import log10
from pathlib import Path from pathlib import Path
import os import os
import numpy as np
KENLM_BUILD_PATH = Path("/home/bartek/Pulpit/challenging-america-word-gap-prediction/kenlm/build") KENLM_BUILD_PATH = Path("/home/bartek/Pulpit/challenging-america-word-gap-prediction/kenlm/build")
@ -28,7 +28,7 @@ def create_train_data():
error_bad_lines=False, error_bad_lines=False,
header=None, header=None,
quoting=csv.QUOTE_NONE, quoting=csv.QUOTE_NONE,
nrows=10000 nrows=50000
) )
train_labels = pd.read_csv( train_labels = pd.read_csv(
"train/expected.tsv", "train/expected.tsv",
@ -36,7 +36,7 @@ def create_train_data():
error_bad_lines=False, error_bad_lines=False,
header=None, header=None,
quoting=csv.QUOTE_NONE, quoting=csv.QUOTE_NONE,
nrows=10000 nrows=50000
) )
train_data = data[[6, 7]] train_data = data[[6, 7]]
@ -58,6 +58,10 @@ def train_model():
os.system('echo %s|sudo -S %s' % (SUDO_PASSWORD, build_binary_command)) os.system('echo %s|sudo -S %s' % (SUDO_PASSWORD, build_binary_command))
def softmax(x):
e_x = np.exp(x - np.max(x))
return e_x / e_x.sum(axis=0)
def predict(model, before, after): def predict(model, before, after):
prob = 0.0 prob = 0.0
best = [] best = []
@ -77,11 +81,13 @@ def predict(model, before, after):
if worst_score[1] < text_score: if worst_score[1] < text_score:
best.remove(worst_score) best.remove(worst_score)
best.append((word, text_score)) best.append((word, text_score))
probs = sorted(best, key=lambda tup: tup[1], reverse=True) words = [word[0] for word in best]
probs = [prob[1] for prob in best]
probs = softmax(probs)
bests = sorted(zip(words, probs), key=lambda x:x[1], reverse=True)
pred_str = '' pred_str = ''
for word, prob in probs: for word, prob in bests:
pred_str += f'{word}:{prob} ' pred_str += f'{word}:{prob} '
pred_str += f':{log10(0.99)}'
return pred_str return pred_str
def make_prediction(model, path, result_path): def make_prediction(model, path, result_path):
@ -99,6 +105,6 @@ def make_prediction(model, path, result_path):
if __name__ == "__main__": if __name__ == "__main__":
create_train_file() create_train_file()
train_model() train_model()
model = kenlm.Model('model.arpa') model = kenlm.Model('model.binary')
make_prediction(model, "dev-0/in.tsv.xz", "dev-0/out.tsv") make_prediction(model, "dev-0/in.tsv.xz", "dev-0/out.tsv")
make_prediction(model, "test-A/in.tsv.xz", "test-A/out.tsv") make_prediction(model, "test-A/in.tsv.xz", "test-A/out.tsv")

14828
test-A/out.tsv
View File

File diff suppressed because it is too large Load Diff