challenging-america-word-gap-prediction-kenlm/run.ipynb

import pandas as pd
from utils import *

data = get_csv("train/in.tsv.xz")

train_labels = get_csv("train/expected.tsv")

train_data = data[[6,7]]

train_data = pd.concat([train_data, train_labels], axis=1)

train_data[607] = train_data[6] + train_data[0] + train_data[7]

train_data[607] = train_data[607].apply(clean_text)

train_data[607]

0         came fiom the last place to thisnplace and thi...
1         mb boot political obeednattempt to imagine a p...
2         thera were in   only aeventyninenuberlbers lo ...
3         a gixnl man y niterertiiiv diiclosurs regard  ...
4         tin  ub tv thf bbabbt qabjenmr schiffs tutemen...
                                ...                        
432017    sam clendenin bad a fancy for uinscience of me...
432018    witahtt halting the party ware dilven to the s...
432019    it was the last thing that either ofnthem expe...
432020    settlement with the departmentnit is also show...
432021    flour quotationslow extras at   r ®   ncity mi...
Name: 607, Length: 432022, dtype: object

with open("tmp",  "w+") as f:
    for t in train_data[607]:
        f.write(t + "\n")

KENLM_BUILD_PATH = "../kenlm/build/"
!$KENLM_BUILD_PATH/bin/lmplz -o 4 < tmp > model.arpa

=== 1/5 Counting and sorting n-grams ===
Reading /home/me/challenging-america-word-gap-prediction-kenlm/tmp
----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100
************************/home/me/kenlm/lm/builder/corpus_count.cc:179 in void lm::builder::{anonymous}::ComplainDisallowed(StringPiece, lm::WarningAction&) threw FormatLoadException.
Special word <s> is not allowed in the corpus.  I plan to support models containing <unk> in the future.  Pass --skip_symbols to convert these symbols to whitespace.
/bin/bash: linia 1:  5055 Przerwane               (zrzut pamięci) ../kenlm/build//bin/lmplz -o 4 < tmp > model.arpa

!rm tmp

import kenlm
model = kenlm.Model("./model.arpa")

Loading the LM will be faster if you build a binary file.
Reading /home/me/challenging-america-word-gap-prediction-kenlm/model.arpa
----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
File kenlm.pyx:139, in kenlm.Model.__init__()

RuntimeError: End of file Byte: 0

The above exception was the direct cause of the following exception:

OSError                                   Traceback (most recent call last)
Input In [14], in <cell line: 2>()
      1 import kenlm
----> 2 model = kenlm.Model("./model.arpa")

File kenlm.pyx:142, in kenlm.Model.__init__()

OSError: Cannot read model './model.arpa' (End of file Byte: 0)

def predict(before, after):
    result = ''
    prob = 0.0
    best = []
    for word in english_words_alpha_set:
        text = ' '.join([before, word, after])
        text_score = model.score(text, bos=False, eos=False)
        if len(best) < 12:
            best.append((word, text_score))
        else:
            is_better = False
            worst_score = None
            for score in best:
                if not worst_score:
                    worst_score = score
                else:
                    if worst_score[1] > score[1]:
                        worst_score = score
            if worst_score[1] < text_score:
                best.remove(worst_score)
                best.append((word, text_score))
    probs = sorted(best, key=lambda tup: tup[1], reverse=True)
    pred_str = ''
    for word, prob in probs:
        pred_str += f'{word}:{prob} '
    pred_str += f':{log10(0.99)}'
    return pred_str

def make_prediction(path, result_path):
    data = pd.read_csv(path, sep='\t', header=None, quoting=csv.QUOTE_NONE)
    with open(result_path, 'w', encoding='utf-8') as file_out:
        for _, row in data.iterrows():
            before, after = word_tokenize(data_preprocessing(str(row[6]))), word_tokenize(data_preprocessing(str(row[7])))
            if len(before) < 2 or len(after) < 2:
                pred = prediction
            else:
                pred = predict(before[-1], after[0])
            file_out.write(pred + '\n')

make_prediction("dev-0/in.tsv.xz", "dev-0/out.tsv")

make_prediction("test-A/in.tsv.xz", "test-A/out.tsv")