challenging-america-word-gap-prediction/run.ipynb

from nltk import trigrams, word_tokenize
import pandas as pd
import csv
import regex as re
from collections import Counter, defaultdict
import kenlm
from english_words import english_words_alpha_set
from math import log10

train_set = pd.read_csv(
    'train/in.tsv.xz',
    sep='\t',
    header=None,
    quoting=csv.QUOTE_NONE,
    nrows=35000)

train_labels = pd.read_csv(
    'train/expected.tsv',
    sep='\t',
    header=None,
    quoting=csv.QUOTE_NONE,
    nrows=35000)

data = pd.concat([train_set, train_labels], axis=1)

data = train_set[6] + train_set[0] + train_set[7]

def data_preprocessing(text):
    return re.sub(r'\p{P}', '', text.lower().replace('-\\\\n', '').replace('\\\\n', ' ').replace("'ll", " will").replace("-", "").replace("'ve", " have").replace("'s", " is"))

data = data.apply(data_preprocessing)
prediction = 'the:0.03 be:0.03 to:0.03 of:0.025 and:0.025 a:0.025 in:0.020 that:0.020 have:0.015 I:0.010 it:0.010 for:0.010 not:0.010 on:0.010 with:0.010 he:0.010 as:0.010 you:0.010 do:0.010 at:0.010 :0.77'

with open("train_file.txt", "w+") as f:
    for text in data:
        f.write(text + "\n")

KENLM_BUILD_PATH='../kenlm/build/bin/lmplz'

!$KENLM_BUILD_PATH -o 4 < train_file.txt > kenlm_model.arpa

=== 1/5 Counting and sorting n-grams ===
Reading /home/maciej/challenging-america-word-gap-prediction/train_file.txt
----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100
****************************************************************************************************
Unigram tokens 11040226 types 580506
=== 2/5 Calculating and sorting adjusted counts ===
Chain sizes: 1:6966072 2:4100520192 3:7688475136 4:12301560832
Statistics:
1 580506 D1=0.841976 D2=0.938008 D3+=1.10537
2 3583875 D1=0.83057 D2=1.0296 D3+=1.2275
3 7705610 D1=0.899462 D2=1.16366 D3+=1.32181
4 9865473 D1=0.942374 D2=1.27613 D3+=1.35073
Memory estimate for binary LM:
type     MB
probing 442 assuming -p 1.5
probing 508 assuming -r models -p 1.5
trie    216 without quantization
trie    126 assuming -q 8 -b 8 quantization 
trie    195 assuming -a 22 array pointer compression
trie    104 assuming -a 22 -q 8 -b 8 array pointer compression and quantization
=== 3/5 Calculating and sorting initial probabilities ===
Chain sizes: 1:6966072 2:57342000 3:154112200 4:236771352
----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100
####################################################################################################
=== 4/5 Calculating and writing order-interpolated probabilities ===
Chain sizes: 1:6966072 2:57342000 3:154112200 4:236771352
----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100
####################################################################################################
=== 5/5 Writing ARPA model ===
----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100
****************************************************************************************************
Name:lmplz	VmPeak:23697780 kB	VmRSS:21496 kB	RSSMax:4963084 kB	user:39.0693	sys:17.6943	CPU:56.7637	real:43.821

import os
print(os.getcwd())
model = kenlm.Model('kenlm_model.arpa')

/home/maciej/challenging-america-word-gap-prediction

Loading the LM will be faster if you build a binary file.
Reading /home/maciej/challenging-america-word-gap-prediction/kenlm_model.arpa
----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100
****************************************************************************************************

def predict(before, after):
    result = ''
    prob = 0.0
    best = []
    for word in english_words_alpha_set:
        text = ' '.join([before, word, after])
        text_score = model.score(text, bos=False, eos=False)
        if len(best) < 12:
            best.append((word, text_score))
        else:
            is_better = False
            worst_score = None
            for score in best:
                if not worst_score:
                    worst_score = score
                else:
                    if worst_score[1] > score[1]:
                        worst_score = score
            if worst_score[1] < text_score:
                best.remove(worst_score)
                best.append((word, text_score))
    probs = sorted(best, key=lambda tup: tup[1], reverse=True)
    pred_str = ''
    for word, prob in probs:
        pred_str += f'{word}:{prob} '
    pred_str += f':{log10(0.99)}'
    return pred_str

def make_prediction(path, result_path):
    data = pd.read_csv(path, sep='\t', header=None, quoting=csv.QUOTE_NONE)
    with open(result_path, 'w', encoding='utf-8') as file_out:
        for _, row in data.iterrows():
            before, after = word_tokenize(data_preprocessing(str(row[6]))), word_tokenize(data_preprocessing(str(row[7])))
            if len(before) < 2 or len(after) < 2:
                pred = prediction
            else:
                pred = predict(before[-1], after[0])
            file_out.write(pred + '\n')

make_prediction("dev-0/in.tsv.xz", "dev-0/out.tsv")

make_prediction("test-A/in.tsv.xz", "test-A/out.tsv")