challenging-america-word-gap-prediction/main.ipynb

import pandas as pd
import numpy as np
import csv
import re
from collections import Counter, defaultdict
import nltk
import math
from tqdm import tqdm

directory = "train/in.tsv.xz"
directory_expected = "train/expected.tsv"
directory_dev_0 = "dev-0/in.tsv.xz"
directory_test_A = "test-A/in.tsv.xz"

MODEL N-GRAM

class Model():
    
    def __init__(self, vocab_size=30_000, UNK_token= '<UNK>', n=3):
        if (n <= 1 or n % 2 == 0):
            raise "change N value !!!"
        self.n = n
        self.vocab_size = vocab_size
        self.UNK_token = UNK_token
    
    def train(self, corpus:list) -> None:
        if(self.n > 1):
            self.n_grams = list(nltk.ngrams(corpus, n=self.n))
        else:
            self.n_grams = corpus
        self.counter = Counter(self.n_grams)
        self.words_counter = Counter(corpus)
        self.all_quantities = Counter([gram[:math.floor(self.n/2)]+gram[math.ceil(self.n/2):] for gram in self.n_grams])

        self.all_grams = defaultdict(set)

        for gram in tqdm(self.n_grams):
            previous_words = tuple(gram[:math.floor(self.n/2)])
            next_words = tuple(gram[math.ceil(self.n/2):])
            word = gram[math.floor(self.n/2)]
            self.all_grams[(previous_words, next_words)].add(word)

    def get_conditional_prob_for_word(self, left_text: list, right_text: list, word: str) -> float:
        previous_words = tuple(left_text[-math.floor(self.n/2):])
        next_words = tuple(right_text[:math.floor(self.n/2)])
        quantity = self.counter[previous_words + tuple([word]) + next_words]
        all_quantity = self.all_quantities[previous_words + next_words]
        if (all_quantity <= 0):
            return 0
        return quantity/all_quantity
    
    def get_prob_for_text(self, text: list) -> float:
        prob = 1
        for gram in list(nltk.ngrams(text, self.n)):
            prob *= self.get_conditional_prob_for_word(gram[:math.floor(self.n/2)], gram[math.ceil(self.n/2):], gram[math.floor(self.n/2)])
        return prob
    
    def most_probable_words(self, left_text: list, right_text: list) -> str:
        previous_words = tuple(left_text[-math.floor(self.n/2):])
        next_words = tuple(right_text[:math.floor(self.n/2)])
        all_words = self.all_grams[(previous_words, next_words)]
        best_words = []
        for word in all_words:
            probability = self.get_conditional_prob_for_word(list(previous_words), list(next_words), word)
            best_words.append((word, probability))
        return sorted(best_words, key=(lambda l: l[1]), reverse=True)[:20]
    
    def generate_text(self, text_beggining:list, text_ending:list, greedy: bool) -> list:
        words = self.most_probable_words(text_beggining, text_ending)
        return words

DATASET

dataframeList = pd.read_csv(directory, sep='\t', header=None, names=['FileId', 'Year', 'LeftContext', 'RightContext'], quoting=csv.QUOTE_NONE, chunksize=10000)

expectedList = pd.read_csv(directory_expected, sep='\t', header=None, names=['Word'], quoting=csv.QUOTE_NONE, chunksize=10000)

DATASET = ""

for number, (dataframe, expected) in enumerate(zip(dataframeList, expectedList)):
    dataframe = dataframe.reset_index()
    dataframe = dataframe.replace(r'\\\\r|\\\\n|\n|\\\\t', ' ', regex=True)

    expected['Word'] = expected['Word'].apply(lambda x: str(x).strip())
    word = expected['Word']

    left_text = dataframe['LeftContext'].to_list()
    right_text = dataframe['RightContext'].to_list()
    word = expected['Word'].to_list()

    lines = zip(left_text, word, right_text)
    lines = list(map(lambda l: " ".join(l), lines))
    DATASET = DATASET + " ".join(lines)

FINAL_DATASET = re.split(r"\s+", DATASET)
print(FINAL_DATASET[:100])

['came', 'fiom', 'the', 'last', 'place', 'to', 'this', 'place,', 'and', 'this', 'place', 'is', 'Where', 'We', 'Were,', 'this', 'is', 'the', 'first', 'road', 'I', 'ever', 'was', 'on', 'where', 'you', 'can', 'ride', 'elsewhere', 'from', 'anywhere', 'and', 'be', 'nowhere.', 'He', 'says,', 'while', 'this', 'train', 'stops', 'every-', 'where,', 'it', 'never', 'stops', 'anywhere', 'un-', 'less', 'its', 'somewhere.', 'Well,', 'I', 'says,', "I'm", 'glad', 'to', 'hear', 'that,', 'but,', 'accord-', 'ing', 'to', 'your', 'figures,', 'I', 'left', 'myself', 'where', '1', 'was,', 'which', 'is', 'five', 'miles', 'near-', 'er', 'to', 'myself', 'than', 'I', 'was', 'when', 'we', 'were', 'where', 'we', 'are', 'now.', 'We', 'have', 'now', 'reached', 'Slidell.', "That's", 'a', 'fine', 'place.', 'The', 'people', 'down']

TRAIN

model_3gram = Model(n = 3)
model_3gram.train(FINAL_DATASET)

100%|██████████| 139475976/139475976 [04:39<00:00, 498903.89it/s]

model = model_3gram

PREDICTION

def convert_predictions(line):
    sum_predictions = np.sum([pred[1] for pred in line])
    result = ""
    all_pred = 0
    for word, pred in line:
        new_pred = math.floor(pred / sum_predictions * 100) / 100
        if(new_pred == 1.0):
            new_pred = 0.99
        all_pred = all_pred + new_pred
        result = result + word + ":" + str(new_pred) + " "
    if(round(all_pred, 2) < 1):
        result = result + ":" + str(round(1 - all_pred, 2))
    else:
        result = result + ":" + str(0.01)
    return result

# PREDICTION FOR DEV-0

dataframe = pd.read_csv(directory_dev_0, sep='\t', header=None, names=['FileId', 'Year', 'LeftContext', 'RightContext'], quoting=csv.QUOTE_NONE)
dataframe = dataframe.replace(r'\\\\r|\\\\n|\n|\\\\t', ' ', regex=True)

left_text = dataframe['LeftContext'].apply(lambda l: re.split(r"\s+", l)).to_list()
right_text = dataframe['RightContext'].apply(lambda l: re.split(r"\s+", l)).to_list()

lines = zip(left_text, right_text)
lines = list(map(lambda l: model.generate_text(l[0], l[1], False), tqdm(lines)))
print(lines[:100])

10519it [00:27, 385.35it/s]

[[], [('successors', 0.006017228274638966), ('passage', 0.005193818089688371), ('place,', 0.005067139599695972), ('growth', 0.004813782619711173), ('use,', 0.004117050924752977), ('head', 0.003737015454775779), ('functions,', 0.0034836584747909806), ('power', 0.0034836584747909806), ('place', 0.003356979984798581), ('own,', 0.0032936407398023817), ('own', 0.0032936407398023817), ('members', 0.0032936407398023817), ('work', 0.003230301494806182), ('principles', 0.0031669622498099823), ('strength', 0.003040283759817583), ('value', 0.003040283759817583), ('beauty', 0.0026602482898403852), ('business', 0.0025969090448441853), ('size', 0.0025969090448441853), ('history', 0.0025969090448441853)], [('a', 0.5), ('lha', 0.25), ('the', 0.25)], [], [], [('a', 0.32934131736526945), ('him', 0.0718562874251497), ('two', 0.0718562874251497), ('only', 0.029940119760479042), ('just', 0.029940119760479042), ('means', 0.023952095808383235), ('money', 0.017964071856287425), ('force', 0.017964071856287425), ('the', 0.017964071856287425), ('barely', 0.011976047904191617), ('earth', 0.011976047904191617), ('all', 0.011976047904191617), ('no', 0.011976047904191617), ('applicants', 0.011976047904191617), ('capital', 0.011976047904191617), ('in\xad', 0.011976047904191617), ('capacity', 0.005988023952095809), ('corncobs', 0.005988023952095809), ('water', 0.005988023952095809), ('stabling', 0.005988023952095809)], [], [], [('arc', 1.0)], [('as', 0.7678571428571429), ('that', 0.11904761904761904), ('ns', 0.020833333333333332), ('confident', 0.008928571428571428), ('sure,', 0.005952380952380952), ('that,', 0.005952380952380952), ('aa', 0.005952380952380952), ('sure', 0.005952380952380952), ('alike;', 0.002976190476190476), ('a3', 0.002976190476190476), ('.as', 0.002976190476190476), ('thst', 0.002976190476190476), ('"as', 0.002976190476190476), ('a', 0.002976190476190476), ('sbuah', 0.002976190476190476), ('bad.', 0.002976190476190476), ('its', 0.002976190476190476), ('tbat', 0.002976190476190476), ('aggravated', 0.002976190476190476), ('defrauded', 0.002976190476190476)], [], [('the', 0.46712158808933), ('show', 0.25930521091811415), ('shew', 0.04404466501240695), ('this', 0.03163771712158809), ('tho', 0.020471464019851116), ('our', 0.018610421836228287), ('a', 0.013027295285359801), ('tbe', 0.009305210918114143), ('their', 0.008064516129032258), ('that', 0.008064516129032258), ('any', 0.00620347394540943), ('said', 0.004342431761786601), ('immediately', 0.004342431761786601), ('find', 0.003101736972704715), ('tlie', 0.003101736972704715), ('some', 0.003101736972704715), ('what', 0.0024813895781637717), ('give', 0.0024813895781637717), ('chow', 0.0018610421836228288), ('snow', 0.0018610421836228288)], [], [], [('to', 0.7446808510638298), ('a', 0.10638297872340426), ('and', 0.0425531914893617), ('for', 0.02127659574468085), ('world,', 0.02127659574468085), ('the', 0.02127659574468085), ('uud', 0.02127659574468085), ('¦', 0.02127659574468085)], [('There', 0.5), ('To', 0.5)], [('to', 0.5135135135135135), ('that', 0.1891891891891892), ('Almighty', 0.1891891891891892), ('for', 0.05405405405405406), ('thai', 0.02702702702702703), ('the', 0.02702702702702703)], [('as', 0.3048780487804878), ('posted', 0.059233449477351915), ('up', 0.050522648083623695), ('informed', 0.050522648083623695), ('started', 0.03832752613240418), ('known', 0.03832752613240418), ('down', 0.01916376306620209), ('fed', 0.017421602787456445), ('Informed', 0.017421602787456445), ('represented', 0.0156794425087108), ('out', 0.013937282229965157), ('back', 0.010452961672473868), ('along', 0.010452961672473868), ('and', 0.008710801393728223), ('established', 0.006968641114982578), ('that', 0.006968641114982578), ('put', 0.005226480836236934), ('over', 0.005226480836236934), ('placed', 0.005226480836236934), ('is', 0.005226480836236934)], [], [], [], [('will', 0.8), ('to', 0.2)], [('went', 0.032606199770378874), ('carried', 0.019402985074626865), ('with-', 0.01928817451205511), ('came', 0.01791044776119403), ('find', 0.01584385763490241), ('set', 0.015499425947187142), ('pointed', 0.014466130884041332), ('go', 0.01285878300803674), ('carry', 0.012399540757749712), ('started', 0.0121699196326062), ('come', 0.01182548794489093), ('brought', 0.010562571756601608), ('put', 0.009988518943742824), ('sent', 0.009299655568312285), ('paid', 0.009184845005740528), ('get', 0.009070034443168772), ('took', 0.008725602755453503), ('struck', 0.008495981630309988), ('got', 0.008266360505166475), ('worn', 0.007462686567164179)], [('city', 0.05304518664047151), ('City', 0.043222003929273084), ('Bay', 0.03339882121807466), ('avenue', 0.023575638506876228), ('and', 0.021611001964636542), ('city,', 0.01768172888015717), ('delegation', 0.01768172888015717), ('is', 0.015717092337917484), ('City,', 0.0137524557956778), ('State', 0.011787819253438114), ('Railroad', 0.009823182711198428), ('amounted', 0.007858546168958742), ('bound', 0.007858546168958742), ('banks', 0.007858546168958742), ('as', 0.007858546168958742), ('refused', 0.007858546168958742), ('or', 0.007858546168958742), ('were', 0.007858546168958742), ('seems', 0.007858546168958742), ('letter', 0.007858546168958742)], [], [('that', 0.05521709425872264), ('in', 0.0460150305086343), ('for', 0.038645840695455506), ('at', 0.03285980163498009), ('to', 0.027502532570028507), ('of', 0.02574504676420006), ('on', 0.02563196456757839), ('all', 0.02384620821259453), ('when', 0.01725445850118972), ('with', 0.016034113129314204), ('by', 0.015694866539449195), ('as', 0.01148726647348458), ('if', 0.01137889603505548), ('from', 0.011204560981930407), ('then', 0.009385822319598558), ('after', 0.007548236624496431), ('under', 0.0068885904775366925), ('In', 0.006596461469597381), ('upon', 0.0062336560887695245), ('during', 0.005296016208448182)], [], [], [], [], [], [], [], [], [('management,', 0.04), ('streets', 0.04), ('errands,', 0.04), ('derived,', 0.04), ('less,', 0.04), ('progressively,', 0.04), ('tomperanoe', 0.04), ('Mack', 0.04), ('child,-', 0.04), ('bland', 0.04), ('rived', 0.04), ('wuss', 0.04), ('saturday', 0.04), ('lode,', 0.04), ('rulers,', 0.04), ('rapped,', 0.04), ('file', 0.04), ('her', 0.04), ('tracted', 0.04), ('gans', 0.04)], [('for', 0.04954128440366973), ('about', 0.047706422018348627), ('twenty-', 0.03853211009174312), ('thirty-', 0.03486238532110092), ('eighty', 0.029357798165137616), ('the', 0.029357798165137616), ('twenty', 0.029357798165137616), ('forty', 0.025688073394495414), ('at', 0.023853211009174313), ('sixty', 0.023853211009174313), ('in', 0.023853211009174313), ('from', 0.022018348623853212), ('some', 0.022018348623853212), ('thirty', 0.02018348623853211), ('fifty', 0.01834862385321101), ('ninety', 0.01834862385321101), ('forty-', 0.01651376146788991), ('seventy', 0.014678899082568808), ('fifty-', 0.014678899082568808), ('that', 0.014678899082568808)], [], [], [('block', 1.0)], [('toward', 0.5), ('it,', 0.5)], [], [], [('of', 0.9597701149425287), ('or', 0.011494252873563218), ('ofecommon', 0.005747126436781609), ('of-', 0.005747126436781609), ('ot', 0.005747126436781609), ('ol', 0.005747126436781609), ('of.', 0.005747126436781609)], [], [], [], [], [('in', 0.21317365269461078), ('to', 0.1281437125748503), ('of', 0.081437125748503), ('on', 0.05269461077844311), ('In', 0.033532934131736525), ('for', 0.031137724550898204), ('if', 0.02874251497005988), ('among', 0.026347305389221556), ('by', 0.022754491017964073), ('with', 0.022754491017964073), ('at', 0.02155688622754491), ('upon', 0.019161676646706587), ('when', 0.017964071856287425), ('during', 0.016766467065868262), ('since', 0.015568862275449102), ('as', 0.01437125748502994), ('from', 0.013173652694610778), ('describe', 0.010778443113772455), ('about', 0.010778443113772455), ('against', 0.010778443113772455)], [], [('the', 0.6666666666666666), ('tlie', 0.3333333333333333)], [('trouble', 0.14285714285714285), ('man', 0.07142857142857142), ('mass', 0.07142857142857142), ('drawback', 0.07142857142857142), ('mission', 0.07142857142857142), ('change,', 0.07142857142857142), ('difficulty', 0.07142857142857142), ('ambition', 0.07142857142857142), ('object', 0.07142857142857142), ('gulf', 0.07142857142857142), ('art', 0.07142857142857142), ('impetus', 0.07142857142857142), ('mountain', 0.07142857142857142)], [('throat,', 0.5), ('yet', 0.5)], [('ralniali', 0.5), ('vocabulary', 0.5)], [('the', 0.8506616257088847), ('one', 0.02835538752362949), ('tho', 0.01890359168241966), ('a', 0.00945179584120983), ('said', 0.00945179584120983), ('Twenty-', 0.005671077504725898), ('tbe', 0.005671077504725898), ('that', 0.003780718336483932), ('its', 0.003780718336483932), ("'he", 0.003780718336483932), ('tbo', 0.003780718336483932), ('his', 0.003780718336483932), ('one-', 0.001890359168241966), ('^he', 0.001890359168241966), ("Beethoven's", 0.001890359168241966), ('Thirty', 0.001890359168241966), ('tLe', 0.001890359168241966), ('oue', 0.001890359168241966), ("Yieuxtemps'", 0.001890359168241966), ('Thirty-', 0.001890359168241966)], [('of', 0.8868894601542416), ('ol', 0.02313624678663239), ('ot', 0.015424164524421594), ('in', 0.010282776349614395), ('at', 0.007712082262210797), ('cf', 0.005141388174807198), ('<>l', 0.005141388174807198), ('on', 0.005141388174807198), ('by', 0.005141388174807198), ('as', 0.005141388174807198), ('or', 0.002570694087403599), ('»>f', 0.002570694087403599), ('o(', 0.002570694087403599), ('and', 0.002570694087403599), ('revives', 0.002570694087403599), ('ef', 0.002570694087403599), ('marked', 0.002570694087403599), ('meetings,', 0.002570694087403599), ('undor', 0.002570694087403599), ('warmed', 0.002570694087403599)], [('the', 1.0)], [], [], [], [('people', 0.008125546284190915), ('sum', 0.007939834891806292), ('city', 0.00787187528807134), ('purpose', 0.00757715938276531), ('office', 0.0074405673079316935), ('use', 0.006454143752729318), ('part', 0.00611232713196342), ('time', 0.0059750621897660915), ('State', 0.005960259107764419), ('amount', 0.005799443807837155), ('name', 0.005660160263548689), ('end', 0.005605658007087985), ('payment', 0.005049196697297832), ('hands', 0.004889054264734282), ('City', 0.004617215849794473), ('place', 0.004611832910884774), ('rate', 0.004344704567490951), ('number', 0.0043218270771247305), ('District', 0.004225607044113857), ('date', 0.0041125653270101745)], [('is', 0.12511220825852784), ('seems', 0.04798773189706763), ('was', 0.04095601436265709), ('necessary', 0.034634949132256135), ('seemed', 0.02779024536205865), ('ought', 0.026069718731298624), ('comes', 0.02378815080789946), ('came', 0.018925792938360265), ('impossible', 0.013502393776181927), ('possible', 0.012978755236385397), ('up', 0.012081089168162777), ('appears', 0.0109590065828845), ('occurred', 0.010210951526032316), ('began', 0.009799521244763614), ('appearing', 0.009724715739078396), ('has', 0.00953770197486535), ('difficult', 0.009088868940754039), ('over', 0.007443147815679234), ('had', 0.0071439257929383606), ('Is', 0.006807301017354877)], [], [('the', 0.6869041645760161), ('a', 0.06924234821876568), ('his', 0.02684395383843452), ('tho', 0.02609131961866533), ('this', 0.018815855494229806), ('each', 0.012543903662819869), ('either', 0.009784244856999498), ('tbe', 0.008529854490717512), ('that', 0.008028098344204716), ('her', 0.007526342197691922), ('my', 0.00577019568489714), ('said', 0.0055193176116407425), ('their', 0.004264927245358756), ('every', 0.003763171098845961), ('our', 0.003763171098845961), ('your', 0.0035122930255895636), ('any', 0.0030105368790767687), ('boarding', 0.0030105368790767687), ('clearing', 0.0027596588058203713), ('one', 0.002508780732563974)], [('be', 0.7804878048780488), ('bo', 0.07317073170731707), ('the', 0.04878048780487805), ('b', 0.024390243902439025), ('1', 0.024390243902439025), ('every', 0.024390243902439025), ('he', 0.024390243902439025)], [('his', 0.29056603773584905), ('her', 0.12075471698113208), ('one', 0.07169811320754717), ('a', 0.06792452830188679), ('little', 0.052830188679245285), ('their', 0.04150943396226415), ('Hud-', 0.022641509433962263), ('bis', 0.018867924528301886), ('wife,', 0.01509433962264151), ('the', 0.011320754716981131), ('my', 0.007547169811320755), ('loving', 0.007547169811320755), ('Jeffer-', 0.007547169811320755), ('adopted', 0.007547169811320755), ('promising', 0.007547169811320755), ('only', 0.007547169811320755), ('infant', 0.007547169811320755), ('Wil-', 0.007547169811320755), ('John-', 0.007547169811320755), ('UiC', 0.0037735849056603774)], [], [], [], [('little', 0.18), ('small', 0.11), ('summer', 0.07), ('neat', 0.06), ('pleasant', 0.04), ('fine', 0.03), ('low', 0.02), ('new', 0.02), ('nice', 0.02), ('modest', 0.02), ('tiny', 0.02), ('charming', 0.02), ('550,000', 0.01), ('rosewood', 0.01), ('frame', 0.01), ('tasteful', 0.01), ('lovely', 0.01), ('vine-covered', 0.01), ('eesy', 0.01), ('seaside', 0.01)], [('way', 0.049309057315524076), ('duty', 0.030832595557950652), ('power', 0.01801455021363409), ('efforts', 0.013549405288887179), ('return', 0.013164479002271065), ('attention', 0.012933523230301397), ('mind', 0.01239462642903884), ('right', 0.012317641171715616), ('ability', 0.01212517802840756), ('intention', 0.011432310712498556), ('wife', 0.0112398475691905), ('name', 0.01039300973863505), ('wife,', 0.009815620308710882), ('time', 0.008391393048231263), ('hand', 0.00796797413295354), ('desire', 0.007929481504291928), ('friends', 0.007814003618307095), ('family', 0.007621540474999038), ('life', 0.006890180530428423), ('visit', 0.006543746872473921)], [('husband', 0.02304036575828671), ('husband,', 0.013896408590507128), ('father', 0.013174517235156108), ('head', 0.01136978884677856), ('mother', 0.010828370330265295), ('hands', 0.009083799554833663), ('face', 0.006737652649942851), ('eyes', 0.006677495036996932), ('home', 0.006677495036996932), ('head,', 0.005895446068699994), ('life', 0.005835288455754076), ('hand', 0.005293869939240811), ('room', 0.005233712326294893), ('face,', 0.005053239487457137), ('arms', 0.004932924261565301), ('life,', 0.004692293809781628), ('eyes,', 0.004692293809781628), ('father,', 0.004632136196835709), ('up', 0.0045719785838897915), ('children', 0.004511820970943873)], [('amount', 0.016734911874666192), ('person', 0.01548869503293573), ('land', 0.00845647142602813), ('property', 0.0072992700729927005), ('more', 0.006320099697347339), ('money', 0.006142068719957272), ('ground', 0.005251913833006943), ('world', 0.005073882855616878), ('law', 0.004984867366921844), ('people', 0.0048068363895317785), ('time', 0.004717820900836746), ('lands', 0.0043617589460566136), ('country', 0.004272743457361581), ('work', 0.004272743457361581), ('party', 0.004272743457361581), ('country,', 0.004272743457361581), ('debt', 0.004183727968666548), ('other,', 0.004005696991276482), ('bonds', 0.0039166815025814495), ('persons', 0.0038276660138864163)], [], [('as', 0.075), ('time', 0.075), ('child,', 0.05), ('is', 0.05), ('concern,', 0.025), ('lady,', 0.025), ('value;', 0.025), ('girl,', 0.025), ('gossip', 0.025), ('space,', 0.025), ('spoil,', 0.025), ('elevated,', 0.025), ('early', 0.025), ('account,', 0.025), ('cousequence;', 0.025), ('ones,', 0.025), ('money,', 0.025), ('speed,', 0.025), ('and', 0.025), ('has', 0.025)], [], [], [], [('their', 0.5), ('tbe', 0.25), ('the', 0.25)], [('owner', 0.011226884445696047), ('person', 0.008054069276260208), ('city', 0.006721111424012016), ('same', 0.00598892330798836), ('same,', 0.004806157889796302), ('whole', 0.00463719140148315), ('State', 0.004543321130198066), ('purchaser', 0.004149065990800714), ('state', 0.004130291936543696), ('county', 0.0038674551769454614), ('time', 0.003717262742889327), ('party', 0.003304233549234957), ('day', 0.00322913733220689), ('military', 0.00322913733220689), ('man', 0.003135267060921806), ('town', 0.003116493006664789), ('right', 0.0030226227353797055), ('people', 0.0029099784098376045), ('State,', 0.0026095935417253355), ('point', 0.0024406270534121843)], [], [('people', 0.008125546284190915), ('sum', 0.007939834891806292), ('city', 0.00787187528807134), ('purpose', 0.00757715938276531), ('office', 0.0074405673079316935), ('use', 0.006454143752729318), ('part', 0.00611232713196342), ('time', 0.0059750621897660915), ('State', 0.005960259107764419), ('amount', 0.005799443807837155), ('name', 0.005660160263548689), ('end', 0.005605658007087985), ('payment', 0.005049196697297832), ('hands', 0.004889054264734282), ('City', 0.004617215849794473), ('place', 0.004611832910884774), ('rate', 0.004344704567490951), ('number', 0.0043218270771247305), ('District', 0.004225607044113857), ('date', 0.0041125653270101745)], [('the', 0.8982647635040583), ('tho', 0.02966694654352085), ('tbe', 0.014553596417576267), ('that', 0.00825636719843269), ('this', 0.006297229219143577), ('tne', 0.003498460677301987), ('tha', 0.002239014833473272), ('thc', 0.0019591379792891126), ('tlie', 0.0018191995521970334), ('these', 0.0018191995521970334), ('ihe', 0.0012594458438287153), ('tiie', 0.0012594458438287153), ('the.', 0.0008396305625524769), ('those', 0.0008396305625524769), ('th', 0.0008396305625524769), ('tlio', 0.0006996921354603975), ('Hie', 0.0006996921354603975), ('tire', 0.000559753708368318), ('tue', 0.000559753708368318), ('Ihe', 0.000559753708368318)], [], [], [('it.', 0.06103286384976526), ('this.', 0.04460093896713615), ('manner.', 0.028169014084507043), ('that.', 0.028169014084507043), ('this:', 0.023474178403755867), ('mad.', 0.014084507042253521), ('him.', 0.011737089201877934), ('results.', 0.011737089201877934), ('metal.', 0.011737089201877934), ('them.', 0.011737089201877934), ('circumstances.', 0.009389671361502348), ('thunder.', 0.009389671361502348), ('me.', 0.009389671361502348), ('admiration.', 0.007042253521126761), ('silver.', 0.007042253521126761), ('nature.', 0.007042253521126761), ('children.', 0.007042253521126761), ('appearance.', 0.007042253521126761), ('It.', 0.007042253521126761), ('satin.', 0.007042253521126761)], [('and', 1.0)], [], [], [], [], [('we', 1.0)], [('of', 0.8636363636363636), ('»f', 0.045454545454545456), ('af', 0.045454545454545456), ('01', 0.045454545454545456)], [('not', 0.10869565217391304), ('made', 0.043478260869565216), ('sufficient,', 0.0391304347826087), ('pleasant,', 0.03695652173913044), ('almost', 0.02826086956521739), ('left', 0.02608695652173913), ('never', 0.021739130434782608), ('still', 0.01956521739130435), ('entirely', 0.017391304347826087), ('now', 0.015217391304347827), ('dispersed', 0.015217391304347827), ('taken', 0.013043478260869565), ('yet', 0.010869565217391304), ('better', 0.010869565217391304), ('wholly', 0.010869565217391304), ('absolutely', 0.010869565217391304), ('people', 0.008695652173913044), ('taxed', 0.008695652173913044), ('grown', 0.008695652173913044), ('utterly', 0.008695652173913044)], [('and', 0.35735735735735735), ('or', 0.03903903903903904), ('made', 0.036036036036036036), ('procured,', 0.02702702702702703), ('killed', 0.012012012012012012), ('printed', 0.009009009009009009), ('found', 0.009009009009009009), ('issued', 0.009009009009009009), ('made,', 0.009009009009009009), ('writing', 0.006006006006006006), ('established', 0.006006006006006006), ('awarded', 0.006006006006006006), ('preserved', 0.006006006006006006), ('tried', 0.006006006006006006), ('exposed', 0.006006006006006006), ('nnd', 0.006006006006006006), ('received', 0.006006006006006006), ('formed,', 0.006006006006006006), ('ami', 0.006006006006006006), ('watching', 0.006006006006006006)], [], [('the', 0.4084507042253521), ('a', 0.36619718309859156), ('it', 0.08450704225352113), ('them', 0.028169014084507043), ('tho', 0.014084507042253521), ('necessary', 0.014084507042253521), ('me', 0.014084507042253521), ('i', 0.014084507042253521), ('her', 0.014084507042253521), ('sleeping', 0.014084507042253521), ('him', 0.014084507042253521), ('al', 0.014084507042253521)], [], [('the', 0.6842105263157895), ('our', 0.10526315789473684), ('for', 0.05263157894736842), ('tbe', 0.05263157894736842), ('offthe', 0.05263157894736842), ('(ho', 0.05263157894736842)], [], [('connected', 0.8611111111111112), ('associated', 0.05555555555555555), ('met', 0.027777777777777776), ('connecte«!', 0.027777777777777776), ('connecte', 0.027777777777777776)]]

with open("dev-0/out.tsv", "w", encoding="UTF-8") as file:
    result = "\n".join(list(map(lambda l: convert_predictions(l), tqdm(lines))))
    file.write(result)
    file.close()

100%|██████████| 10519/10519 [00:00<00:00, 106254.34it/s]

# PREDICTION FOR TEST-A

dataframe = pd.read_csv(directory_test_A, sep='\t', header=None, names=['FileId', 'Year', 'LeftContext', 'RightContext'], quoting=csv.QUOTE_NONE)
dataframe = dataframe.replace(r'\\\\r|\\\\n|\n|\\\\t', ' ', regex=True)

left_text = dataframe['LeftContext'].apply(lambda l: re.split(r"\s+", l)).to_list()
right_text = dataframe['RightContext'].apply(lambda l: re.split(r"\s+", l)).to_list()

lines = zip(left_text, right_text)
lines = list(map(lambda l: model.generate_text(l[0], l[1], False), tqdm(lines)))
print(lines[:100])

7414it [00:16, 457.43it/s]

[[], [], [('the', 0.9), ('tho', 0.1)], [('man', 0.02725228204788993), ('plan', 0.012567799973541474), ('trial', 0.010715703135335361), ('living', 0.009921947347532743), ('statement', 0.009525069453631433), ('law', 0.008334435771927504), ('class', 0.008202143140627068), ('time', 0.007937557878026195), ('government', 0.005953168408519645), ('bill', 0.0054239978833179), ('year', 0.0054239978833179), ('question', 0.005291705252017462), ('sensation', 0.005291705252017462), ('day', 0.005159412620717026), ('corporation,', 0.005159412620717026), ('little', 0.0050271199894165895), ('vote', 0.004894827358116153), ('single', 0.004762534726815717), ('means', 0.00423336420161397), ('speech', 0.004101071570313534)], [], [('to', 0.16666666666666666), ('here', 0.16666666666666666), ('youngsters,', 0.08333333333333333), ('vines', 0.08333333333333333), ('material', 0.08333333333333333), ('plaster,', 0.08333333333333333), ('fabrics', 0.08333333333333333), ('mist', 0.08333333333333333), ('arms,', 0.08333333333333333), ('close,', 0.08333333333333333)], [('tho', 1.0)], [], [('employed', 0.045454545454545456), ('alarmed', 0.045454545454545456), ('burned', 0.045454545454545456), ('put', 0.045454545454545456), ('above', 0.045454545454545456), ('early', 0.045454545454545456), ('visible', 0.045454545454545456), ('hanging', 0.045454545454545456), ('allowed', 0.045454545454545456), ('received', 0.045454545454545456), ('seated', 0.045454545454545456), ('outlined', 0.045454545454545456), ('drunk', 0.045454545454545456), ('typewritten', 0.045454545454545456), ('conferred', 0.045454545454545456), ('landed', 0.045454545454545456), ('gathered', 0.045454545454545456), ('poured', 0.045454545454545456), ('living', 0.045454545454545456), ('apartments', 0.045454545454545456)], [], [('since', 0.15228966986155484), ('in', 0.042243521476748314), ('had', 0.03123890663826766), ('on', 0.029463968761093362), ('been', 0.028399006034788784), ('saw', 0.012069577564785232), ('seen', 0.011714589989350373), ('to', 0.011714589989350373), ('be', 0.011004614838480652), ('that', 0.008874689385871494), ('see', 0.008874689385871494), ('heard', 0.008519701810436636), ('of', 0.007809726659566915), ('In', 0.007454739084132056), ('forget', 0.0067447639332623354), ('before', 0.006389776357827476), ('reached', 0.006389776357827476), ('visited', 0.006389776357827476), ('at', 0.006034788782392616), ('for', 0.006034788782392616)], [], [('days', 0.122), ('minutes', 0.072), ('years', 0.07), ('months', 0.05), ('weeks', 0.05), ('moments', 0.042), ('cents', 0.036), ('at', 0.03), ('dollars', 0.028), ('days,', 0.026), ('years,', 0.02), ('hours', 0.016), ('day', 0.012), ('that', 0.01), ('weeks,', 0.01), ('months,', 0.01), ('in', 0.01), ('furnish', 0.008), ('miles', 0.008), ('of', 0.008)], [], [('there', 1.0)], [('from', 1.0)], [('which', 0.45422535211267606), ('what', 0.05985915492957746), ('whom', 0.03873239436619718), ('until', 0.02992957746478873), ('as', 0.017605633802816902), ('them', 0.011443661971830986), ('if', 0.008802816901408451), ('till', 0.008802816901408451), ('and', 0.007922535211267605), ('when', 0.007922535211267605), ('this', 0.007042253521126761), ('them,', 0.006161971830985915), ('whether', 0.006161971830985915), ('that', 0.006161971830985915), ('board,', 0.00528169014084507), ('Monday', 0.00528169014084507), ('hand,', 0.00528169014084507), ('Saturday', 0.00528169014084507), ('hand', 0.00528169014084507), ('everything', 0.0044014084507042256)], [('this', 0.4745762711864407), ('any', 0.11016949152542373), ('every', 0.0903954802259887), ('the', 0.08757062146892655), ('that', 0.08192090395480225), ('some', 0.022598870056497175), ('a', 0.01977401129943503), ('one', 0.01977401129943503), ('what', 0.011299435028248588), ('no', 0.00847457627118644), ('each', 0.00847457627118644), ("th's", 0.005649717514124294), ('nny', 0.002824858757062147), ('thie', 0.002824858757062147), ('aay', 0.002824858757062147), ('all', 0.002824858757062147), ('ono', 0.002824858757062147), ('ever?', 0.002824858757062147), ('driving', 0.002824858757062147), ('tha', 0.002824858757062147)], [('little', 0.21714285714285714), ('whole', 0.05142857142857143), ('Indian', 0.02857142857142857), ('beautiful', 0.025714285714285714), ('nearest', 0.025714285714285714), ('said', 0.022857142857142857), ('entire', 0.022857142857142857), ('neighboring', 0.02), ('same', 0.02), ('old', 0.017142857142857144), ('next', 0.014285714285714285), ('quiet', 0.014285714285714285), ('small', 0.014285714285714285), ('first', 0.014285714285714285), ('present', 0.014285714285714285), ('town,', 0.011428571428571429), ('British', 0.011428571428571429), ('obscure', 0.008571428571428572), ('city,', 0.008571428571428572), ('agricultural', 0.008571428571428572)], [], [], [], [('inches', 0.20833333333333334), ('chains', 0.08333333333333333), ('feet', 0.075), ('poles', 0.041666666666666664), ('Inches', 0.029166666666666667), ('.00', 0.025), ('links', 0.025), ('years', 0.016666666666666666), ("o'clock", 0.016666666666666666), ('perches', 0.016666666666666666), ('miles', 0.0125), ('.', 0.0125), ('00', 0.0125), ('.50', 0.0125), ('in.)', 0.0125), ('rods', 0.0125), ('25', 0.008333333333333333), ('ft.', 0.008333333333333333), ('east,', 0.008333333333333333), ('1-4', 0.008333333333333333)], [], [], [], [], [('the', 0.75), ('silent', 0.25)], [('it', 0.23333333333333334), ('than', 0.2), ('that', 0.1), ('this', 0.1), ('there', 0.06666666666666667), ('cheerfulness', 0.03333333333333333), ('but', 0.03333333333333333), ('realized', 0.03333333333333333), ('what', 0.03333333333333333), ('she', 0.03333333333333333), ('who', 0.03333333333333333), ('It', 0.03333333333333333), ('he', 0.03333333333333333), ('t)an', 0.03333333333333333)], [('justice', 1.0)], [], [('going', 0.03335208550992526), ('placed', 0.029896327252270354), ('made', 0.027806799003455757), ('posted', 0.023547376034718317), ('put', 0.02250261191031102), ('found', 0.019930884834846903), ('held', 0.016716225990516757), ('laid', 0.015189263039459937), ('arrested', 0.013823033030619625), ('carried', 0.01350156714618661), ('not', 0.013340834203970104), ('standing', 0.01326046773286185), ('born', 0.013099734790645343), ('lying', 0.01301936831953709), ('taken', 0.012778268906212328), ('based', 0.012537169492887567), ('sitting', 0.011170939484047255), ('called', 0.010045808888531705), ('set', 0.00908141123523266), ('attached', 0.006348951217552037)], [], [], [], [('Final', 0.8484848484848485), ('Finir', 0.030303030303030304), ("People*'", 0.030303030303030304), ('International', 0.030303030303030304), ('Finsl', 0.030303030303030304), ('Jewish', 0.030303030303030304)], [('said', 0.07003515544220032), ('seen', 0.0625904735644861), ('remembered', 0.058868132625629004), ('hoped', 0.03577583235679327), ('found', 0.029847659750465293), ('expected', 0.02019714620528021), ('sure', 0.02005928172606328), ('denied', 0.014200041359343765), ('supposed', 0.013648583442476046), ('admitted', 0.013510718963259116), ('told', 0.013372854484042187), ('understood', 0.013028193285999861), ('shown', 0.012269938650306749), ('stated', 0.0115806162542221), ('true', 0.011511684014613634), ('forgotten', 0.01130488729578824), ('regretted', 0.011029158337354381), ('assured', 0.010546632660095126), ('observed', 0.009857310264010478), ('noted', 0.009788378024402012)], [], [], [('be', 0.06784660766961652), ('sell', 0.04129793510324484), ('look', 0.035398230088495575), ('work', 0.02359882005899705), ('appear', 0.017699115044247787), ('meet', 0.014749262536873156), ('get', 0.014749262536873156), ('bo', 0.014749262536873156), ('stop', 0.011799410029498525), ('go', 0.011799410029498525), ('remain', 0.008849557522123894), ('sit', 0.008849557522123894), ('him', 0.008849557522123894), ('stand', 0.008849557522123894), ('order', 0.008849557522123894), ('voto', 0.008849557522123894), ('redemption', 0.008849557522123894), ('call', 0.0058997050147492625), ('play', 0.0058997050147492625), ('It', 0.0058997050147492625)], [('shall', 0.3333333333333333), ('must', 0.3333333333333333), ('may', 0.3333333333333333)], [], [('of', 0.9357798165137615), ('or', 0.01834862385321101), ('ot', 0.01834862385321101), ('oi', 0.009174311926605505), ('f', 0.009174311926605505), ('as', 0.009174311926605505)], [], [('come,', 0.07692307692307693), ('deteriorate', 0.038461538461538464), ('die.', 0.038461538461538464), ('lost', 0.038461538461538464), ('wrecked', 0.038461538461538464), ('decay', 0.038461538461538464), ('rot', 0.038461538461538464), ('will', 0.038461538461538464), ('show,', 0.038461538461538464), ('end,', 0.038461538461538464), ('learn', 0.038461538461538464), ('come', 0.038461538461538464), ('as', 0.038461538461538464), ('evident', 0.038461538461538464), ('enough,', 0.038461538461538464), ('illed', 0.038461538461538464), ('see,', 0.038461538461538464), ('tires;', 0.038461538461538464), ('die', 0.038461538461538464), ('after-especially', 0.038461538461538464)], [('of', 0.9000720634158059), ('ot', 0.011450076066938906), ('in', 0.007286412042597485), ('to', 0.006245496036512131), ('ol', 0.005845143726479302), ('and', 0.005604932340459604), ('from', 0.004964368644407078), ('on', 0.003843382176315157), ('or', 0.003202818480262631), ('at', 0.0031227480182560653), ('for', 0.0030426775562494997), ('cf', 0.0020818320121707102), ('that', 0.0015213387781247499), ('oi', 0.001441268316118184), ('o', 0.001441268316118184), ('during', 0.0010409160060853551), ('by', 0.0010409160060853551), ('In', 0.0009608455440787893), ('into', 0.0008807750820722236), ('Of', 0.0008007046200656577)], [], [], [], [('half', 0.19625292740046837), ('for', 0.059484777517564404), ('in', 0.047775175644028105), ('such', 0.03231850117096019), ('like', 0.03044496487119438), ('twice', 0.023887587822014052), ('with', 0.022950819672131147), ('by', 0.019672131147540985), ('on', 0.019672131147540985), ('once', 0.01639344262295082), ('him', 0.015456674473067917), ('making', 0.0117096018735363), ('In', 0.009836065573770493), ('as', 0.009836065573770493), ('it', 0.008430913348946136), ('halt', 0.00702576112412178), ('which', 0.006088992974238876), ('through', 0.006088992974238876), ('$1.50', 0.006088992974238876), ('having', 0.005620608899297424)], [], [], [('that', 0.3333333333333333), ('the', 0.3333333333333333), ('ther', 0.3333333333333333)], [('to', 0.855072463768116), ('find', 0.028985507246376812), ('lectures,', 0.028985507246376812), ('school', 0.014492753623188406), ('ing', 0.014492753623188406), ('tothatafterawhilo;', 0.014492753623188406), ('and', 0.014492753623188406), ('as', 0.014492753623188406), ('to?', 0.014492753623188406)], [], [('is', 0.10810810810810811), ('to', 0.06486486486486487), ('shows', 0.06486486486486487), ('was', 0.05945945945945946), ('showing', 0.043243243243243246), ('made', 0.032432432432432434), ('says', 0.032432432432432434), ('said', 0.02702702702702703), ('of', 0.02702702702702703), ('saying', 0.021621621621621623), ('Is', 0.021621621621621623), ('and', 0.016216216216216217), ('by', 0.016216216216216217), ('showed', 0.016216216216216217), ('published', 0.016216216216216217), ('so', 0.010810810810810811), ('as', 0.010810810810810811), ('declares', 0.010810810810810811), ('or', 0.010810810810810811), ('that', 0.010810810810810811)], [], [], [], [('unerjoycd', 0.125), ("'JO,", 0.125), ('73.', 0.125), ('£Sl', 0.125), ("17.'t,", 0.125), (':iroc,', 0.125), ('338,', 0.125), ('broken', 0.125)], [], [], [], [], [('is', 0.6226415094339622), ('was', 0.16981132075471697), ('hates', 0.03773584905660377), ('all', 0.018867924528301886), ('Is', 0.018867924528301886), ('ia', 0.012578616352201259), ('i', 0.012578616352201259), ('of', 0.006289308176100629), ('bates', 0.006289308176100629), ('places', 0.006289308176100629), ('wholly', 0.006289308176100629), ('id', 0.006289308176100629), ('altogether', 0.006289308176100629), ('i-', 0.006289308176100629), ('seems', 0.006289308176100629), ('hales', 0.006289308176100629), ('iiates', 0.006289308176100629), ('w«g', 0.006289308176100629), ('almost', 0.006289308176100629), ('for', 0.006289308176100629)], [('of', 0.8779342723004695), ('by', 0.04225352112676056), ('ol', 0.014084507042253521), ('half', 0.009389671361502348), ('in', 0.009389671361502348), ('hostess,', 0.004694835680751174), ('upon', 0.004694835680751174), ('resembled', 0.004694835680751174), ('cf', 0.004694835680751174), ('only', 0.004694835680751174), (">1'", 0.004694835680751174), ('ot', 0.004694835680751174), ('with', 0.004694835680751174), ('as', 0.004694835680751174), ('on', 0.004694835680751174)], [('is', 0.15842770034843207), ('was', 0.1253266550522648), ('up', 0.026023519163763068), ('out', 0.015352787456445994), ('has', 0.014263937282229966), ('did', 0.014263937282229966), ('be', 0.013066202090592335), ('all', 0.011541811846689896), ('will', 0.011215156794425087), ('does', 0.009690766550522648), ('were', 0.00860191637630662), ('is,', 0.008275261324041812), ('appears', 0.00816637630662021), ('and', 0.007621951219512195), ('comes', 0.007513066202090592), ('away', 0.006315331010452962), ('down', 0.006315331010452962), ('Is', 0.005879790940766551), ('lies', 0.00544425087108014), ('would', 0.005117595818815331)], [('be', 0.6666666666666666), ('show', 0.13333333333333333), ('remain', 0.06666666666666667), ('tie', 0.06666666666666667), ('continue', 0.06666666666666667)], [], [], [('in', 0.06992230854605994), ('of', 0.06659267480577137), ('throughout', 0.051054384017758046), ('on', 0.03662597114317425), ('to', 0.03218645948945616), ('at', 0.02774694783573807), ('restoring', 0.02774694783573807), ('by', 0.022197558268590455), ('that', 0.021087680355160933), ('for', 0.017758046614872364), ('from', 0.017758046614872364), ('with', 0.016648168701442843), ('and', 0.014428412874583796), ('speaking,', 0.01220865704772475), ('In', 0.011098779134295227), ('known,', 0.009988901220865706), ('believed', 0.008879023307436182), ('through', 0.008879023307436182), ('over', 0.008879023307436182), ('considered', 0.00776914539400666)], [], [], [('»nd', 0.3333333333333333), ('and', 0.3333333333333333), ('nnd', 0.3333333333333333)], [('that', 0.1881807180314643), ('when', 0.11315046389673256), ('then', 0.058693021379588546), ('as', 0.05425574828559903), ('if', 0.043969342476805166), ('said', 0.029850746268656716), ('while', 0.017950786607503027), ('yet', 0.015328761597418314), ('there', 0.013916901976603469), ('now', 0.013110125050423558), ('which', 0.012505042355788625), ('so', 0.011294876966518758), ('where', 0.010689794271883823), ('how', 0.010286405808793869), ('what', 0.01008471157724889), ('after', 0.008874546187979023), ('though', 0.00826946349334409), ('before', 0.007260992335619202), ('although', 0.006252521177894313), ('says', 0.005647438483259379)], [], [], [('he', 0.35815602836879434), ('I', 0.1773049645390071), ('she', 0.07801418439716312), ('they', 0.04964539007092199), ('we', 0.040780141843971635), ('is', 0.030141843971631204), ('was', 0.026595744680851064), ('1', 0.026595744680851064), ('ho', 0.024822695035460994), ('lie', 0.015957446808510637), ('you', 0.010638297872340425), ('be', 0.010638297872340425), ('have', 0.0070921985815602835), ('were', 0.005319148936170213), ('one', 0.005319148936170213), ('time', 0.005319148936170213), ('evening', 0.0035460992907801418), ('had', 0.0035460992907801418), ('all', 0.0035460992907801418), ('a', 0.0035460992907801418)], [], [('and', 1.0)], [], [], [], [('great', 0.3333333333333333), ('gay', 0.3333333333333333), ('fanatic,', 0.3333333333333333)], [], [('would', 0.6666666666666666), ('they', 0.3333333333333333)], [], [], [], [], [], [], [], [], [], [], [], [('world,', 0.0048833423765599565), ('city', 0.004612045577862181), ('time', 0.004340748779164406), ('house', 0.004340748779164406), ('country', 0.004069451980466631), ('day', 0.0037981551817688553), ('city,', 0.00352685838307108), ('people,', 0.0032555615843733042), ('world', 0.002984264785675529), ('ground', 0.002984264785675529), ('people', 0.0027129679869777536), ('water', 0.0027129679869777536), ('highest', 0.0024416711882799783), ('time,', 0.0024416711882799783), ('building', 0.0024416711882799783), ('house,', 0.0024416711882799783), ('ground,', 0.0024416711882799783), ('State', 0.002170374389582203), ('man', 0.002170374389582203), ('door', 0.002170374389582203)], [('to', 0.1543071161048689), ('of', 0.15222638368705785), ('in', 0.07890137328339576), ('on', 0.056679151061173536), ('for', 0.04186433624635872), ('that', 0.039700374531835204), ('at', 0.039034540158135664), ('from', 0.03603828547648772), ('with', 0.025551394090719933), ('up', 0.015064502704952144), ('upon', 0.014565126924677487), ('over', 0.013816063254265501), ('into', 0.013316687473990845), ('and', 0.012900540990428632), ('by', 0.012068248023304202), ('through', 0.011652101539741989), ('In', 0.011319184352892218), ('all', 0.010486891385767791), ('out', 0.010320432792342904), ('down', 0.00807324178110695)], []]

with open("test-A/out.tsv", "w", encoding="UTF-8") as file:
    result = "\n".join(list(map(lambda l: convert_predictions(l), tqdm(lines))))
    file.write(result)
    file.close()

100%|██████████| 7414/7414 [00:00<00:00, 112933.06it/s]