challenging-america-word-gap-prediction-kenlm/run.py

#!/usr/bin/env python
# coding: utf-8

# In[1]:


KENLM_BUILD_PATH='/home/haskell/kenlm/build'


# ### Preprocessing danych

# In[2]:


import pandas as pd
import csv
import regex as re


# In[3]:


def clean_text(text):
    text = text.lower().replace('-\\n', '').replace('\\n', ' ')
    text = re.sub(r'\p{P}', '', text)

    return text


# In[4]:


train_data = pd.read_csv('train/in.tsv.xz', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
train_labels = pd.read_csv('train/expected.tsv', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)

train_data = train_data[[6, 7]]
train_data = pd.concat([train_data, train_labels], axis=1)

train_data['text'] = train_data[6] + train_data[0] + train_data[7]
train_data = train_data[['text']]

with open('processed_train.txt', 'w') as file:
    for _, row in train_data.iterrows():
        text = clean_text(str(row['text']))
        file.write(text + '\n')


# ### Model kenLM

# In[4]:


get_ipython().system('$KENLM_BUILD_PATH/bin/lmplz -o 5 --skip_symbols < processed_train.txt > model/model.arpa')


# In[5]:


get_ipython().system('$KENLM_BUILD_PATH/bin/build_binary model/model.arpa model/model.binary')


# In[6]:


get_ipython().system('rm processed_train.txt')


# In[7]:


get_ipython().system('rm model/model.arpa')


# ### Predykcje

# In[1]:


import kenlm
import csv
import pandas as pd
import regex as re
from math import log10
from nltk import word_tokenize
from english_words import english_words_alpha_set


# In[2]:


model = kenlm.Model('model/model.binary')


# In[3]:


def clean_text(text):
    text = text.lower().replace('-\\n', '').replace('\\n', ' ')
    text = re.sub(r'\p{P}', '', text)

    return text


# In[4]:


def predict_probs(w1, w3):
    best_scores = []
    for word in english_words_alpha_set:
        text = ' '.join([w1, word, w3])
        text_score = model.score(text, bos=False, eos=False)
        if len(best_scores) < 12:
            best_scores.append((word, text_score))
        else:
            is_better = False
            worst_score = None
            for score in best_scores:
                if not worst_score:
                    worst_score = score
                else:
                    if worst_score[1] > score[1]:
                        worst_score = score
            if worst_score[1] < text_score:
                best_scores.remove(worst_score)
                best_scores.append((word, text_score))
    probs = sorted(best_scores, key=lambda tup: tup[1], reverse=True)
    pred_str = ''
    for word, prob in probs:
        pred_str += f'{word}:{prob} '
    pred_str += f':{log10(0.99)}'
    return pred_str


# In[5]:


dev_data = pd.read_csv('dev-0/in.tsv.xz', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
test_data = pd.read_csv('test-A/in.tsv.xz', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)


# In[7]:


with open('dev-0/out.tsv', 'w') as file:
    for index, row in dev_data.iterrows():
        left_text = clean_text(str(row[6]))
        right_text = clean_text(str(row[7]))
        left_words = word_tokenize(left_text)
        right_words = word_tokenize(right_text)
        if len(left_words) < 2 or len(right_words) < 2:
            prediction = ':1.0'
        else:
            prediction = predict_probs(left_words[len(left_words) - 1], right_words[0])
        file.write(prediction + '\n')


# In[8]:


with open('test-A/out.tsv', 'w') as file:
    for index, row in test_data.iterrows():
        left_text = clean_text(str(row[6]))
        right_text = clean_text(str(row[7]))
        left_words = word_tokenize(left_text)
        right_words = word_tokenize(right_text)
        if len(left_words) < 2 or len(right_words) < 2:
            prediction = ':1.0'
        else:
            prediction = predict_probs(left_words[len(left_words) - 1], right_words[0])
        file.write(prediction + '\n')