172 lines
3.9 KiB
Python
172 lines
3.9 KiB
Python
#!/usr/bin/env python
|
|
# coding: utf-8
|
|
|
|
# In[1]:
|
|
|
|
|
|
KENLM_BUILD_PATH='/home/haskell/kenlm/build'
|
|
|
|
|
|
# ### Preprocessing danych
|
|
|
|
# In[2]:
|
|
|
|
|
|
import pandas as pd
|
|
import csv
|
|
import regex as re
|
|
|
|
|
|
# In[3]:
|
|
|
|
|
|
def clean_text(text):
|
|
text = text.lower().replace('-\\n', '').replace('\\n', ' ')
|
|
text = re.sub(r'\p{P}', '', text)
|
|
|
|
return text
|
|
|
|
|
|
# In[4]:
|
|
|
|
|
|
train_data = pd.read_csv('train/in.tsv.xz', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
|
|
train_labels = pd.read_csv('train/expected.tsv', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
|
|
|
|
train_data = train_data[[6, 7]]
|
|
train_data = pd.concat([train_data, train_labels], axis=1)
|
|
|
|
train_data['text'] = train_data[6] + train_data[0] + train_data[7]
|
|
train_data = train_data[['text']]
|
|
|
|
with open('processed_train.txt', 'w') as file:
|
|
for _, row in train_data.iterrows():
|
|
text = clean_text(str(row['text']))
|
|
file.write(text + '\n')
|
|
|
|
|
|
# ### Model kenLM
|
|
|
|
# In[4]:
|
|
|
|
|
|
get_ipython().system('$KENLM_BUILD_PATH/bin/lmplz -o 5 --skip_symbols < processed_train.txt > model/model.arpa')
|
|
|
|
|
|
# In[5]:
|
|
|
|
|
|
get_ipython().system('$KENLM_BUILD_PATH/bin/build_binary model/model.arpa model/model.binary')
|
|
|
|
|
|
# In[6]:
|
|
|
|
|
|
get_ipython().system('rm processed_train.txt')
|
|
|
|
|
|
# In[7]:
|
|
|
|
|
|
get_ipython().system('rm model/model.arpa')
|
|
|
|
|
|
# ### Predykcje
|
|
|
|
# In[1]:
|
|
|
|
|
|
import kenlm
|
|
import csv
|
|
import pandas as pd
|
|
import regex as re
|
|
from math import log10
|
|
from nltk import word_tokenize
|
|
from english_words import english_words_alpha_set
|
|
|
|
|
|
# In[2]:
|
|
|
|
|
|
model = kenlm.Model('model/model.binary')
|
|
|
|
|
|
# In[3]:
|
|
|
|
|
|
def clean_text(text):
|
|
text = text.lower().replace('-\\n', '').replace('\\n', ' ')
|
|
text = re.sub(r'\p{P}', '', text)
|
|
|
|
return text
|
|
|
|
|
|
# In[4]:
|
|
|
|
|
|
def predict_probs(w1, w3):
|
|
best_scores = []
|
|
for word in english_words_alpha_set:
|
|
text = ' '.join([w1, word, w3])
|
|
text_score = model.score(text, bos=False, eos=False)
|
|
if len(best_scores) < 12:
|
|
best_scores.append((word, text_score))
|
|
else:
|
|
is_better = False
|
|
worst_score = None
|
|
for score in best_scores:
|
|
if not worst_score:
|
|
worst_score = score
|
|
else:
|
|
if worst_score[1] > score[1]:
|
|
worst_score = score
|
|
if worst_score[1] < text_score:
|
|
best_scores.remove(worst_score)
|
|
best_scores.append((word, text_score))
|
|
probs = sorted(best_scores, key=lambda tup: tup[1], reverse=True)
|
|
pred_str = ''
|
|
for word, prob in probs:
|
|
pred_str += f'{word}:{prob} '
|
|
pred_str += f':{log10(0.99)}'
|
|
return pred_str
|
|
|
|
|
|
# In[5]:
|
|
|
|
|
|
dev_data = pd.read_csv('dev-0/in.tsv.xz', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
|
|
test_data = pd.read_csv('test-A/in.tsv.xz', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
|
|
|
|
|
|
# In[7]:
|
|
|
|
|
|
with open('dev-0/out.tsv', 'w') as file:
|
|
for index, row in dev_data.iterrows():
|
|
left_text = clean_text(str(row[6]))
|
|
right_text = clean_text(str(row[7]))
|
|
left_words = word_tokenize(left_text)
|
|
right_words = word_tokenize(right_text)
|
|
if len(left_words) < 2 or len(right_words) < 2:
|
|
prediction = ':1.0'
|
|
else:
|
|
prediction = predict_probs(left_words[len(left_words) - 1], right_words[0])
|
|
file.write(prediction + '\n')
|
|
|
|
|
|
# In[8]:
|
|
|
|
|
|
with open('test-A/out.tsv', 'w') as file:
|
|
for index, row in test_data.iterrows():
|
|
left_text = clean_text(str(row[6]))
|
|
right_text = clean_text(str(row[7]))
|
|
left_words = word_tokenize(left_text)
|
|
right_words = word_tokenize(right_text)
|
|
if len(left_words) < 2 or len(right_words) < 2:
|
|
prediction = ':1.0'
|
|
else:
|
|
prediction = predict_probs(left_words[len(left_words) - 1], right_words[0])
|
|
file.write(prediction + '\n')
|
|
|