s434784
/
challenging-america-word-gap-prediction
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

s434784

This commit is contained in:
Maciej(Linux) 2022-04-11 00:56:48 +02:00
parent 58127c0cf0
commit b20466a23f
1 changed files with 18 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
run.py
View File

@ -1,11 +1,11 @@
from nltk import trigrams, word_tokenize
from nltk import tris, word_tokenize
import pandas as pd
import csv
import regex as re
from collections import Counter, defaultdict
train_set = pd.read_csv(
train = pd.read_csv(
'train/in.tsv.xz',
sep='\t',
on_bad_lines='skip',
@ -14,7 +14,7 @@ train_set = pd.read_csv(
nrows=50000)
train_labels = pd.read_csv(
labels = pd.read_csv(
'train/expected.tsv',
sep='\t',
on_bad_lines='skip',
@ -28,7 +28,7 @@ def data_preprocessing(text):
def predict(before, after):
prediction = dict(Counter(dict(trigram[before, after])).most_common(5))
prediction = dict(Counter(dict(tri[before, after])).most_common(5))
result = ''
prob = 0.0
for key, value in prediction.items():
@ -52,28 +52,28 @@ def make_prediction(file):
file_out.write(prediction + '\n')
train_set = train_set[[6, 7]]
train_set = pd.concat([train_set, train_labels], axis=1)
train_set['line'] = train_set[6] + train_set[0] + train_set[7]
train = train[[6, 7]]
train = pd.concat([train, labels], axis=1)
train['line'] = train[6] + train[0] + train[7]
trigram = defaultdict(lambda: defaultdict(lambda: 0))
tri = defaultdict(lambda: defaultdict(lambda: 0))
rows = train_set.iterrows()
rows_len = len(train_set)
rows = train.iterrows()
rows_len = len(train)
for index, (_, row) in enumerate(rows):
text = data_preprocessing(str(row['line']))
words = word_tokenize(text)
for word_1, word_2, word_3 in trigrams(words, pad_right=True, pad_left=True):
for word_1, word_2, word_3 in tris(words, pad_right=True, pad_left=True):
if word_1 and word_2 and word_3:
trigram[(word_1, word_3)][word_2] += 1
tri[(word_1, word_3)][word_2] += 1
model_len = len(trigram)
for index, words_1_3 in enumerate(trigram):
count = sum(trigram[words_1_3].values())
for word_2 in trigram[words_1_3]:
trigram[words_1_3][word_2] += 0.25
trigram[words_1_3][word_2] /= float(count + 0.25 + len(word_2))
model_len = len(tri)
for index, words_1_3 in enumerate(tri):
count = sum(tri[words_1_3].values())
for word_2 in tri[words_1_3]:
tri[words_1_3][word_2] += 0.25
tri[words_1_3][word_2] /= float(count + 0.25 + len(word_2))
make_prediction('test-A')