test 4 version
This commit is contained in:
parent
9d75188a0f
commit
fa9858ae9a
10519
dev-0/out.tsv
10519
dev-0/out.tsv
File diff suppressed because one or more lines are too long
21
model.py
21
model.py
@ -14,7 +14,10 @@ def preprocess(text):
|
|||||||
for character in _text:
|
for character in _text:
|
||||||
if character not in string.ascii_lowercase + ' ':
|
if character not in string.ascii_lowercase + ' ':
|
||||||
_text = _text.replace(character, '')
|
_text = _text.replace(character, '')
|
||||||
return word_tokenize(_text)
|
words = word_tokenize(_text)
|
||||||
|
if len(words):
|
||||||
|
return words
|
||||||
|
return ['']
|
||||||
|
|
||||||
|
|
||||||
def predict(word_before, word_after):
|
def predict(word_before, word_after):
|
||||||
@ -42,11 +45,14 @@ with open('train/in.tsv', encoding='utf-8') as file_in, open('train/expected.tsv
|
|||||||
before, expected, after = preprocess(before), preprocess(expected), preprocess(after)
|
before, expected, after = preprocess(before), preprocess(expected), preprocess(after)
|
||||||
words = before + expected + after
|
words = before + expected + after
|
||||||
trigrams_list += trigrams(words, pad_right=True, pad_left=True)
|
trigrams_list += trigrams(words, pad_right=True, pad_left=True)
|
||||||
|
length = len(trigrams_list)
|
||||||
|
print('zbieranie trigramów:', length)
|
||||||
|
if length > 1000000:
|
||||||
|
break
|
||||||
|
|
||||||
trigrams_len = len(trigrams_list)
|
|
||||||
for index, trigram in enumerate(trigrams_list):
|
for index, trigram in enumerate(trigrams_list):
|
||||||
if index % 100000 == 0:
|
if index % 100000 == 0:
|
||||||
print('uczenie modelu', index, '/', trigrams_len)
|
print('uczenie modelu', index)
|
||||||
if not trigram[0] or not trigram[1] or not trigram[2]:
|
if not trigram[0] or not trigram[1] or not trigram[2]:
|
||||||
continue
|
continue
|
||||||
model[(trigram[0], trigram[2])][trigram[1]] += 1
|
model[(trigram[0], trigram[2])][trigram[1]] += 1
|
||||||
@ -62,7 +68,14 @@ for index, words_1_3 in enumerate(model):
|
|||||||
model[words_1_3][word_2] /= float(count)
|
model[words_1_3][word_2] /= float(count)
|
||||||
|
|
||||||
with open('test-A/in.tsv', encoding='utf-8') as file_in, open('test-A/out.tsv', 'w', encoding='utf-8') as file_out:
|
with open('test-A/in.tsv', encoding='utf-8') as file_in, open('test-A/out.tsv', 'w', encoding='utf-8') as file_out:
|
||||||
print('zapisywanie wyników')
|
print('zapisywanie test-A')
|
||||||
|
for line_in in file_in:
|
||||||
|
_, _, _, _, _, _, before, after = line_in.split('\t')
|
||||||
|
word_before_in, word_after_in = preprocess(before)[-1], preprocess(after)[0]
|
||||||
|
file_out.write(predict(word_before_in, word_after_in) + '\n')
|
||||||
|
|
||||||
|
with open('dev-0/in.tsv', encoding='utf-8') as file_in, open('dev-0/out.tsv', 'w', encoding='utf-8') as file_out:
|
||||||
|
print('zapisywanie dev-0')
|
||||||
for line_in in file_in:
|
for line_in in file_in:
|
||||||
_, _, _, _, _, _, before, after = line_in.split('\t')
|
_, _, _, _, _, _, before, after = line_in.split('\t')
|
||||||
word_before_in, word_after_in = preprocess(before)[-1], preprocess(after)[0]
|
word_before_in, word_after_in = preprocess(before)[-1], preprocess(after)[0]
|
||||||
|
294
test-A/out.tsv
294
test-A/out.tsv
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue
Block a user