first try
This commit is contained in:
parent
331a2ba38c
commit
9a1a5628a9
4846
dev-0/out.tsv
4846
dev-0/out.tsv
File diff suppressed because it is too large
Load Diff
155
run.py
155
run.py
@ -1,15 +1,144 @@
|
|||||||
import lzma
|
import lzma
|
||||||
with lzma.open('dev-0/in.tsv.xz', 'r') as file:
|
import matplotlib.pyplot as plt
|
||||||
for line in file:
|
from math import log
|
||||||
line = line.decode("utf-8")
|
from collections import OrderedDict
|
||||||
line = line.strip()
|
from collections import Counter
|
||||||
if 'I\t' in line:
|
import regex as re
|
||||||
print('am:0.9 :0.1')
|
from itertools import islice
|
||||||
elif 'you\t' in line:
|
|
||||||
print('are:0.9 :0.1')
|
def freq_list(g, top=None):
|
||||||
elif 'he\t' in line:
|
c = Counter(g)
|
||||||
print('is:0.9 :0.1')
|
|
||||||
elif 'she\t' in line:
|
if top is None:
|
||||||
print('is:0.9 :0.1')
|
items = c.items()
|
||||||
else:
|
else:
|
||||||
print('the:0.5 a:0.3 :0.2')
|
items = c.most_common(top)
|
||||||
|
|
||||||
|
return OrderedDict(sorted(items, key=lambda t: -t[1]))
|
||||||
|
|
||||||
|
def get_words(t):
|
||||||
|
for m in re.finditer(r'[\p{L}0-9-\*]+', t):
|
||||||
|
yield m.group(0)
|
||||||
|
|
||||||
|
def ngrams(iter, size):
|
||||||
|
ngram = []
|
||||||
|
for item in iter:
|
||||||
|
ngram.append(item)
|
||||||
|
if len(ngram) == size:
|
||||||
|
yield tuple(ngram)
|
||||||
|
ngram = ngram[1:]
|
||||||
|
|
||||||
|
PREFIX_TRAIN = 'dev-0'
|
||||||
|
trainset = ''
|
||||||
|
|
||||||
|
with lzma.open(f'{PREFIX_TRAIN}/in.tsv.xz', 'r') as train, open(f'{PREFIX_TRAIN}/expected.tsv', 'r') as expected:
|
||||||
|
for t_line, e_line in zip(train, expected):
|
||||||
|
t_line = t_line.decode("utf-8")
|
||||||
|
|
||||||
|
t_line = t_line.rstrip()
|
||||||
|
e_line = e_line.rstrip()
|
||||||
|
|
||||||
|
t_line_splitted_by_tab = t_line.split('\t')
|
||||||
|
|
||||||
|
t_line_cleared = t_line_splitted_by_tab[-2] + ' ' + e_line + ' ' + t_line_splitted_by_tab[-1]
|
||||||
|
|
||||||
|
|
||||||
|
trainset += t_line_cleared + ' '
|
||||||
|
|
||||||
|
trainset = trainset.replace('\\n', ' ')
|
||||||
|
# lzmaFile = lzma.open('dev-0/in.tsv.xz', 'rb')
|
||||||
|
|
||||||
|
# content = lzmaFile.read().decode("utf-8")
|
||||||
|
|
||||||
|
words = trainset.split()
|
||||||
|
# words = get_words(trainset)
|
||||||
|
|
||||||
|
ngrams_ = ngrams(words, 2)
|
||||||
|
|
||||||
|
|
||||||
|
def create_probabilities_bigrams(w_c, b_c):
|
||||||
|
probabilities_bigrams = {}
|
||||||
|
for bigram, bigram_amount in b_c.items():
|
||||||
|
if bigram_amount <=6:
|
||||||
|
continue
|
||||||
|
p_word_before = bigram_amount / w_c[bigram[0]]
|
||||||
|
p_word_after = bigram_amount / w_c[bigram[1]]
|
||||||
|
probabilities_bigrams[bigram] = (p_word_before, p_word_after)
|
||||||
|
|
||||||
|
return probabilities_bigrams
|
||||||
|
|
||||||
|
|
||||||
|
words_c = Counter(words)
|
||||||
|
bigram_c = Counter(ngrams_)
|
||||||
|
probabilities = create_probabilities_bigrams(words_c, bigram_c)
|
||||||
|
|
||||||
|
|
||||||
|
items = probabilities.items()
|
||||||
|
probabilities = OrderedDict(sorted(items, key=lambda t:t[1], reverse=True))
|
||||||
|
probabilities
|
||||||
|
# sorted_by_freq = freq_list(ngrams)
|
||||||
|
# print(bigram_c)
|
||||||
|
|
||||||
|
|
||||||
|
PREFIX_VALID = 'test-A'
|
||||||
|
|
||||||
|
def count_probabilities(w_b, w_a, probs, w_c, b_c):
|
||||||
|
results_before = {}
|
||||||
|
results_after = {}
|
||||||
|
for bigram, probses in probs.items():
|
||||||
|
if len(results_before) > 20 or len(results_after) > 20:
|
||||||
|
break
|
||||||
|
if w_b == bigram[0]:
|
||||||
|
results_before[bigram] = probses[0]
|
||||||
|
if w_a == bigram[1]:
|
||||||
|
results_after[bigram] = probses[1]
|
||||||
|
a=1
|
||||||
|
best_ = {}
|
||||||
|
|
||||||
|
for bigram, probses in results_before.items():
|
||||||
|
for bigram_2, probses_2 in results_after.items():
|
||||||
|
best_[bigram[1]] = probses * probses_2
|
||||||
|
|
||||||
|
for bigram, probses in results_after.items():
|
||||||
|
for bigram_2, probses_2 in results_before.items():
|
||||||
|
if bigram[0] in best_:
|
||||||
|
if probses * probses_2 < probses_2:
|
||||||
|
continue
|
||||||
|
best_[bigram[0]] = probses * probses_2
|
||||||
|
|
||||||
|
items = best_.items()
|
||||||
|
return OrderedDict(sorted(items, key=lambda t:t[1], reverse=True))
|
||||||
|
|
||||||
|
|
||||||
|
with lzma.open(f'{PREFIX_VALID}/in.tsv.xz', 'r') as train:
|
||||||
|
for t_line in train:
|
||||||
|
t_line = t_line.decode("utf-8")
|
||||||
|
|
||||||
|
t_line = t_line.rstrip()
|
||||||
|
t_line = t_line.replace('\\n', ' ')
|
||||||
|
|
||||||
|
|
||||||
|
t_line_splitted_by_tab = t_line.split('\t')
|
||||||
|
|
||||||
|
|
||||||
|
words_pre = t_line_splitted_by_tab[-2].split()
|
||||||
|
|
||||||
|
words_po = t_line_splitted_by_tab[-1].split()
|
||||||
|
|
||||||
|
w_pre = words_pre[-1]
|
||||||
|
w_po = words_po[0]
|
||||||
|
|
||||||
|
probs_ordered = count_probabilities(w_pre, w_po,probabilities, words_c, bigram_c)
|
||||||
|
if len(probs_ordered) ==0:
|
||||||
|
print(f"the:0.5 a:0.3 :0.2")
|
||||||
|
continue
|
||||||
|
result_string = ''
|
||||||
|
counter_ = 0
|
||||||
|
for word_, p in probs_ordered.items():
|
||||||
|
if counter_>4:
|
||||||
|
break
|
||||||
|
result_string += f"{word_}:{str(p)} "
|
||||||
|
counter_+=1
|
||||||
|
result_string += ':0.1'
|
||||||
|
print(result_string)
|
||||||
|
a=1
|
||||||
|
4314
test-A/out.tsv
4314
test-A/out.tsv
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user