s434708/challenging-america-word-gap-prediction
1
0
Fork 0

10000 rows top12

Browse Source
This commit is contained in:
Łukasz Jędyk 2022-04-02 17:05:56 +02:00
parent 2f6c8330e5
commit 20f3c70aea
3 changed files with 17939 additions and 17937 deletions

21034
dev-0/out.tsv
View File

File diff suppressed because it is too large Load Diff

14
run.py
View File

@ -20,6 +20,8 @@ for index, row in train_data.iterrows():
words = word_tokenize(text) words = word_tokenize(text)
for w1, w2, w3 in trigrams(words, pad_right=True, pad_left=True): for w1, w2, w3 in trigrams(words, pad_right=True, pad_left=True):
model[(w2, w3)][w1] += 1 model[(w2, w3)][w1] += 1
if index > 10000:
break
for w2_w3 in model: for w2_w3 in model:
total_count = float(sum(model[w2_w3].values())) total_count = float(sum(model[w2_w3].values()))
@ -28,7 +30,7 @@ for w2_w3 in model:
def predict_probs(word1, word2): def predict_probs(word1, word2):
raw_prediction = dict(model[word1, word2]) raw_prediction = dict(model[word1, word2])
prediction = dict(Counter(raw_prediction).most_common(20)) prediction = dict(Counter(raw_prediction).most_common(12))
total_prob = 0.0 total_prob = 0.0
str_prediction = '' str_prediction = ''
@ -39,10 +41,10 @@ def predict_probs(word1, word2):
remaining_prob = 1 - total_prob remaining_prob = 1 - total_prob
if remaining_prob < 0.0000000001: if remaining_prob < 0.0001:
remaining_prob = 0.0000000001 remaining_prob = 0.0001
str_prediction += f':{1-total_prob}' str_prediction += f':{remaining_prob}'
return str_prediction return str_prediction
@ -56,7 +58,7 @@ with open('dev-0/out.tsv', 'w') as file:
text = text.replace('\\n', ' ') text = text.replace('\\n', ' ')
words = word_tokenize(text) words = word_tokenize(text)
if len(words) < 4: if len(words) < 4:
prediction = ':1.0' prediction = 'and:0.01 :0.99'
else: else:
prediction = predict_probs(words[0], words[1]) prediction = predict_probs(words[0], words[1])
file.write(prediction + '\n') file.write(prediction + '\n')
@ -68,7 +70,7 @@ with open('test-A/out.tsv', 'w') as file:
text = text.replace('\\n', ' ') text = text.replace('\\n', ' ')
words = word_tokenize(text) words = word_tokenize(text)
if len(words) < 4: if len(words) < 4:
prediction = ':1.0' prediction = 'and:0.01 :0.99'
else: else:
prediction = predict_probs(words[0], words[1]) prediction = predict_probs(words[0], words[1])
file.write(prediction + '\n') file.write(prediction + '\n')

14828
test-A/out.tsv
View File

File diff suppressed because it is too large Load Diff