10000 rows top12
This commit is contained in:
parent
2f6c8330e5
commit
20f3c70aea
21034
dev-0/out.tsv
21034
dev-0/out.tsv
File diff suppressed because it is too large
Load Diff
14
run.py
14
run.py
@ -20,6 +20,8 @@ for index, row in train_data.iterrows():
|
|||||||
words = word_tokenize(text)
|
words = word_tokenize(text)
|
||||||
for w1, w2, w3 in trigrams(words, pad_right=True, pad_left=True):
|
for w1, w2, w3 in trigrams(words, pad_right=True, pad_left=True):
|
||||||
model[(w2, w3)][w1] += 1
|
model[(w2, w3)][w1] += 1
|
||||||
|
if index > 10000:
|
||||||
|
break
|
||||||
|
|
||||||
for w2_w3 in model:
|
for w2_w3 in model:
|
||||||
total_count = float(sum(model[w2_w3].values()))
|
total_count = float(sum(model[w2_w3].values()))
|
||||||
@ -28,7 +30,7 @@ for w2_w3 in model:
|
|||||||
|
|
||||||
def predict_probs(word1, word2):
|
def predict_probs(word1, word2):
|
||||||
raw_prediction = dict(model[word1, word2])
|
raw_prediction = dict(model[word1, word2])
|
||||||
prediction = dict(Counter(raw_prediction).most_common(20))
|
prediction = dict(Counter(raw_prediction).most_common(12))
|
||||||
|
|
||||||
total_prob = 0.0
|
total_prob = 0.0
|
||||||
str_prediction = ''
|
str_prediction = ''
|
||||||
@ -39,10 +41,10 @@ def predict_probs(word1, word2):
|
|||||||
|
|
||||||
remaining_prob = 1 - total_prob
|
remaining_prob = 1 - total_prob
|
||||||
|
|
||||||
if remaining_prob < 0.0000000001:
|
if remaining_prob < 0.0001:
|
||||||
remaining_prob = 0.0000000001
|
remaining_prob = 0.0001
|
||||||
|
|
||||||
str_prediction += f':{1-total_prob}'
|
str_prediction += f':{remaining_prob}'
|
||||||
|
|
||||||
return str_prediction
|
return str_prediction
|
||||||
|
|
||||||
@ -56,7 +58,7 @@ with open('dev-0/out.tsv', 'w') as file:
|
|||||||
text = text.replace('\\n', ' ')
|
text = text.replace('\\n', ' ')
|
||||||
words = word_tokenize(text)
|
words = word_tokenize(text)
|
||||||
if len(words) < 4:
|
if len(words) < 4:
|
||||||
prediction = ':1.0'
|
prediction = 'and:0.01 :0.99'
|
||||||
else:
|
else:
|
||||||
prediction = predict_probs(words[0], words[1])
|
prediction = predict_probs(words[0], words[1])
|
||||||
file.write(prediction + '\n')
|
file.write(prediction + '\n')
|
||||||
@ -68,7 +70,7 @@ with open('test-A/out.tsv', 'w') as file:
|
|||||||
text = text.replace('\\n', ' ')
|
text = text.replace('\\n', ' ')
|
||||||
words = word_tokenize(text)
|
words = word_tokenize(text)
|
||||||
if len(words) < 4:
|
if len(words) < 4:
|
||||||
prediction = ':1.0'
|
prediction = 'and:0.01 :0.99'
|
||||||
else:
|
else:
|
||||||
prediction = predict_probs(words[0], words[1])
|
prediction = predict_probs(words[0], words[1])
|
||||||
file.write(prediction + '\n')
|
file.write(prediction + '\n')
|
||||||
|
14828
test-A/out.tsv
14828
test-A/out.tsv
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user