v3 all top6

run.py

@@ -27,8 +27,6 @@ for index, row in train_data.iterrows():
     for w1, w2, w3 in trigrams(words, pad_right=True, pad_left=True):
         if w1 and w2 and w3:
             model[(w2, w3)][w1] += 1
-    if index > 20000:
-        break
 
 for w2_w3 in model:
     total_count = float(sum(model[w2_w3].values()))
@@ -37,7 +35,7 @@ for w2_w3 in model:
 
 def predict_probs(word1, word2):
     raw_prediction = dict(model[word1, word2])
-    prediction = dict(Counter(raw_prediction).most_common(12))
+    prediction = dict(Counter(raw_prediction).most_common(6))
     
     total_prob = 0.0
     str_prediction = ''
@@ -47,12 +45,12 @@ def predict_probs(word1, word2):
         str_prediction += f'{word}:{prob} '
 
     if total_prob == 0.0:
-        return 'the:0.3 be:0.2 to:0.2 of:0.2 :0.1'
+        return 'the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1'
 
     remaining_prob = 1 - total_prob
 
-    if remaining_prob < 0.0001:
-        remaining_prob = 0.0001
+    if remaining_prob < 0.01:
+        remaining_prob = 0.01
         
     str_prediction += f':{remaining_prob}'
     
@@ -67,7 +65,7 @@ with open('dev-0/out.tsv', 'w') as file:
         text = clean_text(str(row[7]))
         words = word_tokenize(text)
         if len(words) < 4:
-            prediction = 'the:0.3 be:0.2 to:0.2 of:0.2 :0.1'
+            prediction = 'the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1'
         else:
             prediction = predict_probs(words[0], words[1])
         file.write(prediction + '\n')
@@ -77,7 +75,7 @@ with open('test-A/out.tsv', 'w') as file:
         text = clean_text(str(row[7]))
         words = word_tokenize(text)
         if len(words) < 4:
-            prediction = 'the:0.3 be:0.2 to:0.2 of:0.2 :0.1'
+            prediction = 'the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1'
         else:
             prediction = predict_probs(words[0], words[1])
         file.write(prediction + '\n')