v1 top20 n-grams 434708

.gitignore

@@ -0,0 +1,8 @@
+
+*~
+*.swp
+*.bak
+*.pyc
+*.o
+.DS_Store
+.token

README.md

@@ -1,2 +1,9 @@
-# challenging-america-word-gap-prediction
+Challenging America word-gap prediction - s434708
+===================================
 
+Guess a word in a gap.
+
+Evaluation metric
+-----------------
+
+LikelihoodHashed is the metric

config.txt

@@ -0,0 +1 @@
+--metric PerplexityHashed --precision 2  --in-header in-header.tsv  --out-header out-header.tsv

in-header.tsv

@@ -0,0 +1 @@
+FileId	Year	LeftContext	RightContext

out-header.tsv

@@ -0,0 +1 @@
+Word

run.py

@@ -0,0 +1,73 @@
+import pandas as pd
+from nltk import trigrams, word_tokenize
+from collections import Counter, defaultdict
+
+train_data = pd.read_csv('train/in.tsv.xz', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None)
+train_labels = pd.read_csv('train/expected.tsv', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None)
+
+train_data = train_data[[6, 7]]
+train_data = pd.concat([train_data, train_labels], axis=1)
+
+train_data['final'] = train_data[6] + train_data[0] + train_data[7]
+
+model = defaultdict(lambda: defaultdict(lambda: 0))
+
+for index, row in train_data.iterrows():
+    text = str(row['final']).lower()
+    text = text.replace('-\\n', '')
+    text = text.replace('\\n', ' ')
+    words = word_tokenize(text)
+    for w1, w2, w3 in trigrams(words, pad_right=True, pad_left=True):
+        model[(w2, w3)][w1] += 1
+
+for w2_w3 in model:
+    total_count = float(sum(model[w2_w3].values()))
+    for w1 in model[w2_w3]:
+        model[w2_w3][w1] /= total_count
+
+def predict_probs(word1, word2):
+    raw_prediction = dict(model[word1, word2])
+    prediction = dict(Counter(raw_prediction).most_common(20))
+    
+    total_prob = 0.0
+    str_prediction = ''
+
+    for word, prob in prediction.items():
+        total_prob += prob
+        str_prediction += f'{word}:{prob} '
+
+    remaining_prob = 1 - total_prob
+
+    if remaining_prob < 0.0000000001:
+        remaining_prob = 0.0000000001
+        
+    str_prediction += f':{1-total_prob}'
+    
+    return str_prediction
+
+dev_data = pd.read_csv('dev-0/in.tsv.xz', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None)
+test_data = pd.read_csv('test-A/in.tsv.xz', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None)
+
+with open('dev-0/out.tsv', 'w') as file:
+    for index, row in dev_data.iterrows():
+        text = str(row[7]).lower()
+        text = text.replace('-\\n', '')
+        text = text.replace('\\n', ' ')
+        words = word_tokenize(text)
+        if len(words) < 4:
+            prediction = ':1.0'
+        else:
+            prediction = predict_probs(words[0], words[1])
+        file.write(prediction + '\n')
+
+with open('test-A/out.tsv', 'w') as file:
+    for index, row in test_data.iterrows():
+        text = str(row[7]).lower()
+        text = text.replace('-\\n', '')
+        text = text.replace('\\n', ' ')
+        words = word_tokenize(text)
+        if len(words) < 4:
+            prediction = ':1.0'
+        else:
+            prediction = predict_probs(words[0], words[1])
+        file.write(prediction + '\n')