test 8 version

model.py

@@ -1,9 +1,9 @@
 from nltk.tokenize import word_tokenize
 from nltk import trigrams
-import string
 from collections import defaultdict, Counter
 import pandas as pd
 import csv
+import regex as re
 
 
 trigrams_list = []
@@ -11,12 +11,9 @@ model = defaultdict(lambda: defaultdict(lambda: 0))
 
 
 def preprocess(text):
-    _text = str(text)
-    _text = _text.lower().replace("-\\n", "").replace('\\n', ' ').strip()
-    for character in _text:
-        if character not in string.ascii_lowercase + ' ':
-            _text = _text.replace(character, '')
-    words = word_tokenize(_text)
+    text = str(text).lower().replace("-\\n", "").replace("\\n", " ")
+    text = re.sub(r'\p{P}', '', text)
+    words = word_tokenize(text)
     if len(words):
         return words
     return ['']
@@ -30,7 +27,7 @@ def predict(word_before, word_after):
         prob_sum += value
         predictions.append(f'{key}:{value}')
     if prob_sum == 0.0:
-        return 'the:0:2 be:0.2 to:0.2 of:0.15 and:0.15 :0.1'
+        return 'the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1'
     elif prob_sum < 1.0:
         predictions.append(f':{max(1 - prob_sum, 0.01)}')
     return ' '.join(predictions)
@@ -68,23 +65,23 @@ for index, words_1_3 in enumerate(model):
     for word_2 in model[words_1_3]:
         model[words_1_3][word_2] /= float(count)
 
-file_in = pd.read_csv('test-A/in.tsv.xz', sep='\t', on_bad_lines='skip', header=None, quoting=csv.QUOTE_NONE)
-with open('test-A/out.tsv', 'w', encoding='utf-8') as file_out:
-    print('zapisywanie test-A')
-    for line_in in file_in.iterrows():
-        before = line_in[1][6]
-        after = line_in[1][7]
-        word_before_in, word_after_in = preprocess(before)[-1], preprocess(after)[0]
-        file_out.write(predict(word_before_in, word_after_in) + '\n')
+
+def make_prediction(file):
+    file_in = pd.read_csv(f'{file}/in.tsv.xz', sep='\t', on_bad_lines='skip', header=None, quoting=csv.QUOTE_NONE)
+    with open(f'{file}/out.tsv', 'w') as file_out:
+        print(f'zapisywanie {file}')
+        for line_in in file_in.iterrows():
+            before = line_in[1][6]
+            after = line_in[1][7]
+            if len(before) < 3 or len(after) < 3:
+                prediction = 'the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1'
+            else:
+                word_before_in, word_after_in = preprocess(before)[-1], preprocess(after)[0]
+                prediction = predict(word_before_in, word_after_in)
+            file_out.write(prediction + '\n')
 
 
-file_in = pd.read_csv('dev-0/in.tsv.xz', sep='\t', on_bad_lines='skip', header=None, quoting=csv.QUOTE_NONE)
-with open('dev-0/out.tsv', 'w', encoding='utf-8') as file_out:
-    print('zapisywanie dev-0')
-    for line_in in file_in.iterrows():
-        before = line_in[1][6]
-        after = line_in[1][7]
-        word_before_in, word_after_in = preprocess(before)[-1], preprocess(after)[0]
-        file_out.write(predict(word_before_in, word_after_in) + '\n')
+make_prediction('test-A')
+make_prediction('dev-0')
 
 print('koniec')