test 4 version

model.py

@@ -14,7 +14,10 @@ def preprocess(text):
     for character in _text:
         if character not in string.ascii_lowercase + ' ':
             _text = _text.replace(character, '')
-    return word_tokenize(_text)
+    words = word_tokenize(_text)
+    if len(words):
+        return words
+    return ['']
 
 
 def predict(word_before, word_after):
@@ -42,11 +45,14 @@ with open('train/in.tsv', encoding='utf-8') as file_in, open('train/expected.tsv
         before, expected, after = preprocess(before), preprocess(expected), preprocess(after)
         words = before + expected + after
         trigrams_list += trigrams(words, pad_right=True, pad_left=True)
+        length = len(trigrams_list)
+        print('zbieranie trigramów:', length)
+        if length > 1000000:
+            break
 
-trigrams_len = len(trigrams_list)
 for index, trigram in enumerate(trigrams_list):
     if index % 100000 == 0:
-        print('uczenie modelu', index, '/', trigrams_len)
+        print('uczenie modelu', index)
     if not trigram[0] or not trigram[1] or not trigram[2]:
         continue
     model[(trigram[0], trigram[2])][trigram[1]] += 1
@@ -62,7 +68,14 @@ for index, words_1_3 in enumerate(model):
         model[words_1_3][word_2] /= float(count)
 
 with open('test-A/in.tsv', encoding='utf-8') as file_in, open('test-A/out.tsv', 'w', encoding='utf-8') as file_out:
-    print('zapisywanie wyników')
+    print('zapisywanie test-A')
+    for line_in in file_in:
+        _, _, _, _, _, _, before, after = line_in.split('\t')
+        word_before_in, word_after_in = preprocess(before)[-1], preprocess(after)[0]
+        file_out.write(predict(word_before_in, word_after_in) + '\n')
+
+with open('dev-0/in.tsv', encoding='utf-8') as file_in, open('dev-0/out.tsv', 'w', encoding='utf-8') as file_out:
+    print('zapisywanie dev-0')
     for line_in in file_in:
         _, _, _, _, _, _, before, after = line_in.split('\t')
         word_before_in, word_after_in = preprocess(before)[-1], preprocess(after)[0]