test 11 version

model.py

@@ -6,80 +6,63 @@ import csv
 import regex as re
 
 
-trigrams_list = []
-model = defaultdict(lambda: defaultdict(lambda: 0))
-
-
 def preprocess(text):
-    text = str(text).lower().replace("-\\n", "").replace("\\n", " ")
-    text = re.sub(r'\p{P}', '', text)
-    words = word_tokenize(text)
-    if len(words):
-        return words
-    return ['']
+    text = text.lower().replace('-\\n', '').replace('\\n', ' ')
+    return re.sub(r'\p{P}', '', text)
 
 
 def predict(word_before, word_after):
-    prob_list = dict(Counter(model[(word_before, word_after)]).most_common(5)).items()
-    predictions = []
-    prob_sum = 0.0
-    for key, value in prob_list:
-        prob_sum += value
-        predictions.append(f'{key}:{value}')
-    if prob_sum == 0.0:
+    prediction = dict(Counter(dict(model[word_before, word_after])).most_common(6))
+    result = ''
+    prob = 0.0
+    for key, value in prediction.items():
+        prob += value
+        result += f'{key}:{value} '
+    if prob == 0.0:
         return 'the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1'
-    elif prob_sum < 1.0:
-        predictions.append(f':{max(1 - prob_sum, 0.01)}')
-    return ' '.join(predictions)
+    result += f':{max(1 - prob, 0.01)}'
+    return result
 
 
-file_in = pd.read_csv('train/in.tsv.xz', sep='\t', on_bad_lines='skip', header=None, quoting=csv.QUOTE_NONE, nrows=200000)
-file_expected = pd.read_csv('train/expected.tsv', sep='\t', on_bad_lines='skip', header=None, quoting=csv.QUOTE_NONE, nrows=200000)
+def make_prediction(file):
+    data = pd.read_csv(f'{file}/in.tsv.xz', sep='\t', on_bad_lines='skip', header=None, quoting=csv.QUOTE_NONE)
+    with open(f'{file}/out.tsv', 'w', encoding='utf-8') as file_out:
+        for _, row in data.iterrows():
+            words_before, words_after = word_tokenize(preprocess(str(row[6]))), word_tokenize(preprocess(str(row[7])))
+            if len(words_before) < 3 or len(words_after) < 3:
+                prediction = 'the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1'
+            else:
+                prediction = predict(words_before[-1], words_after[0])
+            file_out.write(prediction + '\n')
 
-for index, (line_in, expected) in enumerate(zip(file_in.iterrows(), file_expected.iterrows())):
+
+train_data = pd.read_csv('train/in.tsv.xz', sep='\t', on_bad_lines='skip', header=None, quoting=csv.QUOTE_NONE, nrows=200000)
+train_labels = pd.read_csv('train/expected.tsv', sep='\t', on_bad_lines='skip', header=None, quoting=csv.QUOTE_NONE, nrows=200000)
+train_data = train_data[[6, 7]]
+train_data = pd.concat([train_data, train_labels], axis=1)
+train_data['line'] = train_data[6] + train_data[0] + train_data[7]
+trigrams_list = []
+model = defaultdict(lambda: defaultdict(lambda: 0))
+
+rows = train_data.iterrows()
+rows_len = len(train_data)
+for index, (_, row) in enumerate(rows):
     if index % 1000 == 0:
-        print('zbieranie trigramów', index)
-    before = line_in[1][6]
-    after = line_in[1][7]
-    expected = expected[1][0]
-    before, expected, after = preprocess(before), preprocess(expected), preprocess(after)
-    words = before + expected + after
-    trigrams_list += trigrams(words, pad_right=True, pad_left=True)
-    length = len(trigrams_list)
-
-trigrams_len = len(trigrams_list)
-
-for index, trigram in enumerate(trigrams_list):
-    if index % 100000 == 0:
-        print(f'uczenie modelu: {index / trigrams_len}')
-    if trigram[0] and trigram[1] and trigram[2]:
-        model[(trigram[0], trigram[2])][trigram[1]] += 1
-
+        print(f'uczenie modelu: {index / rows_len}')
+    text = preprocess(str(row['line']))
+    words = word_tokenize(text)
+    for word_1, word_2, word_3 in trigrams(words, pad_right=True, pad_left=True):
+        if word_1 and word_2 and word_3:
+            model[(word_1, word_3)][word_2] += 1
 
 model_len = len(model)
-
 for index, words_1_3 in enumerate(model):
     if index % 100000 == 0:
         print(f'normalizacja: {index / model_len}')
     count = sum(model[words_1_3].values())
     for word_2 in model[words_1_3]:
-        model[words_1_3][word_2] += 1
-        model[words_1_3][word_2] /= float(count + len(word_2)))
-
-
-def make_prediction(file):
-    file_in = pd.read_csv(f'{file}/in.tsv.xz', sep='\t', on_bad_lines='skip', header=None, quoting=csv.QUOTE_NONE)
-    with open(f'{file}/out.tsv', 'w', encoding='utf-8') as file_out:
-        print(f'zapisywanie {file}')
-        for line_in in file_in.iterrows():
-            before = line_in[1][6]
-            after = line_in[1][7]
-            if len(before) < 3 or len(after) < 3:
-                prediction = 'the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1'
-            else:
-                word_before_in, word_after_in = preprocess(before)[-1], preprocess(after)[0]
-                prediction = predict(word_before_in, word_after_in)
-            file_out.write(prediction + '\n')
+        model[words_1_3][word_2] += 0.25
+        model[words_1_3][word_2] /= float(count + 0.25 + len(word_2))
 
 
 make_prediction('test-A')