first try

run.py

@@ -1,15 +1,144 @@
 import lzma
-with lzma.open('dev-0/in.tsv.xz', 'r') as file:
+import matplotlib.pyplot as plt
-    for line in file:
+from math import log
-        line = line.decode("utf-8")
+from collections import OrderedDict
-        line = line.strip()
+from collections import Counter
-        if 'I\t' in line:
+import regex as re
-            print('am:0.9 :0.1')
+from itertools import islice
-        elif 'you\t' in line:
+
-            print('are:0.9 :0.1')
+def freq_list(g, top=None):
-        elif 'he\t' in line:
+    c = Counter(g)
-            print('is:0.9 :0.1')
+
-        elif 'she\t' in line:
+    if top is None:
-            print('is:0.9 :0.1')
+       items = c.items()
-        else:
+    else:
-            print('the:0.5 a:0.3 :0.2')
+       items = c.most_common(top)
+
+    return OrderedDict(sorted(items, key=lambda t: -t[1]))
+
+def get_words(t):
+    for m in re.finditer(r'[\p{L}0-9-\*]+', t):
+        yield m.group(0)
+
+def ngrams(iter, size):
+  ngram = []
+  for item in iter:
+    ngram.append(item)
+    if len(ngram) == size:
+        yield tuple(ngram)
+        ngram = ngram[1:]
+
+PREFIX_TRAIN = 'dev-0' 
+trainset = ''
+
+with lzma.open(f'{PREFIX_TRAIN}/in.tsv.xz', 'r') as train, open(f'{PREFIX_TRAIN}/expected.tsv', 'r') as expected:
+    for t_line, e_line in zip(train, expected):
+        t_line = t_line.decode("utf-8")
+
+        t_line = t_line.rstrip()
+        e_line = e_line.rstrip()
+
+        t_line_splitted_by_tab = t_line.split('\t')
+        
+        t_line_cleared = t_line_splitted_by_tab[-2] + ' ' + e_line + ' ' + t_line_splitted_by_tab[-1]
+
+
+        trainset += t_line_cleared + ' '
+
+trainset = trainset.replace('\\n', ' ')
+# lzmaFile = lzma.open('dev-0/in.tsv.xz', 'rb')
+
+# content = lzmaFile.read().decode("utf-8")
+
+words = trainset.split()
+# words = get_words(trainset)
+
+ngrams_ = ngrams(words, 2)
+
+
+def create_probabilities_bigrams(w_c, b_c):
+    probabilities_bigrams = {}
+    for bigram, bigram_amount in b_c.items():
+        if bigram_amount <=6:
+            continue
+        p_word_before = bigram_amount / w_c[bigram[0]] 
+        p_word_after = bigram_amount / w_c[bigram[1]]
+        probabilities_bigrams[bigram] = (p_word_before, p_word_after)
+
+    return probabilities_bigrams
+
+
+words_c = Counter(words)
+bigram_c = Counter(ngrams_)
+probabilities = create_probabilities_bigrams(words_c, bigram_c)
+
+
+items = probabilities.items()
+probabilities = OrderedDict(sorted(items, key=lambda t:t[1], reverse=True))
+probabilities
+# sorted_by_freq = freq_list(ngrams)
+# print(bigram_c)
+
+
+PREFIX_VALID = 'test-A'
+
+def count_probabilities(w_b, w_a, probs, w_c, b_c):
+    results_before = {}
+    results_after = {}
+    for bigram, probses in probs.items():
+        if len(results_before) > 20 or len(results_after) > 20:
+            break
+        if w_b == bigram[0]:
+            results_before[bigram] = probses[0]
+        if w_a == bigram[1]:
+            results_after[bigram] = probses[1]
+    a=1
+    best_ = {}
+
+    for bigram, probses in results_before.items():
+        for bigram_2, probses_2 in results_after.items():
+            best_[bigram[1]] = probses * probses_2
+
+    for bigram, probses in results_after.items():
+            for bigram_2, probses_2 in results_before.items():
+                if bigram[0] in best_:
+                    if probses * probses_2 < probses_2:
+                        continue
+                best_[bigram[0]] = probses * probses_2
+
+    items = best_.items()
+    return OrderedDict(sorted(items, key=lambda t:t[1], reverse=True))
+
+
+with lzma.open(f'{PREFIX_VALID}/in.tsv.xz', 'r') as train:
+    for t_line in train:
+        t_line = t_line.decode("utf-8")
+
+        t_line = t_line.rstrip()
+        t_line = t_line.replace('\\n', ' ')
+
+
+        t_line_splitted_by_tab = t_line.split('\t')
+        
+
+        words_pre = t_line_splitted_by_tab[-2].split()
+
+        words_po = t_line_splitted_by_tab[-1].split()
+
+        w_pre = words_pre[-1]
+        w_po = words_po[0]
+
+        probs_ordered = count_probabilities(w_pre, w_po,probabilities, words_c, bigram_c)
+        if len(probs_ordered) ==0:
+            print(f"the:0.5 a:0.3 :0.2")
+            continue
+        result_string = ''
+        counter_ = 0
+        for word_, p in probs_ordered.items():
+            if counter_>4:
+                break
+            result_string += f"{word_}:{str(p)} "
+            counter_+=1
+        result_string += ':0.1'
+        print(result_string)
+        a=1