rm

.gitignore

@@ -1,3 +1 @@
-mt-summit-corpora
+.idea
-.idea
-kompendium_lem*

inject.py

@@ -1,136 +0,0 @@
-import copy
-import pandas as pd
-import spacy
-from spaczz.matcher import FuzzyMatcher
-
-# spacy.require_gpu()
-
-spacy_nlp_en = spacy.load('en_core_web_sm')
-spacy_nlp_pl = spacy.load('pl_core_news_sm')
-
-print('lemmatizing glossary')
-
-glossary = pd.read_csv('glossary.tsv', sep='\t', header=None, names=['source', 'result'])
-
-source_lemmatized = []
-for word in glossary['source']:
-    temp = []
-    for token in spacy_nlp_en(word):
-        temp.append(token.lemma_)
-    source_lemmatized.append(' '.join(temp).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))
-
-result_lemmatized = []
-for word in glossary['result']:
-    temp = []
-    for token in spacy_nlp_pl(word):
-        temp.append(token.lemma_)
-    result_lemmatized.append(' '.join(temp).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))
-
-glossary['source_lem'] = source_lemmatized
-glossary['result_lem'] = result_lemmatized
-glossary = glossary[['source', 'source_lem', 'result', 'result_lem']]
-glossary.set_index('source_lem')
-
-glossary.to_csv('glossary_lem.tsv', sep='\t')
-
-dev_path = 'dev-0/'
-
-print('lemmatizing corpus ' + dev_path)
-
-skip_chars = ''',./!?'''
-
-with open(dev_path + 'in.tsv', 'r') as file:
-    file_lemmatized = []
-    for line in file:
-        temp = []
-        for token in spacy_nlp_en(line):
-            temp.append(token.lemma_)
-        file_lemmatized.append(' '.join([x for x in temp if x not in skip_chars])
-                .replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))
-
-with open(dev_path + 'expected.tsv', 'r') as file:
-    file_pl_lemmatized = []
-    for line in file:
-        temp = []
-        for token in spacy_nlp_pl(line):
-            temp.append(token.lemma_)
-        file_pl_lemmatized.append(' '.join([x for x in temp if x not in skip_chars])
-                .replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))
-
-# glossary
-glossary = pd.read_csv('glossary_lem.tsv', sep='\t', header=0, index_col=0)
-train_glossary = glossary.iloc[[x for x in range(len(glossary)) if x % 6 != 0]]
-
-# add rules to English matcher
-nlp = spacy.blank("en")
-matcher = FuzzyMatcher(nlp.vocab)
-for word in train_glossary['source_lem']:
-    matcher.add(word, [nlp(word)])
-
-# add rules to Polish matcher
-nlp_pl = spacy.blank("pl")
-matcher_pl = FuzzyMatcher(nlp_pl.vocab)
-for word, word_id in zip(train_glossary['result_lem'], train_glossary['source_lem']):
-    matcher_pl.add(word, [nlp_pl(word)])
-
-en = []
-translation_line_counts = []
-for line_id in range(len(file_lemmatized)):
-
-    if line_id % 100 == 0:
-        print('injecting glossary: ' + str(line_id) + "/" + str(len(file_lemmatized)), end='\r')
-
-    doc = nlp(file_lemmatized[line_id])
-    matches = matcher(doc)
-
-    line_counter = 0
-    for match_id, start, end, ratio in matches:
-        if ratio > 90:
-            doc_pl = nlp_pl(file_pl_lemmatized[line_id])
-            matches_pl = matcher_pl(doc_pl)
-
-            for match_id_pl, start_pl, end_pl, ratio_pl in matches_pl:
-                if match_id_pl == glossary[glossary['source_lem'] == match_id].values[0][3]:
-                    line_counter += 1
-                    en.append(''.join(doc[:end].text + ' ' + train_glossary.loc[lambda df: df['source_lem'] == match_id]['result'].astype(str).values.flatten() + ' ' + doc[end:].text))
-
-    if line_counter == 0:
-        line_counter = 1
-        en.append(file_lemmatized[line_id])
-    translation_line_counts.append(line_counter)
-
-print('saving files')
-tlcs = copy.deepcopy(translation_line_counts)
-
-translations = pd.read_csv(dev_path + 'expected.tsv', sep='\t', header=None, names=['text'])
-translations['id'] = [x for x in range(len(translations))]
-
-ctr = 0
-sentence = ''
-with open(dev_path + 'in.tsv.injected.crossvalidated', 'w') as file_en:
-    with open(dev_path + 'expected.tsv.injected.crossvalidated', 'w') as file_pl:
-        for i in range(len(en)):
-            if i > 0:
-                if en[i-1] != en[i]:
-                    if ctr == 0:
-                        sentence = translations.iloc[0]
-                        translations.drop(sentence['id'], inplace=True)
-                        sentence = sentence['text']
-                        try:
-                            ctr = tlcs.pop(0)
-                        except:
-                            pass
-                    file_en.write(en[i])
-                    file_pl.write(sentence + '\n')
-                ctr = ctr - 1
-            else:
-                try:
-                    ctr = tlcs.pop(0) - 1
-                except:
-                    pass
-                sentence = translations.iloc[0]
-                translations.drop(sentence['id'], inplace=True)
-                sentence = sentence['text']
-                file_en.write(en[i])
-                file_pl.write(sentence + '\n')
-

random-scripts/inject_rapid.py

@@ -1,130 +0,0 @@
-import spacy
-import copy
-import pandas as pd
-import rapidfuzz
-from rapidfuzz.fuzz import partial_ratio
-import time
-from rapidfuzz.utils import default_process
-import sys
-
-spacy.require_gpu()
-
-spacy_nlp_en = spacy.load('en_core_web_sm')
-spacy_nlp_pl = spacy.load("pl_core_news_sm")
-
-
-def read_arguments():
-    try:
-        corpus_path, glossary_path = sys.argv
-        return corpus_path, glossary_path
-    except:
-        print("ERROR: Wrong argument amount.")
-        sys.exit(1)
-
-
-
-glossary = pd.read_csv('mt-summit-corpora/glossary.tsv', sep='\t', header=None, names=['source', 'result'])
-
-source_lemmatized = []
-for word in glossary['source']:
-    temp = []
-    for token in spacy_nlp_en(word):
-        temp.append(token.lemma_)
-    source_lemmatized.append(' '.join(temp).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))
-
-result_lemmatized = []
-for word in glossary['result']:
-    temp = []
-    for token in spacy_nlp_pl(word):
-        temp.append(token.lemma_)
-    result_lemmatized.append(' '.join(temp).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))
-
-glossary['source_lem'] = source_lemmatized
-glossary['result_lem'] = result_lemmatized
-glossary = glossary[['source', 'source_lem', 'result', 'result_lem']]
-glossary.to_csv('kompendium_lem.tsv', sep='\t')
-
-corpus_path = 'mt-summit-corpora/train/'
-
-skip_chars = ''',./!?'''
-
-with open(corpus_path + 'in.tsv', 'r') as file:
-    file_lemmatized = []
-    for line in file:
-        if len(file_lemmatized) % 10000 == 0:
-            print(len(file_lemmatized), end='\r')
-        temp = []
-        for token in spacy_nlp_en(line):
-            temp.append(token.lemma_)
-        file_lemmatized.append(' '.join([x for x in temp if x not in skip_chars]).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))
-
-with open(corpus_path + 'expected.tsv', 'r') as file:
-    file_pl_lemmatized = []
-    for line in file:
-        if len(file_pl_lemmatized) % 10000 == 0:
-            print(len(file_lemmatized), end='\r')
-        temp = []
-        for token in spacy_nlp_pl(line):
-            temp.append(token.lemma_)
-        file_pl_lemmatized.append(' '.join([x for x in temp if x not in skip_chars]).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))
-
-THRESHOLD = 88
-
-def is_injectable(sentence_pl, sequence):
-    sen = sentence_pl.split()
-    window_size = len(sequence.split())
-    maxx = 0
-    for i in range(len(sen) - window_size):
-        current = rapidfuzz.fuzz.partial_ratio(' '.join(sen[i:i + window_size]), sequence)
-        if current > maxx:
-            maxx = current
-    return maxx
-
-def inject(sentence, sequence):
-    sen = sentence.split()
-    window_size = len(sequence.split())
-    maxx = 0
-    maxxi = 0
-    for i in range(len(sen) - window_size):
-        current = rapidfuzz.fuzz.partial_ratio(' '.join(sen[i:i + window_size]), sequence)
-        if current > maxx:
-            maxx = current
-            maxxi = i
-    return ' '.join(sen[:maxxi + window_size]) + ' ' \
-           + glossary.loc[lambda df: df['source_lem'] == sequence]['result'].astype(str).values.flatten() \
-           + ' ' + ' '.join(sen[maxxi + window_size:])
-
-glossary = pd.read_csv('../kompendium_lem_cleaned.tsv', sep='\t', header=0, index_col=0)
-glossary['source_lem'] = [default_process(x) for x in glossary['source_lem']]
-
-start_time = time.time_ns()
-en = []
-translation_line_counts = []
-for line, line_pl in zip(file_lemmatized, file_pl_lemmatized):
-    if len(translation_line_counts) % 50000 == 0:
-        print(str(len(translation_line_counts)) + '/' + str(len(file_lemmatized), end='\r'))
-    line = default_process(line)
-    line_pl = default_process(line_pl)
-    matchez = rapidfuzz.process.extract(query=line, choices=glossary['source_lem'], limit=5, score_cutoff=THRESHOLD, scorer=partial_ratio)
-    translation_line_counts.append(len(matchez))
-    for match in matchez:
-        # if is_injectable(line_pl, match[0]):
-        en.append(inject(line, match[0])[0])
-
-
-stop = time.time_ns()
-timex = (stop - start_time) / 1000000000
-print(timex)
-
-tlcs = copy.deepcopy(translation_line_counts)
-
-translations = pd.read_csv(corpus_path + 'expected.tsv', sep='\t', header=None, names=['text'])
-with open(corpus_path + 'extected.tsv.injected.crossvalidated.pl', 'w') as file_pl:
-    for line, translation_line_ct in zip(translations, tlcs):
-        for i in range(translation_line_ct):
-            file_pl.write(line)
-
-
-with open(corpus_path + 'in.tsv.injected.crossvalidated.en', 'w') as file_en:
-    for e in en:
-        file_en.write(e + '\n')

random-scripts/training-command.txt

@@ -1,30 +0,0 @@
-first iteration:
-./marian/build/marian --model mt.npz \
---type transformer --overwrite \
---train-sets mt-summit-corpora/mt-summit-corpora/dev/dev.en \
-mt-summit-corpora/mt-summit-corpora/dev/dev.pl \
---disp-freq 1000 \
---save-freq 1000 \
---optimizer adam \
---lr-report
-
-next iterations:
-./marian/build/marian --model mt.npz \
---type transformer --overwrite \
---train-sets mt-summit-corpora/mt-summit-corpora/dev/dev.en \
-mt-summit-corpora/mt-summit-corpora/dev/dev.pl \
---disp-freq 1000 \
---save-freq 1000 \
---optimizer adam \
---lr-report \
---pretrained-model mt.npz
-
-./marian/build/marian --model mt.npz \
---type transformer --overwrite \
---train-sets mt-summit-corpora/mt-summit-corpora/train/train.en \
-mt-summit-corpora/mt-summit-corpora/train/train.pl \
---disp-freq 1000 \
---save-freq 10000 \
---optimizer adam \
---lr-report \
---pretrained-model mt.npz

random-scripts/venv-setup.sh

@@ -1,12 +0,0 @@
-#!/bin.bash
-
-apt install python3-pip
-apt install python3-virtualenv
-virtualenv -p python3.8 gpu
-source gpu/bin/activate
-pip install pandas ipython
-pip install spacy[cuda114]
-python -m spacy download en_core_web_sm
-python -m spacy download pl_core_news_sm
-pip install spaczz
-pip install rapidfuzz

test.py

@@ -1,26 +0,0 @@
-import time
-import nltk
-from nltk.stem import WordNetLemmatizer
-
-# nltk.download('omw-1.4')
-# nltk.download('punkt')
-nltk.download('wordnet')
-
-wl = WordNetLemmatizer()
-
-start_time = time.time_ns()
-filex = []
-with open('mt-summit-corpora/train/in.tsv', 'r') as file:
-    for line in file:
-        if len(filex) % 50000 == 0:
-            print(len(filex), end='\r')
-        line = nltk.word_tokenize(line)
-        filex.append(' '.join([wl.lemmatize(x) for x in line]))
-
-
-stop = time.time_ns()
-timex = (stop - start_time) / 1000000000
-print(timex)
-f = open('temp', 'w')
-for line in filex:
-    f.write(line + '\n')