This commit is contained in:
parent
6c296fd9fc
commit
738d542f61
|
|
@ -1,3 +1 @@
|
|||
mt-summit-corpora
|
||||
.idea
|
||||
kompendium_lem*
|
||||
.idea
|
||||
136
inject.py
136
inject.py
|
|
@ -1,136 +0,0 @@
|
|||
import copy
|
||||
import pandas as pd
|
||||
import spacy
|
||||
from spaczz.matcher import FuzzyMatcher
|
||||
|
||||
# spacy.require_gpu()
|
||||
|
||||
spacy_nlp_en = spacy.load('en_core_web_sm')
|
||||
spacy_nlp_pl = spacy.load('pl_core_news_sm')
|
||||
|
||||
print('lemmatizing glossary')
|
||||
|
||||
glossary = pd.read_csv('glossary.tsv', sep='\t', header=None, names=['source', 'result'])
|
||||
|
||||
source_lemmatized = []
|
||||
for word in glossary['source']:
|
||||
temp = []
|
||||
for token in spacy_nlp_en(word):
|
||||
temp.append(token.lemma_)
|
||||
source_lemmatized.append(' '.join(temp).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))
|
||||
|
||||
result_lemmatized = []
|
||||
for word in glossary['result']:
|
||||
temp = []
|
||||
for token in spacy_nlp_pl(word):
|
||||
temp.append(token.lemma_)
|
||||
result_lemmatized.append(' '.join(temp).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))
|
||||
|
||||
glossary['source_lem'] = source_lemmatized
|
||||
glossary['result_lem'] = result_lemmatized
|
||||
glossary = glossary[['source', 'source_lem', 'result', 'result_lem']]
|
||||
glossary.set_index('source_lem')
|
||||
|
||||
glossary.to_csv('glossary_lem.tsv', sep='\t')
|
||||
|
||||
dev_path = 'dev-0/'
|
||||
|
||||
print('lemmatizing corpus ' + dev_path)
|
||||
|
||||
skip_chars = ''',./!?'''
|
||||
|
||||
with open(dev_path + 'in.tsv', 'r') as file:
|
||||
file_lemmatized = []
|
||||
for line in file:
|
||||
temp = []
|
||||
for token in spacy_nlp_en(line):
|
||||
temp.append(token.lemma_)
|
||||
file_lemmatized.append(' '.join([x for x in temp if x not in skip_chars])
|
||||
.replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))
|
||||
|
||||
with open(dev_path + 'expected.tsv', 'r') as file:
|
||||
file_pl_lemmatized = []
|
||||
for line in file:
|
||||
temp = []
|
||||
for token in spacy_nlp_pl(line):
|
||||
temp.append(token.lemma_)
|
||||
file_pl_lemmatized.append(' '.join([x for x in temp if x not in skip_chars])
|
||||
.replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))
|
||||
|
||||
# glossary
|
||||
glossary = pd.read_csv('glossary_lem.tsv', sep='\t', header=0, index_col=0)
|
||||
train_glossary = glossary.iloc[[x for x in range(len(glossary)) if x % 6 != 0]]
|
||||
|
||||
# add rules to English matcher
|
||||
nlp = spacy.blank("en")
|
||||
matcher = FuzzyMatcher(nlp.vocab)
|
||||
for word in train_glossary['source_lem']:
|
||||
matcher.add(word, [nlp(word)])
|
||||
|
||||
# add rules to Polish matcher
|
||||
nlp_pl = spacy.blank("pl")
|
||||
matcher_pl = FuzzyMatcher(nlp_pl.vocab)
|
||||
for word, word_id in zip(train_glossary['result_lem'], train_glossary['source_lem']):
|
||||
matcher_pl.add(word, [nlp_pl(word)])
|
||||
|
||||
en = []
|
||||
translation_line_counts = []
|
||||
for line_id in range(len(file_lemmatized)):
|
||||
|
||||
if line_id % 100 == 0:
|
||||
print('injecting glossary: ' + str(line_id) + "/" + str(len(file_lemmatized)), end='\r')
|
||||
|
||||
doc = nlp(file_lemmatized[line_id])
|
||||
matches = matcher(doc)
|
||||
|
||||
line_counter = 0
|
||||
for match_id, start, end, ratio in matches:
|
||||
if ratio > 90:
|
||||
doc_pl = nlp_pl(file_pl_lemmatized[line_id])
|
||||
matches_pl = matcher_pl(doc_pl)
|
||||
|
||||
for match_id_pl, start_pl, end_pl, ratio_pl in matches_pl:
|
||||
if match_id_pl == glossary[glossary['source_lem'] == match_id].values[0][3]:
|
||||
line_counter += 1
|
||||
en.append(''.join(doc[:end].text + ' ' + train_glossary.loc[lambda df: df['source_lem'] == match_id]['result'].astype(str).values.flatten() + ' ' + doc[end:].text))
|
||||
|
||||
if line_counter == 0:
|
||||
line_counter = 1
|
||||
en.append(file_lemmatized[line_id])
|
||||
translation_line_counts.append(line_counter)
|
||||
|
||||
print('saving files')
|
||||
tlcs = copy.deepcopy(translation_line_counts)
|
||||
|
||||
translations = pd.read_csv(dev_path + 'expected.tsv', sep='\t', header=None, names=['text'])
|
||||
translations['id'] = [x for x in range(len(translations))]
|
||||
|
||||
ctr = 0
|
||||
sentence = ''
|
||||
with open(dev_path + 'in.tsv.injected.crossvalidated', 'w') as file_en:
|
||||
with open(dev_path + 'expected.tsv.injected.crossvalidated', 'w') as file_pl:
|
||||
for i in range(len(en)):
|
||||
if i > 0:
|
||||
if en[i-1] != en[i]:
|
||||
if ctr == 0:
|
||||
sentence = translations.iloc[0]
|
||||
translations.drop(sentence['id'], inplace=True)
|
||||
sentence = sentence['text']
|
||||
try:
|
||||
ctr = tlcs.pop(0)
|
||||
except:
|
||||
pass
|
||||
file_en.write(en[i])
|
||||
file_pl.write(sentence + '\n')
|
||||
ctr = ctr - 1
|
||||
else:
|
||||
try:
|
||||
ctr = tlcs.pop(0) - 1
|
||||
except:
|
||||
pass
|
||||
sentence = translations.iloc[0]
|
||||
translations.drop(sentence['id'], inplace=True)
|
||||
sentence = sentence['text']
|
||||
file_en.write(en[i])
|
||||
file_pl.write(sentence + '\n')
|
||||
|
||||
|
|
@ -1,130 +0,0 @@
|
|||
import spacy
|
||||
import copy
|
||||
import pandas as pd
|
||||
import rapidfuzz
|
||||
from rapidfuzz.fuzz import partial_ratio
|
||||
import time
|
||||
from rapidfuzz.utils import default_process
|
||||
import sys
|
||||
|
||||
spacy.require_gpu()
|
||||
|
||||
spacy_nlp_en = spacy.load('en_core_web_sm')
|
||||
spacy_nlp_pl = spacy.load("pl_core_news_sm")
|
||||
|
||||
|
||||
def read_arguments():
|
||||
try:
|
||||
corpus_path, glossary_path = sys.argv
|
||||
return corpus_path, glossary_path
|
||||
except:
|
||||
print("ERROR: Wrong argument amount.")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
|
||||
glossary = pd.read_csv('mt-summit-corpora/glossary.tsv', sep='\t', header=None, names=['source', 'result'])
|
||||
|
||||
source_lemmatized = []
|
||||
for word in glossary['source']:
|
||||
temp = []
|
||||
for token in spacy_nlp_en(word):
|
||||
temp.append(token.lemma_)
|
||||
source_lemmatized.append(' '.join(temp).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))
|
||||
|
||||
result_lemmatized = []
|
||||
for word in glossary['result']:
|
||||
temp = []
|
||||
for token in spacy_nlp_pl(word):
|
||||
temp.append(token.lemma_)
|
||||
result_lemmatized.append(' '.join(temp).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))
|
||||
|
||||
glossary['source_lem'] = source_lemmatized
|
||||
glossary['result_lem'] = result_lemmatized
|
||||
glossary = glossary[['source', 'source_lem', 'result', 'result_lem']]
|
||||
glossary.to_csv('kompendium_lem.tsv', sep='\t')
|
||||
|
||||
corpus_path = 'mt-summit-corpora/train/'
|
||||
|
||||
skip_chars = ''',./!?'''
|
||||
|
||||
with open(corpus_path + 'in.tsv', 'r') as file:
|
||||
file_lemmatized = []
|
||||
for line in file:
|
||||
if len(file_lemmatized) % 10000 == 0:
|
||||
print(len(file_lemmatized), end='\r')
|
||||
temp = []
|
||||
for token in spacy_nlp_en(line):
|
||||
temp.append(token.lemma_)
|
||||
file_lemmatized.append(' '.join([x for x in temp if x not in skip_chars]).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))
|
||||
|
||||
with open(corpus_path + 'expected.tsv', 'r') as file:
|
||||
file_pl_lemmatized = []
|
||||
for line in file:
|
||||
if len(file_pl_lemmatized) % 10000 == 0:
|
||||
print(len(file_lemmatized), end='\r')
|
||||
temp = []
|
||||
for token in spacy_nlp_pl(line):
|
||||
temp.append(token.lemma_)
|
||||
file_pl_lemmatized.append(' '.join([x for x in temp if x not in skip_chars]).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))
|
||||
|
||||
THRESHOLD = 88
|
||||
|
||||
def is_injectable(sentence_pl, sequence):
|
||||
sen = sentence_pl.split()
|
||||
window_size = len(sequence.split())
|
||||
maxx = 0
|
||||
for i in range(len(sen) - window_size):
|
||||
current = rapidfuzz.fuzz.partial_ratio(' '.join(sen[i:i + window_size]), sequence)
|
||||
if current > maxx:
|
||||
maxx = current
|
||||
return maxx
|
||||
|
||||
def inject(sentence, sequence):
|
||||
sen = sentence.split()
|
||||
window_size = len(sequence.split())
|
||||
maxx = 0
|
||||
maxxi = 0
|
||||
for i in range(len(sen) - window_size):
|
||||
current = rapidfuzz.fuzz.partial_ratio(' '.join(sen[i:i + window_size]), sequence)
|
||||
if current > maxx:
|
||||
maxx = current
|
||||
maxxi = i
|
||||
return ' '.join(sen[:maxxi + window_size]) + ' ' \
|
||||
+ glossary.loc[lambda df: df['source_lem'] == sequence]['result'].astype(str).values.flatten() \
|
||||
+ ' ' + ' '.join(sen[maxxi + window_size:])
|
||||
|
||||
glossary = pd.read_csv('../kompendium_lem_cleaned.tsv', sep='\t', header=0, index_col=0)
|
||||
glossary['source_lem'] = [default_process(x) for x in glossary['source_lem']]
|
||||
|
||||
start_time = time.time_ns()
|
||||
en = []
|
||||
translation_line_counts = []
|
||||
for line, line_pl in zip(file_lemmatized, file_pl_lemmatized):
|
||||
if len(translation_line_counts) % 50000 == 0:
|
||||
print(str(len(translation_line_counts)) + '/' + str(len(file_lemmatized), end='\r'))
|
||||
line = default_process(line)
|
||||
line_pl = default_process(line_pl)
|
||||
matchez = rapidfuzz.process.extract(query=line, choices=glossary['source_lem'], limit=5, score_cutoff=THRESHOLD, scorer=partial_ratio)
|
||||
translation_line_counts.append(len(matchez))
|
||||
for match in matchez:
|
||||
# if is_injectable(line_pl, match[0]):
|
||||
en.append(inject(line, match[0])[0])
|
||||
|
||||
|
||||
stop = time.time_ns()
|
||||
timex = (stop - start_time) / 1000000000
|
||||
print(timex)
|
||||
|
||||
tlcs = copy.deepcopy(translation_line_counts)
|
||||
|
||||
translations = pd.read_csv(corpus_path + 'expected.tsv', sep='\t', header=None, names=['text'])
|
||||
with open(corpus_path + 'extected.tsv.injected.crossvalidated.pl', 'w') as file_pl:
|
||||
for line, translation_line_ct in zip(translations, tlcs):
|
||||
for i in range(translation_line_ct):
|
||||
file_pl.write(line)
|
||||
|
||||
|
||||
with open(corpus_path + 'in.tsv.injected.crossvalidated.en', 'w') as file_en:
|
||||
for e in en:
|
||||
file_en.write(e + '\n')
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
first iteration:
|
||||
./marian/build/marian --model mt.npz \
|
||||
--type transformer --overwrite \
|
||||
--train-sets mt-summit-corpora/mt-summit-corpora/dev/dev.en \
|
||||
mt-summit-corpora/mt-summit-corpora/dev/dev.pl \
|
||||
--disp-freq 1000 \
|
||||
--save-freq 1000 \
|
||||
--optimizer adam \
|
||||
--lr-report
|
||||
|
||||
next iterations:
|
||||
./marian/build/marian --model mt.npz \
|
||||
--type transformer --overwrite \
|
||||
--train-sets mt-summit-corpora/mt-summit-corpora/dev/dev.en \
|
||||
mt-summit-corpora/mt-summit-corpora/dev/dev.pl \
|
||||
--disp-freq 1000 \
|
||||
--save-freq 1000 \
|
||||
--optimizer adam \
|
||||
--lr-report \
|
||||
--pretrained-model mt.npz
|
||||
|
||||
./marian/build/marian --model mt.npz \
|
||||
--type transformer --overwrite \
|
||||
--train-sets mt-summit-corpora/mt-summit-corpora/train/train.en \
|
||||
mt-summit-corpora/mt-summit-corpora/train/train.pl \
|
||||
--disp-freq 1000 \
|
||||
--save-freq 10000 \
|
||||
--optimizer adam \
|
||||
--lr-report \
|
||||
--pretrained-model mt.npz
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
#!/bin.bash
|
||||
|
||||
apt install python3-pip
|
||||
apt install python3-virtualenv
|
||||
virtualenv -p python3.8 gpu
|
||||
source gpu/bin/activate
|
||||
pip install pandas ipython
|
||||
pip install spacy[cuda114]
|
||||
python -m spacy download en_core_web_sm
|
||||
python -m spacy download pl_core_news_sm
|
||||
pip install spaczz
|
||||
pip install rapidfuzz
|
||||
11740
rapidfuzztest.ipynb
11740
rapidfuzztest.ipynb
File diff suppressed because it is too large
Load Diff
26
test.py
26
test.py
|
|
@ -1,26 +0,0 @@
|
|||
import time
|
||||
import nltk
|
||||
from nltk.stem import WordNetLemmatizer
|
||||
|
||||
# nltk.download('omw-1.4')
|
||||
# nltk.download('punkt')
|
||||
nltk.download('wordnet')
|
||||
|
||||
wl = WordNetLemmatizer()
|
||||
|
||||
start_time = time.time_ns()
|
||||
filex = []
|
||||
with open('mt-summit-corpora/train/in.tsv', 'r') as file:
|
||||
for line in file:
|
||||
if len(filex) % 50000 == 0:
|
||||
print(len(filex), end='\r')
|
||||
line = nltk.word_tokenize(line)
|
||||
filex.append(' '.join([wl.lemmatize(x) for x in line]))
|
||||
|
||||
|
||||
stop = time.time_ns()
|
||||
timex = (stop - start_time) / 1000000000
|
||||
print(timex)
|
||||
f = open('temp', 'w')
|
||||
for line in filex:
|
||||
f.write(line + '\n')
|
||||
Loading…
Reference in New Issue