219 lines
6.3 KiB
Plaintext
219 lines
6.3 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"outputs": [],
|
|
"source": [
|
|
"\n",
|
|
"import copy\n",
|
|
"import nltk\n",
|
|
"import pandas as pd\n",
|
|
"import rapidfuzz\n",
|
|
"import time\n",
|
|
"\n",
|
|
"from nltk.stem import WordNetLemmatizer\n",
|
|
"from rapidfuzz.fuzz import partial_ratio\n",
|
|
"from rapidfuzz.utils import default_process\n",
|
|
"\n",
|
|
"\n",
|
|
"wl = WordNetLemmatizer()\n",
|
|
"\n",
|
|
"glossary = pd.read_csv('mt-summit-corpora/glossary.tsv', sep='\\t', header=None, names=['source', 'result'])\n",
|
|
"\n",
|
|
"source_lemmatized = []\n",
|
|
"for word in glossary['source']:\n",
|
|
" word = nltk.word_tokenize(word)\n",
|
|
" source_lemmatized.append(' '.join([wl.lemmatize(x) for x in word]))\n",
|
|
"\n",
|
|
"glossary['source_lem'] = source_lemmatized\n",
|
|
"glossary = glossary[['source', 'source_lem', 'result']]\n",
|
|
"glossary.set_index('source_lem')\n",
|
|
"\n"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%%\n",
|
|
"is_executing": true
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 36,
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"0.194806501\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# train_in_path = 'mt-summit-corpora/train/in.tsv'\n",
|
|
"# train_expected_path = 'mt-summit-corpora/train/expected.tsv'\n",
|
|
"\n",
|
|
"train_in_path = 'mt-summit-corpora/dev-0/in.tsv'\n",
|
|
"train_expected_path = 'mt-summit-corpora/dev-0/expected.tsv'\n",
|
|
"\n",
|
|
"\n",
|
|
"file_pl = pd.read_csv(train_expected_path, sep='\\t', header=None, names=['text'])\n",
|
|
"\n",
|
|
"start_time = time.time_ns()\n",
|
|
"file_lemmatized = []\n",
|
|
"with open(train_in_path, 'r') as file:\n",
|
|
" for line in file:\n",
|
|
" if len(file_lemmatized) % 50000 == 0:\n",
|
|
" print(len(file_lemmatized), end='\\r')\n",
|
|
" line = nltk.word_tokenize(line)\n",
|
|
" file_lemmatized.append(' '.join([wl.lemmatize(x) for x in line]))\n",
|
|
"\n",
|
|
"stop = time.time_ns()\n",
|
|
"timex = (stop - start_time) / 1000000000\n",
|
|
"print(timex)\n"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%%\n"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 45,
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"6.904260614\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"\n",
|
|
"THRESHOLD = 88\n",
|
|
"\n",
|
|
"def is_injectable(sentence_pl, sequence):\n",
|
|
" sen = sentence_pl.split()\n",
|
|
" window_size = len(sequence.split())\n",
|
|
" maxx = 0\n",
|
|
" for i in range(len(sen) - window_size):\n",
|
|
" current = rapidfuzz.fuzz.ratio(' '.join(sen[i:i + window_size]), sequence)\n",
|
|
" if current > maxx:\n",
|
|
" maxx = current\n",
|
|
" if maxx >= THRESHOLD:\n",
|
|
" return True\n",
|
|
" else:\n",
|
|
" return False\n",
|
|
"\n",
|
|
"def get_injected(sentence, sequence, inject):\n",
|
|
" sen = sentence.split()\n",
|
|
" window_size = len(sequence.split())\n",
|
|
" maxx = 0\n",
|
|
" maxxi = 0\n",
|
|
" for i in range(len(sen) - window_size + 1):\n",
|
|
" current = rapidfuzz.fuzz.ratio(' '.join(sen[i:i + window_size]), sequence)\n",
|
|
" if current >= maxx:\n",
|
|
" maxx = current\n",
|
|
" maxxi = i\n",
|
|
" return ' '.join(sen[:maxxi + window_size]) + ' $' + inject + '$ ' + ' '.join(sen[maxxi + window_size:])\n",
|
|
"\n",
|
|
"glossary['source_lem'] = [' ' + str(default_process(x)) + ' ' for x in glossary['source_lem']]\n",
|
|
"\n",
|
|
"start_time = time.time_ns()\n",
|
|
"en = []\n",
|
|
"translation_line_counts = []\n",
|
|
"for line, line_pl in zip(file_lemmatized, file_pl['text'].values.tolist()):\n",
|
|
" line = default_process(line)\n",
|
|
" line_pl = default_process(line_pl)\n",
|
|
" matchez = rapidfuzz.process.extract(query=line, choices=glossary['source_lem'], limit=5, score_cutoff=THRESHOLD, scorer=partial_ratio)\n",
|
|
" if len(matchez) > 0:\n",
|
|
" lines_added = 0\n",
|
|
" for match in matchez:\n",
|
|
" polish_translation = glossary.loc[lambda df: df['source_lem'] == match[0]]['result'].astype(str).values.flatten()[0]\n",
|
|
" if is_injectable(line_pl, polish_translation):\n",
|
|
" en.append(get_injected(line, match[0], polish_translation))\n",
|
|
" lines_added += 1\n",
|
|
" if lines_added == 0:\n",
|
|
" en.append(line)\n",
|
|
" lines_added = 1\n",
|
|
" translation_line_counts.append(lines_added)\n",
|
|
" else:\n",
|
|
" translation_line_counts.append(1)\n",
|
|
" en.append(line)\n",
|
|
"\n",
|
|
"\n",
|
|
"stop = time.time_ns()\n",
|
|
"timex = (stop - start_time) / 1000000000\n",
|
|
"print(timex)\n"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%%\n"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 46,
|
|
"outputs": [],
|
|
"source": [
|
|
"\n",
|
|
"translations = pd.read_csv(train_expected_path, sep='\\t', header=0, names=['text'])\n",
|
|
"with open(train_expected_path + '.injected', 'w') as file_plx:\n",
|
|
" for line, translation_line_ct in zip(translations['text'].values.tolist(), translation_line_counts):\n",
|
|
" for i in range(translation_line_ct):\n",
|
|
" file_plx.write(line + '\\n')\n",
|
|
"\n",
|
|
"\n",
|
|
"with open(train_in_path + '.injected', 'w') as file_en:\n",
|
|
" for e in en:\n",
|
|
" file_en.write(e + '\\n')"
|
|
],
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%%\n"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"outputs": [],
|
|
"source": [],
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"pycharm": {
|
|
"name": "#%%\n"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 2
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython2",
|
|
"version": "2.7.6"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 0
|
|
} |