s470607/transfix-mt
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

inject in non-lemmatized

Browse Source
This commit is contained in:
jakubknczny 2022-01-22 01:50:09 +01:00
parent 3cbab11535
commit 716d7b7072
1 changed files with 23 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
rapidfuzztest.ipynb
View File

@ -6,7 +6,6 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"\n", "\n",
"import copy\n",
"import nltk\n", "import nltk\n",
"import pandas as pd\n", "import pandas as pd\n",
"import rapidfuzz\n", "import rapidfuzz\n",
@ -59,8 +58,6 @@
"train_expected_path = 'mt-summit-corpora/dev-0/expected.tsv'\n", "train_expected_path = 'mt-summit-corpora/dev-0/expected.tsv'\n",
"\n", "\n",
"\n", "\n",
"file_pl = pd.read_csv(train_expected_path, sep='\\t', header=None, names=['text'])\n",
"\n",
"start_time = time.time_ns()\n", "start_time = time.time_ns()\n",
"file_lemmatized = []\n", "file_lemmatized = []\n",
"with open(train_in_path, 'r') as file:\n", "with open(train_in_path, 'r') as file:\n",
@ -83,25 +80,25 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 45, "execution_count": 64,
"outputs": [ "outputs": [
{ {
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"6.904260614\n" "6.915366953\n"
] ]
} }
], ],
"source": [ "source": [
"\n", "\n",
"THRESHOLD = 88\n", "THRESHOLD = 70\n",
"\n", "\n",
"def is_injectable(sentence_pl, sequence):\n", "def is_injectable(sentence_pl, sequence):\n",
" sen = sentence_pl.split()\n", " sen = sentence_pl.split()\n",
" window_size = len(sequence.split())\n", " window_size = len(sequence.split())\n",
" maxx = 0\n", " maxx = 0\n",
" for i in range(len(sen) - window_size):\n", " for i in range(len(sen) - window_size + 1):\n",
" current = rapidfuzz.fuzz.ratio(' '.join(sen[i:i + window_size]), sequence)\n", " current = rapidfuzz.fuzz.ratio(' '.join(sen[i:i + window_size]), sequence)\n",
" if current > maxx:\n", " if current > maxx:\n",
" maxx = current\n", " maxx = current\n",
@ -110,8 +107,9 @@
" else:\n", " else:\n",
" return False\n", " return False\n",
"\n", "\n",
"def get_injected(sentence, sequence, inject):\n", "def get_injected(sentence, sentence_en, sequence, inject):\n",
" sen = sentence.split()\n", " sen = sentence.split()\n",
" sen_en = sentence_en.split()\n",
" window_size = len(sequence.split())\n", " window_size = len(sequence.split())\n",
" maxx = 0\n", " maxx = 0\n",
" maxxi = 0\n", " maxxi = 0\n",
@ -120,31 +118,35 @@
" if current >= maxx:\n", " if current >= maxx:\n",
" maxx = current\n", " maxx = current\n",
" maxxi = i\n", " maxxi = i\n",
" return ' '.join(sen[:maxxi + window_size]) + ' $' + inject + '$ ' + ' '.join(sen[maxxi + window_size:])\n", " temp = ' '.join(sen_en[:maxxi + window_size]) + ' $' + inject + '$ ' + ' '.join(sen_en[maxxi + window_size:])\n",
" return temp\n",
"\n", "\n",
"glossary['source_lem'] = [' ' + str(default_process(x)) + ' ' for x in glossary['source_lem']]\n", "glossary['source_lem'] = [str(default_process(x)) for x in glossary['source_lem']]\n",
"file_pl = pd.read_csv(train_expected_path, sep='\\t', header=None, names=['text'])\n",
"file_pl['text'] = [default_process(text) for text in file_pl['text'].values.tolist()]\n",
"file_en= pd.read_csv(train_in_path, sep='\\t', header=None, names=['text'])\n",
"file_en['text'] = [default_process(text) for text in file_en['text'].values.tolist()]\n",
"\n", "\n",
"start_time = time.time_ns()\n", "start_time = time.time_ns()\n",
"en = []\n", "en = []\n",
"translation_line_counts = []\n", "translation_line_counts = []\n",
"for line, line_pl in zip(file_lemmatized, file_pl['text'].values.tolist()):\n", "for line, line_en, line_pl in zip(file_lemmatized, file_en['text'].values.tolist(), file_pl['text'].values.tolist()):\n",
" line = default_process(line)\n", " line = default_process(line)\n",
" line_pl = default_process(line_pl)\n",
" matchez = rapidfuzz.process.extract(query=line, choices=glossary['source_lem'], limit=5, score_cutoff=THRESHOLD, scorer=partial_ratio)\n", " matchez = rapidfuzz.process.extract(query=line, choices=glossary['source_lem'], limit=5, score_cutoff=THRESHOLD, scorer=partial_ratio)\n",
" if len(matchez) > 0:\n", " if len(matchez) > 0:\n",
" lines_added = 0\n", " lines_added = 0\n",
" for match in matchez:\n", " for match in matchez:\n",
" polish_translation = glossary.loc[lambda df: df['source_lem'] == match[0]]['result'].astype(str).values.flatten()[0]\n", " polish_translation = glossary.loc[lambda df: df['source_lem'] == match[0]]['result'].astype(str).values.flatten()[0]\n",
" if is_injectable(line_pl, polish_translation):\n", " if is_injectable(line_pl, polish_translation):\n",
" en.append(get_injected(line, match[0], polish_translation))\n", " en.append(get_injected(line, line_en, match[0], polish_translation))\n",
" lines_added += 1\n", " lines_added += 1\n",
" if lines_added == 0:\n", " if lines_added == 0:\n",
" en.append(line)\n", " en.append(line_en)\n",
" lines_added = 1\n", " lines_added = 1\n",
" translation_line_counts.append(lines_added)\n", " translation_line_counts.append(lines_added)\n",
" else:\n", " else:\n",
" translation_line_counts.append(1)\n", " translation_line_counts.append(1)\n",
" en.append(line)\n", " en.append(line_en)\n",
"\n", "\n",
"\n", "\n",
"stop = time.time_ns()\n", "stop = time.time_ns()\n",
@ -160,20 +162,19 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 46, "execution_count": 65,
"outputs": [], "outputs": [],
"source": [ "source": [
"\n", "\n",
"translations = pd.read_csv(train_expected_path, sep='\\t', header=0, names=['text'])\n", "with open(train_expected_path + '.injected', 'w') as file_pl_write:\n",
"with open(train_expected_path + '.injected', 'w') as file_plx:\n", " for line, translation_line_ct in zip(file_pl['text'].values.tolist(), translation_line_counts):\n",
" for line, translation_line_ct in zip(translations['text'].values.tolist(), translation_line_counts):\n",
" for i in range(translation_line_ct):\n", " for i in range(translation_line_ct):\n",
" file_plx.write(line + '\\n')\n", " file_pl_write.write(line + '\\n')\n",
"\n", "\n",
"\n", "\n",
"with open(train_in_path + '.injected', 'w') as file_en:\n", "with open(train_in_path + '.injected', 'w') as file_en_write:\n",
" for e in en:\n", " for e in en:\n",
" file_en.write(e + '\\n')" " file_en_write.write(e + '\\n')"
], ],
"metadata": { "metadata": {
"collapsed": false, "collapsed": false,