From 716d7b707247fb422e9d89dbaf094c9dbf0e9e3e Mon Sep 17 00:00:00 2001
From: jakubknczny <jakubknczny@gmail.com>
Date: Sat, 22 Jan 2022 01:50:09 +0100
Subject: [PATCH] inject in non-lemmatized

---
 rapidfuzztest.ipynb | 45 +++++++++++++++++++++++----------------------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/rapidfuzztest.ipynb b/rapidfuzztest.ipynb
index 6ab8edf..4e98ef0 100644
--- a/rapidfuzztest.ipynb
+++ b/rapidfuzztest.ipynb
@@ -6,7 +6,6 @@
    "outputs": [],
    "source": [
     "\n",
-    "import copy\n",
     "import nltk\n",
     "import pandas as pd\n",
     "import rapidfuzz\n",
@@ -59,8 +58,6 @@
     "train_expected_path = 'mt-summit-corpora/dev-0/expected.tsv'\n",
     "\n",
     "\n",
-    "file_pl = pd.read_csv(train_expected_path, sep='\\t', header=None, names=['text'])\n",
-    "\n",
     "start_time = time.time_ns()\n",
     "file_lemmatized = []\n",
     "with open(train_in_path, 'r') as file:\n",
@@ -83,25 +80,25 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 45,
+   "execution_count": 64,
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "6.904260614\n"
+      "6.915366953\n"
      ]
     }
    ],
    "source": [
     "\n",
-    "THRESHOLD = 88\n",
+    "THRESHOLD = 70\n",
     "\n",
     "def is_injectable(sentence_pl, sequence):\n",
     "    sen = sentence_pl.split()\n",
     "    window_size = len(sequence.split())\n",
     "    maxx = 0\n",
-    "    for i in range(len(sen) - window_size):\n",
+    "    for i in range(len(sen) - window_size + 1):\n",
     "        current = rapidfuzz.fuzz.ratio(' '.join(sen[i:i + window_size]), sequence)\n",
     "        if current > maxx:\n",
     "            maxx = current\n",
@@ -110,8 +107,9 @@
     "    else:\n",
     "        return False\n",
     "\n",
-    "def get_injected(sentence, sequence, inject):\n",
+    "def get_injected(sentence, sentence_en, sequence, inject):\n",
     "    sen = sentence.split()\n",
+    "    sen_en = sentence_en.split()\n",
     "    window_size = len(sequence.split())\n",
     "    maxx = 0\n",
     "    maxxi = 0\n",
@@ -120,31 +118,35 @@
     "        if current >= maxx:\n",
     "            maxx = current\n",
     "            maxxi = i\n",
-    "    return ' '.join(sen[:maxxi + window_size]) + ' $' + inject + '$ ' + ' '.join(sen[maxxi + window_size:])\n",
+    "    temp = ' '.join(sen_en[:maxxi + window_size]) + ' $' + inject + '$ ' + ' '.join(sen_en[maxxi + window_size:])\n",
+    "    return temp\n",
     "\n",
-    "glossary['source_lem'] = [' ' + str(default_process(x)) + ' ' for x in glossary['source_lem']]\n",
+    "glossary['source_lem'] = [str(default_process(x)) for x in glossary['source_lem']]\n",
+    "file_pl = pd.read_csv(train_expected_path, sep='\\t', header=None, names=['text'])\n",
+    "file_pl['text'] = [default_process(text) for text in file_pl['text'].values.tolist()]\n",
+    "file_en= pd.read_csv(train_in_path, sep='\\t', header=None, names=['text'])\n",
+    "file_en['text'] = [default_process(text) for text in file_en['text'].values.tolist()]\n",
     "\n",
     "start_time = time.time_ns()\n",
     "en = []\n",
     "translation_line_counts = []\n",
-    "for line, line_pl in zip(file_lemmatized, file_pl['text'].values.tolist()):\n",
+    "for line, line_en, line_pl in zip(file_lemmatized, file_en['text'].values.tolist(), file_pl['text'].values.tolist()):\n",
     "    line = default_process(line)\n",
-    "    line_pl = default_process(line_pl)\n",
     "    matchez = rapidfuzz.process.extract(query=line, choices=glossary['source_lem'], limit=5, score_cutoff=THRESHOLD, scorer=partial_ratio)\n",
     "    if len(matchez) > 0:\n",
     "        lines_added = 0\n",
     "        for match in matchez:\n",
     "            polish_translation = glossary.loc[lambda df: df['source_lem'] == match[0]]['result'].astype(str).values.flatten()[0]\n",
     "            if is_injectable(line_pl, polish_translation):\n",
-    "                en.append(get_injected(line, match[0], polish_translation))\n",
+    "                en.append(get_injected(line, line_en, match[0], polish_translation))\n",
     "                lines_added += 1\n",
     "        if lines_added == 0:\n",
-    "            en.append(line)\n",
+    "            en.append(line_en)\n",
     "            lines_added = 1\n",
     "        translation_line_counts.append(lines_added)\n",
     "    else:\n",
     "        translation_line_counts.append(1)\n",
-    "        en.append(line)\n",
+    "        en.append(line_en)\n",
     "\n",
     "\n",
     "stop = time.time_ns()\n",
@@ -160,20 +162,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 46,
+   "execution_count": 65,
    "outputs": [],
    "source": [
     "\n",
-    "translations = pd.read_csv(train_expected_path, sep='\\t', header=0, names=['text'])\n",
-    "with open(train_expected_path + '.injected', 'w') as file_plx:\n",
-    "    for line, translation_line_ct in zip(translations['text'].values.tolist(), translation_line_counts):\n",
+    "with open(train_expected_path + '.injected', 'w') as file_pl_write:\n",
+    "    for line, translation_line_ct in zip(file_pl['text'].values.tolist(), translation_line_counts):\n",
     "        for i in range(translation_line_ct):\n",
-    "            file_plx.write(line + '\\n')\n",
+    "            file_pl_write.write(line + '\\n')\n",
     "\n",
     "\n",
-    "with open(train_in_path + '.injected', 'w') as file_en:\n",
+    "with open(train_in_path + '.injected', 'w') as file_en_write:\n",
     "    for e in en:\n",
-    "        file_en.write(e + '\\n')"
+    "        file_en_write.write(e + '\\n')"
    ],
    "metadata": {
     "collapsed": false,