diff --git a/rapidfuzztest.ipynb b/rapidfuzztest.ipynb index 6ab8edf..4e98ef0 100644 --- a/rapidfuzztest.ipynb +++ b/rapidfuzztest.ipynb @@ -6,7 +6,6 @@ "outputs": [], "source": [ "\n", - "import copy\n", "import nltk\n", "import pandas as pd\n", "import rapidfuzz\n", @@ -59,8 +58,6 @@ "train_expected_path = 'mt-summit-corpora/dev-0/expected.tsv'\n", "\n", "\n", - "file_pl = pd.read_csv(train_expected_path, sep='\\t', header=None, names=['text'])\n", - "\n", "start_time = time.time_ns()\n", "file_lemmatized = []\n", "with open(train_in_path, 'r') as file:\n", @@ -83,25 +80,25 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 64, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "6.904260614\n" + "6.915366953\n" ] } ], "source": [ "\n", - "THRESHOLD = 88\n", + "THRESHOLD = 70\n", "\n", "def is_injectable(sentence_pl, sequence):\n", " sen = sentence_pl.split()\n", " window_size = len(sequence.split())\n", " maxx = 0\n", - " for i in range(len(sen) - window_size):\n", + " for i in range(len(sen) - window_size + 1):\n", " current = rapidfuzz.fuzz.ratio(' '.join(sen[i:i + window_size]), sequence)\n", " if current > maxx:\n", " maxx = current\n", @@ -110,8 +107,9 @@ " else:\n", " return False\n", "\n", - "def get_injected(sentence, sequence, inject):\n", + "def get_injected(sentence, sentence_en, sequence, inject):\n", " sen = sentence.split()\n", + " sen_en = sentence_en.split()\n", " window_size = len(sequence.split())\n", " maxx = 0\n", " maxxi = 0\n", @@ -120,31 +118,35 @@ " if current >= maxx:\n", " maxx = current\n", " maxxi = i\n", - " return ' '.join(sen[:maxxi + window_size]) + ' $' + inject + '$ ' + ' '.join(sen[maxxi + window_size:])\n", + " temp = ' '.join(sen_en[:maxxi + window_size]) + ' $' + inject + '$ ' + ' '.join(sen_en[maxxi + window_size:])\n", + " return temp\n", "\n", - "glossary['source_lem'] = [' ' + str(default_process(x)) + ' ' for x in glossary['source_lem']]\n", + "glossary['source_lem'] = [str(default_process(x)) for x in glossary['source_lem']]\n", + "file_pl = pd.read_csv(train_expected_path, sep='\\t', header=None, names=['text'])\n", + "file_pl['text'] = [default_process(text) for text in file_pl['text'].values.tolist()]\n", + "file_en= pd.read_csv(train_in_path, sep='\\t', header=None, names=['text'])\n", + "file_en['text'] = [default_process(text) for text in file_en['text'].values.tolist()]\n", "\n", "start_time = time.time_ns()\n", "en = []\n", "translation_line_counts = []\n", - "for line, line_pl in zip(file_lemmatized, file_pl['text'].values.tolist()):\n", + "for line, line_en, line_pl in zip(file_lemmatized, file_en['text'].values.tolist(), file_pl['text'].values.tolist()):\n", " line = default_process(line)\n", - " line_pl = default_process(line_pl)\n", " matchez = rapidfuzz.process.extract(query=line, choices=glossary['source_lem'], limit=5, score_cutoff=THRESHOLD, scorer=partial_ratio)\n", " if len(matchez) > 0:\n", " lines_added = 0\n", " for match in matchez:\n", " polish_translation = glossary.loc[lambda df: df['source_lem'] == match[0]]['result'].astype(str).values.flatten()[0]\n", " if is_injectable(line_pl, polish_translation):\n", - " en.append(get_injected(line, match[0], polish_translation))\n", + " en.append(get_injected(line, line_en, match[0], polish_translation))\n", " lines_added += 1\n", " if lines_added == 0:\n", - " en.append(line)\n", + " en.append(line_en)\n", " lines_added = 1\n", " translation_line_counts.append(lines_added)\n", " else:\n", " translation_line_counts.append(1)\n", - " en.append(line)\n", + " en.append(line_en)\n", "\n", "\n", "stop = time.time_ns()\n", @@ -160,20 +162,19 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 65, "outputs": [], "source": [ "\n", - "translations = pd.read_csv(train_expected_path, sep='\\t', header=0, names=['text'])\n", - "with open(train_expected_path + '.injected', 'w') as file_plx:\n", - " for line, translation_line_ct in zip(translations['text'].values.tolist(), translation_line_counts):\n", + "with open(train_expected_path + '.injected', 'w') as file_pl_write:\n", + " for line, translation_line_ct in zip(file_pl['text'].values.tolist(), translation_line_counts):\n", " for i in range(translation_line_ct):\n", - " file_plx.write(line + '\\n')\n", + " file_pl_write.write(line + '\\n')\n", "\n", "\n", - "with open(train_in_path + '.injected', 'w') as file_en:\n", + "with open(train_in_path + '.injected', 'w') as file_en_write:\n", " for e in en:\n", - " file_en.write(e + '\\n')" + " file_en_write.write(e + '\\n')" ], "metadata": { "collapsed": false,