inject to lemmatized with pl crosscheck

.gitignore

@@ -1 +1,3 @@
 mt-summit-corpora
+.idea
+kompendium_lem.tsv

jupyter-injector.ipynb

@@ -0,0 +1,388 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "source": [
+    "## Lemmatize glossary\n",
+    "TODO: train test split glossary"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "outputs": [
+    {
+     "data": {
+      "text/plain": "                                    source  \\\nsource_lem                                   \naaofi                                aaofi   \naca                                    aca   \nacca                                  acca   \nabacus                              abacus   \nabandonment cost         abandonment costs   \n...                                    ...   \nytd                                    ytd   \nyear-end                          year-end   \nyear-to-date                  year-to-date   \nzog                                    zog   \nzero overhead growth  zero overhead growth   \n\n                                                                 result  \\\nsource_lem                                                                \naaofi                 organizacja rachunkowości i audytu dla islamsk...   \naca                   członek stowarzyszenia dyplomowanych biegłych ...   \nacca                   stowarzyszenie dyplomowanych biegłych rewidentów   \nabacus                                                         liczydło   \nabandonment cost                                     koszty zaniechania   \n...                                                                 ...   \nytd                                                    od początku roku   \nyear-end                                                    koniec roku   \nyear-to-date                                           od początku roku   \nzog                                       zero wzrostu kosztów ogólnych   \nzero overhead growth                      zero wzrostu kosztów ogólnych   \n\n                                                             result_lem  \nsource_lem                                                               \naaofi                 organizacja rachunkowość i audyt dla islamski ...  \naca                   członek stowarzyszenie dyplomowany biegły rewi...  \nacca                         stowarzyszenie dyplomowany biegły rewident  \nabacus                                                         liczydło  \nabandonment cost                                      koszt zaniechanie  \n...                                                                 ...  \nytd                                                     od początek rok  \nyear-end                                                     koniec rok  \nyear-to-date                                            od początek rok  \nzog                                            zero wzrost koszt ogólny  \nzero overhead growth                           zero wzrost koszt ogólny  \n\n[1197 rows x 3 columns]",
+      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>source</th>\n      <th>result</th>\n      <th>result_lem</th>\n    </tr>\n    <tr>\n      <th>source_lem</th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>aaofi</th>\n      <td>aaofi</td>\n      <td>organizacja rachunkowości i audytu dla islamsk...</td>\n      <td>organizacja rachunkowość i audyt dla islamski ...</td>\n    </tr>\n    <tr>\n      <th>aca</th>\n      <td>aca</td>\n      <td>członek stowarzyszenia dyplomowanych biegłych ...</td>\n      <td>członek stowarzyszenie dyplomowany biegły rewi...</td>\n    </tr>\n    <tr>\n      <th>acca</th>\n      <td>acca</td>\n      <td>stowarzyszenie dyplomowanych biegłych rewidentów</td>\n      <td>stowarzyszenie dyplomowany biegły rewident</td>\n    </tr>\n    <tr>\n      <th>abacus</th>\n      <td>abacus</td>\n      <td>liczydło</td>\n      <td>liczydło</td>\n    </tr>\n    <tr>\n      <th>abandonment cost</th>\n      <td>abandonment costs</td>\n      <td>koszty zaniechania</td>\n      <td>koszt zaniechanie</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>ytd</th>\n      <td>ytd</td>\n      <td>od początku roku</td>\n      <td>od początek rok</td>\n    </tr>\n    <tr>\n      <th>year-end</th>\n      <td>year-end</td>\n      <td>koniec roku</td>\n      <td>koniec rok</td>\n    </tr>\n    <tr>\n      <th>year-to-date</th>\n      <td>year-to-date</td>\n      <td>od początku roku</td>\n      <td>od początek rok</td>\n    </tr>\n    <tr>\n      <th>zog</th>\n      <td>zog</td>\n      <td>zero wzrostu kosztów ogólnych</td>\n      <td>zero wzrost koszt ogólny</td>\n    </tr>\n    <tr>\n      <th>zero overhead growth</th>\n      <td>zero overhead growth</td>\n      <td>zero wzrostu kosztów ogólnych</td>\n      <td>zero wzrost koszt ogólny</td>\n    </tr>\n  </tbody>\n</table>\n<p>1197 rows × 3 columns</p>\n</div>"
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "import spacy\n",
+    "\n",
+    "\n",
+    "spacy_nlp_en = spacy.load('en_core_web_sm')\n",
+    "spacy_nlp_pl = spacy.load(\"pl_core_news_sm\")\n",
+    "\n",
+    "glossary = pd.read_csv('kompendium.tsv', sep='\\t', header=None, names=['source', 'result'])\n",
+    "\n",
+    "source_lemmatized = []\n",
+    "for word in glossary['source']:\n",
+    "    temp = []\n",
+    "    for token in spacy_nlp_en(word):\n",
+    "        temp.append(token.lemma_)\n",
+    "    source_lemmatized.append(' '.join(temp).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))\n",
+    "\n",
+    "result_lemmatized = []\n",
+    "for word in glossary['result']:\n",
+    "    temp = []\n",
+    "    for token in spacy_nlp_pl(word):\n",
+    "        temp.append(token.lemma_)\n",
+    "    result_lemmatized.append(' '.join(temp).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))\n",
+    "\n",
+    "glossary['source_lem'] = source_lemmatized\n",
+    "glossary['result_lem'] = result_lemmatized\n",
+    "glossary = glossary[['source', 'source_lem', 'result', 'result_lem']]\n",
+    "glossary.set_index('source_lem')\n"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "outputs": [],
+   "source": [
+    "glossary.to_csv('kompendium_lem.tsv', sep='\\t')"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "## Lemmatize corpus"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "outputs": [],
+   "source": [
+    "dev_path = 'mt-summit-corpora/dev/dev'\n",
+    "\n",
+    "skip_chars = ''',./!?'''\n",
+    "\n",
+    "with open(dev_path + '.en', 'r') as file:\n",
+    "    file_lemmatized = []\n",
+    "    for line in file:\n",
+    "        temp = []\n",
+    "        for token in spacy_nlp_en(line):\n",
+    "            temp.append(token.lemma_)\n",
+    "        file_lemmatized.append(' '.join([x for x in temp if x not in skip_chars]).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))\n",
+    "\n",
+    "with open(dev_path + '.pl', 'r') as file:\n",
+    "    file_pl_lemmatized = []\n",
+    "    for line in file:\n",
+    "        temp = []\n",
+    "        for token in spacy_nlp_pl(line):\n",
+    "            temp.append(token.lemma_)\n",
+    "        file_pl_lemmatized.append(' '.join([x for x in temp if x not in skip_chars]).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))\n",
+    "\n"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "in the course of the control the control audit firm shall fulfil the responsibility refer to in article 114 on date and in form specify by the controller \n",
+      "\n",
+      "w czas trwanie kontrola kontrolowany firma audytorski wypełnia obowiązek o których mowa w art 114 w ter-mina i forma wskazany przez osoba kontrolującą \n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(file_lemmatized[2])\n",
+    "print(file_pl_lemmatized[2])"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "## Inject glossary\n",
+    "# !!! Obsolete !!!"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "outputs": [
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
+      "\u001B[0;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
+      "\u001B[0;32m/tmp/ipykernel_1418662/149035253.py\u001B[0m in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[1;32m     18\u001B[0m \u001B[0;32mfor\u001B[0m \u001B[0mline_id\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mline\u001B[0m \u001B[0;32min\u001B[0m \u001B[0menumerate\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mfile_lemmatized\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     19\u001B[0m     \u001B[0mdoc\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mnlp\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mline\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 20\u001B[0;31m     \u001B[0mmatches\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mmatcher\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mdoc\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m     21\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     22\u001B[0m     \u001B[0mline_counter\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;36m0\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
+      "\u001B[0;32m~/Workspace/Envs/trainMT/lib/python3.8/site-packages/spaczz/matcher/_phrasematcher.py\u001B[0m in \u001B[0;36m__call__\u001B[0;34m(self, doc)\u001B[0m\n\u001B[1;32m     95\u001B[0m                 \u001B[0;32mif\u001B[0m \u001B[0;32mnot\u001B[0m \u001B[0mkwargs\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     96\u001B[0m                     \u001B[0mkwargs\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mdefaults\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 97\u001B[0;31m                 \u001B[0mmatches_wo_label\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_searcher\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mmatch\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mdoc\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mpattern\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m     98\u001B[0m                 \u001B[0;32mif\u001B[0m \u001B[0mmatches_wo_label\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     99\u001B[0m                     matches_w_label = [\n",
+      "\u001B[0;32m~/Workspace/Envs/trainMT/lib/python3.8/site-packages/spaczz/search/_phrasesearcher.py\u001B[0m in \u001B[0;36mmatch\u001B[0;34m(self, doc, query, flex, min_r1, min_r2, thresh, *args, **kwargs)\u001B[0m\n\u001B[1;32m    137\u001B[0m         \u001B[0mflex\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_calc_flex\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mquery\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mflex\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    138\u001B[0m         \u001B[0mmin_r1\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmin_r2\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mthresh\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_check_ratios\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmin_r1\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmin_r2\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mthresh\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mflex\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 139\u001B[0;31m         \u001B[0mmatch_values\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_scan\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mdoc\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mquery\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmin_r1\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    140\u001B[0m         \u001B[0;32mif\u001B[0m \u001B[0mmatch_values\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    141\u001B[0m             \u001B[0mpositions\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mlist\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmatch_values\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mkeys\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
+      "\u001B[0;32m~/Workspace/Envs/trainMT/lib/python3.8/site-packages/spaczz/search/_phrasesearcher.py\u001B[0m in \u001B[0;36m_scan\u001B[0;34m(self, doc, query, min_r1, *args, **kwargs)\u001B[0m\n\u001B[1;32m    282\u001B[0m         \u001B[0mi\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;36m0\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    283\u001B[0m         \u001B[0;32mwhile\u001B[0m \u001B[0mi\u001B[0m \u001B[0;34m+\u001B[0m \u001B[0mlen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mquery\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;34m<=\u001B[0m \u001B[0mlen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mdoc\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 284\u001B[0;31m             \u001B[0mmatch\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mcompare\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mquery\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mdoc\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mi\u001B[0m \u001B[0;34m:\u001B[0m \u001B[0mi\u001B[0m \u001B[0;34m+\u001B[0m \u001B[0mlen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mquery\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    285\u001B[0m             \u001B[0;32mif\u001B[0m \u001B[0mmatch\u001B[0m \u001B[0;34m>=\u001B[0m \u001B[0mmin_r1\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    286\u001B[0m                 \u001B[0mmatch_values\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mi\u001B[0m\u001B[0;34m]\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mmatch\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
+      "\u001B[0;32m~/Workspace/Envs/trainMT/lib/python3.8/site-packages/spacy/tokens/doc.pyx\u001B[0m in \u001B[0;36mspacy.tokens.doc.Doc.__getitem__\u001B[0;34m()\u001B[0m\n",
+      "\u001B[0;32m~/Workspace/Envs/trainMT/lib/python3.8/site-packages/spacy/util.py\u001B[0m in \u001B[0;36mnormalize_slice\u001B[0;34m(length, start, stop, step)\u001B[0m\n\u001B[1;32m   1199\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1200\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 1201\u001B[0;31m def normalize_slice(\n\u001B[0m\u001B[1;32m   1202\u001B[0m     \u001B[0mlength\u001B[0m\u001B[0;34m:\u001B[0m \u001B[0mint\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mstart\u001B[0m\u001B[0;34m:\u001B[0m \u001B[0mint\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mstop\u001B[0m\u001B[0;34m:\u001B[0m \u001B[0mint\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mstep\u001B[0m\u001B[0;34m:\u001B[0m \u001B[0mOptional\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mint\u001B[0m\u001B[0;34m]\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;32mNone\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1203\u001B[0m ) -> Tuple[int, int]:\n",
+      "\u001B[0;31mKeyboardInterrupt\u001B[0m: "
+     ]
+    }
+   ],
+   "source": [
+    "import spacy\n",
+    "from spaczz.matcher import FuzzyMatcher\n",
+    "\n",
+    "\n",
+    "glossary = pd.read_csv('kompendium_lem.tsv', sep='\\t', header=0, index_col=0)\n",
+    "bad_words = ['ocf', 'toc', 'vas', 'vat']\n",
+    "train_glossary = glossary.iloc[[x for x in range(len(glossary)) if x % 6 != 0]]\n",
+    "\n",
+    "nlp = spacy.blank(\"en\")\n",
+    "matcher = FuzzyMatcher(nlp.vocab)\n",
+    "for word in train_glossary['source_lem']:\n",
+    "    if word not in bad_words:\n",
+    "        matcher.add(word, [nlp(word)])\n",
+    "\n",
+    "\n",
+    "en = []\n",
+    "translation_line_counts = []\n",
+    "for line_id, line in enumerate(file_lemmatized):\n",
+    "    doc = nlp(line)\n",
+    "    matches = matcher(doc)\n",
+    "\n",
+    "    line_counter = 0\n",
+    "    for match_id, start, end, ratio in matches:\n",
+    "        if ratio > 90:\n",
+    "            line_counter += 1\n",
+    "            en.append(''.join(doc[:end].text + ' ' + train_glossary.loc[lambda df: df['source_lem'] == match_id]['result'].astype(str).values.flatten() + ' ' + doc[end:].text))\n",
+    "\n",
+    "\n",
+    "    if line_counter == 0:\n",
+    "        line_counter = 1\n",
+    "        en.append(line)\n",
+    "    translation_line_counts.append(line_counter)\n",
+    "\n"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "outputs": [],
+   "source": [
+    "import copy\n",
+    "tlcs = copy.deepcopy(translation_line_counts)\n",
+    "\n",
+    "translations = pd.read_csv(dev_path + '.pl', sep='\\t', header=None, names=['text'])\n",
+    "with open(dev_path + '.injected.pl', 'w') as file_pl:\n",
+    "    for trans in translations.iterrows():\n",
+    "        try:\n",
+    "            for _ in range(tlcs.pop(0)):\n",
+    "                file_pl.write(trans[1]['text'] + '\\n')\n",
+    "        except:\n",
+    "            pass\n",
+    "\n",
+    "\n",
+    "with open(dev_path + '.injected.en', 'w') as file_en:\n",
+    "    for line in en:\n",
+    "        file_en.write(line)\n"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "## Inject glossary Polish crosscheck"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "outputs": [],
+   "source": [
+    "import spacy\n",
+    "from spaczz.matcher import FuzzyMatcher\n",
+    "\n",
+    "# glossary\n",
+    "glossary = pd.read_csv('kompendium_lem.tsv', sep='\\t', header=0, index_col=0)\n",
+    "train_glossary = glossary.iloc[[x for x in range(len(glossary)) if x % 6 != 0]]\n",
+    "\n",
+    "# add rules to English matcher\n",
+    "nlp = spacy.blank(\"en\")\n",
+    "matcher = FuzzyMatcher(nlp.vocab)\n",
+    "for word in train_glossary['source_lem']:\n",
+    "    matcher.add(word, [nlp(word)])\n",
+    "\n",
+    "# add rules to Polish matcher\n",
+    "nlp_pl = spacy.blank(\"pl\")\n",
+    "matcher_pl = FuzzyMatcher(nlp_pl.vocab)\n",
+    "for word, word_id in zip(train_glossary['result_lem'], train_glossary['source_lem']):\n",
+    "    matcher_pl.add(word, [nlp_pl(word)])\n",
+    "\n",
+    "# todo\n",
+    "en = []\n",
+    "translation_line_counts = []\n",
+    "for line_id in range(len(file_lemmatized)):\n",
+    "\n",
+    "    doc = nlp(file_lemmatized[line_id])\n",
+    "    matches = matcher(doc)\n",
+    "\n",
+    "    line_counter = 0\n",
+    "    for match_id, start, end, ratio in matches:\n",
+    "        if ratio > 90:\n",
+    "            doc_pl = nlp_pl(file_pl_lemmatized[line_id])\n",
+    "            matches_pl = matcher_pl(doc_pl)\n",
+    "\n",
+    "            for match_id_pl, start_pl, end_pl, ratio_pl in matches_pl:\n",
+    "                if match_id_pl == glossary[glossary['source_lem'] == match_id].values[0][3]:\n",
+    "                    line_counter += 1\n",
+    "                    en.append(''.join(doc[:end].text + ' ' + train_glossary.loc[lambda df: df['source_lem'] == match_id]['result'].astype(str).values.flatten() + ' ' + doc[end:].text))\n",
+    "\n",
+    "    if line_counter == 0:\n",
+    "        line_counter = 1\n",
+    "        en.append(file_lemmatized[line_id])\n",
+    "    translation_line_counts.append(line_counter)\n"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "outputs": [],
+   "source": [
+    "import copy\n",
+    "\n",
+    "\n",
+    "tlcs = copy.deepcopy(translation_line_counts)\n",
+    "\n",
+    "translations = pd.read_csv(dev_path + '.pl', sep='\\t', header=None, names=['text'])\n",
+    "translations['id'] = [x for x in range(len(translations))]\n",
+    "\n",
+    "ctr = 0\n",
+    "sentence = ''\n",
+    "with open(dev_path + '.injected.crossvalidated.en', 'w') as file_en:\n",
+    "    with open(dev_path + '.injected.crossvalidated.pl', 'w') as file_pl:\n",
+    "        for i in range(len(en)):\n",
+    "            if i > 0:\n",
+    "                if en[i-1] != en[i]:\n",
+    "                    if ctr == 0:\n",
+    "                        sentence = translations.iloc[0]\n",
+    "                        translations.drop(sentence['id'], inplace=True)\n",
+    "                        sentence = sentence['text']\n",
+    "                        try:\n",
+    "                            ctr = tlcs.pop(0)\n",
+    "                        except:\n",
+    "                            pass\n",
+    "                    file_en.write(en[i])\n",
+    "                    file_pl.write(sentence + '\\n')\n",
+    "                ctr = ctr - 1\n",
+    "            else:\n",
+    "                try:\n",
+    "                    ctr = tlcs.pop(0) - 1\n",
+    "                except:\n",
+    "                    pass\n",
+    "                sentence = translations.iloc[0]\n",
+    "                translations.drop(sentence['id'], inplace=True)\n",
+    "                sentence = sentence['text']\n",
+    "                file_en.write(en[i])\n",
+    "                file_pl.write(sentence + '\\n')"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}

training-command.txt

@@ -0,0 +1,18 @@
+first iteration:
+./marian/build/marian --model mt.npz --type transformer --overwrite \
+--train-sets mt-summit-corpora/mt-summit-corpora/dev/dev.en \
+mt-summit-corpora/mt-summit-corpora/dev/dev.pl \
+--disp-freq 1000 --save-freq 1000 --optimizer adam --lr-report
+
+next iterations:
+./marian/build/marian --model mt.npz --type transformer --overwrite \
+--train-sets mt-summit-corpora/mt-summit-corpora/dev/dev.en \
+mt-summit-corpora/mt-summit-corpora/dev/dev.pl \
+--disp-freq 1000 --save-freq 1000 --optimizer adam --lr-report \
+--pretrained-model mt.npz
+
+./marian/build/marian --model mt.npz --type transformer --overwrite \
+--train-sets mt-summit-corpora/mt-summit-corpora/train/train.en \
+mt-summit-corpora/mt-summit-corpora/train/train.pl \
+--disp-freq 1000 --save-freq 10000 --optimizer adam --lr-report \
+--pretrained-model mt.npz