inject to lemmatized with pl crosscheck
This commit is contained in:
parent
19486adbb5
commit
cb4a413644
2
.gitignore
vendored
2
.gitignore
vendored
@ -1 +1,3 @@
|
||||
mt-summit-corpora
|
||||
.idea
|
||||
kompendium_lem.tsv
|
||||
388
jupyter-injector.ipynb
Normal file
388
jupyter-injector.ipynb
Normal file
@ -0,0 +1,388 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Lemmatize glossary\n",
|
||||
"TODO: train test split glossary"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%% md\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": " source \\\nsource_lem \naaofi aaofi \naca aca \nacca acca \nabacus abacus \nabandonment cost abandonment costs \n... ... \nytd ytd \nyear-end year-end \nyear-to-date year-to-date \nzog zog \nzero overhead growth zero overhead growth \n\n result \\\nsource_lem \naaofi organizacja rachunkowości i audytu dla islamsk... \naca członek stowarzyszenia dyplomowanych biegłych ... \nacca stowarzyszenie dyplomowanych biegłych rewidentów \nabacus liczydło \nabandonment cost koszty zaniechania \n... ... \nytd od początku roku \nyear-end koniec roku \nyear-to-date od początku roku \nzog zero wzrostu kosztów ogólnych \nzero overhead growth zero wzrostu kosztów ogólnych \n\n result_lem \nsource_lem \naaofi organizacja rachunkowość i audyt dla islamski ... \naca członek stowarzyszenie dyplomowany biegły rewi... \nacca stowarzyszenie dyplomowany biegły rewident \nabacus liczydło \nabandonment cost koszt zaniechanie \n... ... \nytd od początek rok \nyear-end koniec rok \nyear-to-date od początek rok \nzog zero wzrost koszt ogólny \nzero overhead growth zero wzrost koszt ogólny \n\n[1197 rows x 3 columns]",
|
||||
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>source</th>\n <th>result</th>\n <th>result_lem</th>\n </tr>\n <tr>\n <th>source_lem</th>\n <th></th>\n <th></th>\n <th></th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>aaofi</th>\n <td>aaofi</td>\n <td>organizacja rachunkowości i audytu dla islamsk...</td>\n <td>organizacja rachunkowość i audyt dla islamski ...</td>\n </tr>\n <tr>\n <th>aca</th>\n <td>aca</td>\n <td>członek stowarzyszenia dyplomowanych biegłych ...</td>\n <td>członek stowarzyszenie dyplomowany biegły rewi...</td>\n </tr>\n <tr>\n <th>acca</th>\n <td>acca</td>\n <td>stowarzyszenie dyplomowanych biegłych rewidentów</td>\n <td>stowarzyszenie dyplomowany biegły rewident</td>\n </tr>\n <tr>\n <th>abacus</th>\n <td>abacus</td>\n <td>liczydło</td>\n <td>liczydło</td>\n </tr>\n <tr>\n <th>abandonment cost</th>\n <td>abandonment costs</td>\n <td>koszty zaniechania</td>\n <td>koszt zaniechanie</td>\n </tr>\n <tr>\n <th>...</th>\n <td>...</td>\n <td>...</td>\n <td>...</td>\n </tr>\n <tr>\n <th>ytd</th>\n <td>ytd</td>\n <td>od początku roku</td>\n <td>od początek rok</td>\n </tr>\n <tr>\n <th>year-end</th>\n <td>year-end</td>\n <td>koniec roku</td>\n <td>koniec rok</td>\n </tr>\n <tr>\n <th>year-to-date</th>\n <td>year-to-date</td>\n <td>od początku roku</td>\n <td>od początek rok</td>\n </tr>\n <tr>\n <th>zog</th>\n <td>zog</td>\n <td>zero wzrostu kosztów ogólnych</td>\n <td>zero wzrost koszt ogólny</td>\n </tr>\n <tr>\n <th>zero overhead growth</th>\n <td>zero overhead growth</td>\n <td>zero wzrostu kosztów ogólnych</td>\n <td>zero wzrost koszt ogólny</td>\n </tr>\n </tbody>\n</table>\n<p>1197 rows × 3 columns</p>\n</div>"
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import spacy\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"spacy_nlp_en = spacy.load('en_core_web_sm')\n",
|
||||
"spacy_nlp_pl = spacy.load(\"pl_core_news_sm\")\n",
|
||||
"\n",
|
||||
"glossary = pd.read_csv('kompendium.tsv', sep='\\t', header=None, names=['source', 'result'])\n",
|
||||
"\n",
|
||||
"source_lemmatized = []\n",
|
||||
"for word in glossary['source']:\n",
|
||||
" temp = []\n",
|
||||
" for token in spacy_nlp_en(word):\n",
|
||||
" temp.append(token.lemma_)\n",
|
||||
" source_lemmatized.append(' '.join(temp).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))\n",
|
||||
"\n",
|
||||
"result_lemmatized = []\n",
|
||||
"for word in glossary['result']:\n",
|
||||
" temp = []\n",
|
||||
" for token in spacy_nlp_pl(word):\n",
|
||||
" temp.append(token.lemma_)\n",
|
||||
" result_lemmatized.append(' '.join(temp).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))\n",
|
||||
"\n",
|
||||
"glossary['source_lem'] = source_lemmatized\n",
|
||||
"glossary['result_lem'] = result_lemmatized\n",
|
||||
"glossary = glossary[['source', 'source_lem', 'result', 'result_lem']]\n",
|
||||
"glossary.set_index('source_lem')\n"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"glossary.to_csv('kompendium_lem.tsv', sep='\\t')"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Lemmatize corpus"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"dev_path = 'mt-summit-corpora/dev/dev'\n",
|
||||
"\n",
|
||||
"skip_chars = ''',./!?'''\n",
|
||||
"\n",
|
||||
"with open(dev_path + '.en', 'r') as file:\n",
|
||||
" file_lemmatized = []\n",
|
||||
" for line in file:\n",
|
||||
" temp = []\n",
|
||||
" for token in spacy_nlp_en(line):\n",
|
||||
" temp.append(token.lemma_)\n",
|
||||
" file_lemmatized.append(' '.join([x for x in temp if x not in skip_chars]).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))\n",
|
||||
"\n",
|
||||
"with open(dev_path + '.pl', 'r') as file:\n",
|
||||
" file_pl_lemmatized = []\n",
|
||||
" for line in file:\n",
|
||||
" temp = []\n",
|
||||
" for token in spacy_nlp_pl(line):\n",
|
||||
" temp.append(token.lemma_)\n",
|
||||
" file_pl_lemmatized.append(' '.join([x for x in temp if x not in skip_chars]).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))\n",
|
||||
"\n"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"in the course of the control the control audit firm shall fulfil the responsibility refer to in article 114 on date and in form specify by the controller \n",
|
||||
"\n",
|
||||
"w czas trwanie kontrola kontrolowany firma audytorski wypełnia obowiązek o których mowa w art 114 w ter-mina i forma wskazany przez osoba kontrolującą \n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(file_lemmatized[2])\n",
|
||||
"print(file_pl_lemmatized[2])"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Inject glossary\n",
|
||||
"# !!! Obsolete !!!"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%% md\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "KeyboardInterrupt",
|
||||
"evalue": "",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
|
||||
"\u001B[0;31mKeyboardInterrupt\u001B[0m Traceback (most recent call last)",
|
||||
"\u001B[0;32m/tmp/ipykernel_1418662/149035253.py\u001B[0m in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[1;32m 18\u001B[0m \u001B[0;32mfor\u001B[0m \u001B[0mline_id\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mline\u001B[0m \u001B[0;32min\u001B[0m \u001B[0menumerate\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mfile_lemmatized\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 19\u001B[0m \u001B[0mdoc\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mnlp\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mline\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 20\u001B[0;31m \u001B[0mmatches\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mmatcher\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mdoc\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 21\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 22\u001B[0m \u001B[0mline_counter\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;36m0\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
|
||||
"\u001B[0;32m~/Workspace/Envs/trainMT/lib/python3.8/site-packages/spaczz/matcher/_phrasematcher.py\u001B[0m in \u001B[0;36m__call__\u001B[0;34m(self, doc)\u001B[0m\n\u001B[1;32m 95\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0;32mnot\u001B[0m \u001B[0mkwargs\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 96\u001B[0m \u001B[0mkwargs\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mdefaults\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 97\u001B[0;31m \u001B[0mmatches_wo_label\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_searcher\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mmatch\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mdoc\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mpattern\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 98\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0mmatches_wo_label\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 99\u001B[0m matches_w_label = [\n",
|
||||
"\u001B[0;32m~/Workspace/Envs/trainMT/lib/python3.8/site-packages/spaczz/search/_phrasesearcher.py\u001B[0m in \u001B[0;36mmatch\u001B[0;34m(self, doc, query, flex, min_r1, min_r2, thresh, *args, **kwargs)\u001B[0m\n\u001B[1;32m 137\u001B[0m \u001B[0mflex\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_calc_flex\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mquery\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mflex\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 138\u001B[0m \u001B[0mmin_r1\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmin_r2\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mthresh\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_check_ratios\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmin_r1\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmin_r2\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mthresh\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mflex\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 139\u001B[0;31m \u001B[0mmatch_values\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_scan\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mdoc\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mquery\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmin_r1\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 140\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0mmatch_values\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 141\u001B[0m \u001B[0mpositions\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mlist\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmatch_values\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mkeys\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
|
||||
"\u001B[0;32m~/Workspace/Envs/trainMT/lib/python3.8/site-packages/spaczz/search/_phrasesearcher.py\u001B[0m in \u001B[0;36m_scan\u001B[0;34m(self, doc, query, min_r1, *args, **kwargs)\u001B[0m\n\u001B[1;32m 282\u001B[0m \u001B[0mi\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;36m0\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 283\u001B[0m \u001B[0;32mwhile\u001B[0m \u001B[0mi\u001B[0m \u001B[0;34m+\u001B[0m \u001B[0mlen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mquery\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;34m<=\u001B[0m \u001B[0mlen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mdoc\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 284\u001B[0;31m \u001B[0mmatch\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mcompare\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mquery\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mdoc\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mi\u001B[0m \u001B[0;34m:\u001B[0m \u001B[0mi\u001B[0m \u001B[0;34m+\u001B[0m \u001B[0mlen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mquery\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 285\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0mmatch\u001B[0m \u001B[0;34m>=\u001B[0m \u001B[0mmin_r1\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 286\u001B[0m \u001B[0mmatch_values\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mi\u001B[0m\u001B[0;34m]\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mmatch\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
|
||||
"\u001B[0;32m~/Workspace/Envs/trainMT/lib/python3.8/site-packages/spacy/tokens/doc.pyx\u001B[0m in \u001B[0;36mspacy.tokens.doc.Doc.__getitem__\u001B[0;34m()\u001B[0m\n",
|
||||
"\u001B[0;32m~/Workspace/Envs/trainMT/lib/python3.8/site-packages/spacy/util.py\u001B[0m in \u001B[0;36mnormalize_slice\u001B[0;34m(length, start, stop, step)\u001B[0m\n\u001B[1;32m 1199\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 1200\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 1201\u001B[0;31m def normalize_slice(\n\u001B[0m\u001B[1;32m 1202\u001B[0m \u001B[0mlength\u001B[0m\u001B[0;34m:\u001B[0m \u001B[0mint\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mstart\u001B[0m\u001B[0;34m:\u001B[0m \u001B[0mint\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mstop\u001B[0m\u001B[0;34m:\u001B[0m \u001B[0mint\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mstep\u001B[0m\u001B[0;34m:\u001B[0m \u001B[0mOptional\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mint\u001B[0m\u001B[0;34m]\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;32mNone\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 1203\u001B[0m ) -> Tuple[int, int]:\n",
|
||||
"\u001B[0;31mKeyboardInterrupt\u001B[0m: "
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import spacy\n",
|
||||
"from spaczz.matcher import FuzzyMatcher\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"glossary = pd.read_csv('kompendium_lem.tsv', sep='\\t', header=0, index_col=0)\n",
|
||||
"bad_words = ['ocf', 'toc', 'vas', 'vat']\n",
|
||||
"train_glossary = glossary.iloc[[x for x in range(len(glossary)) if x % 6 != 0]]\n",
|
||||
"\n",
|
||||
"nlp = spacy.blank(\"en\")\n",
|
||||
"matcher = FuzzyMatcher(nlp.vocab)\n",
|
||||
"for word in train_glossary['source_lem']:\n",
|
||||
" if word not in bad_words:\n",
|
||||
" matcher.add(word, [nlp(word)])\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"en = []\n",
|
||||
"translation_line_counts = []\n",
|
||||
"for line_id, line in enumerate(file_lemmatized):\n",
|
||||
" doc = nlp(line)\n",
|
||||
" matches = matcher(doc)\n",
|
||||
"\n",
|
||||
" line_counter = 0\n",
|
||||
" for match_id, start, end, ratio in matches:\n",
|
||||
" if ratio > 90:\n",
|
||||
" line_counter += 1\n",
|
||||
" en.append(''.join(doc[:end].text + ' ' + train_glossary.loc[lambda df: df['source_lem'] == match_id]['result'].astype(str).values.flatten() + ' ' + doc[end:].text))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" if line_counter == 0:\n",
|
||||
" line_counter = 1\n",
|
||||
" en.append(line)\n",
|
||||
" translation_line_counts.append(line_counter)\n",
|
||||
"\n"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import copy\n",
|
||||
"tlcs = copy.deepcopy(translation_line_counts)\n",
|
||||
"\n",
|
||||
"translations = pd.read_csv(dev_path + '.pl', sep='\\t', header=None, names=['text'])\n",
|
||||
"with open(dev_path + '.injected.pl', 'w') as file_pl:\n",
|
||||
" for trans in translations.iterrows():\n",
|
||||
" try:\n",
|
||||
" for _ in range(tlcs.pop(0)):\n",
|
||||
" file_pl.write(trans[1]['text'] + '\\n')\n",
|
||||
" except:\n",
|
||||
" pass\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"with open(dev_path + '.injected.en', 'w') as file_en:\n",
|
||||
" for line in en:\n",
|
||||
" file_en.write(line)\n"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Inject glossary Polish crosscheck"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import spacy\n",
|
||||
"from spaczz.matcher import FuzzyMatcher\n",
|
||||
"\n",
|
||||
"# glossary\n",
|
||||
"glossary = pd.read_csv('kompendium_lem.tsv', sep='\\t', header=0, index_col=0)\n",
|
||||
"train_glossary = glossary.iloc[[x for x in range(len(glossary)) if x % 6 != 0]]\n",
|
||||
"\n",
|
||||
"# add rules to English matcher\n",
|
||||
"nlp = spacy.blank(\"en\")\n",
|
||||
"matcher = FuzzyMatcher(nlp.vocab)\n",
|
||||
"for word in train_glossary['source_lem']:\n",
|
||||
" matcher.add(word, [nlp(word)])\n",
|
||||
"\n",
|
||||
"# add rules to Polish matcher\n",
|
||||
"nlp_pl = spacy.blank(\"pl\")\n",
|
||||
"matcher_pl = FuzzyMatcher(nlp_pl.vocab)\n",
|
||||
"for word, word_id in zip(train_glossary['result_lem'], train_glossary['source_lem']):\n",
|
||||
" matcher_pl.add(word, [nlp_pl(word)])\n",
|
||||
"\n",
|
||||
"# todo\n",
|
||||
"en = []\n",
|
||||
"translation_line_counts = []\n",
|
||||
"for line_id in range(len(file_lemmatized)):\n",
|
||||
"\n",
|
||||
" doc = nlp(file_lemmatized[line_id])\n",
|
||||
" matches = matcher(doc)\n",
|
||||
"\n",
|
||||
" line_counter = 0\n",
|
||||
" for match_id, start, end, ratio in matches:\n",
|
||||
" if ratio > 90:\n",
|
||||
" doc_pl = nlp_pl(file_pl_lemmatized[line_id])\n",
|
||||
" matches_pl = matcher_pl(doc_pl)\n",
|
||||
"\n",
|
||||
" for match_id_pl, start_pl, end_pl, ratio_pl in matches_pl:\n",
|
||||
" if match_id_pl == glossary[glossary['source_lem'] == match_id].values[0][3]:\n",
|
||||
" line_counter += 1\n",
|
||||
" en.append(''.join(doc[:end].text + ' ' + train_glossary.loc[lambda df: df['source_lem'] == match_id]['result'].astype(str).values.flatten() + ' ' + doc[end:].text))\n",
|
||||
"\n",
|
||||
" if line_counter == 0:\n",
|
||||
" line_counter = 1\n",
|
||||
" en.append(file_lemmatized[line_id])\n",
|
||||
" translation_line_counts.append(line_counter)\n"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import copy\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"tlcs = copy.deepcopy(translation_line_counts)\n",
|
||||
"\n",
|
||||
"translations = pd.read_csv(dev_path + '.pl', sep='\\t', header=None, names=['text'])\n",
|
||||
"translations['id'] = [x for x in range(len(translations))]\n",
|
||||
"\n",
|
||||
"ctr = 0\n",
|
||||
"sentence = ''\n",
|
||||
"with open(dev_path + '.injected.crossvalidated.en', 'w') as file_en:\n",
|
||||
" with open(dev_path + '.injected.crossvalidated.pl', 'w') as file_pl:\n",
|
||||
" for i in range(len(en)):\n",
|
||||
" if i > 0:\n",
|
||||
" if en[i-1] != en[i]:\n",
|
||||
" if ctr == 0:\n",
|
||||
" sentence = translations.iloc[0]\n",
|
||||
" translations.drop(sentence['id'], inplace=True)\n",
|
||||
" sentence = sentence['text']\n",
|
||||
" try:\n",
|
||||
" ctr = tlcs.pop(0)\n",
|
||||
" except:\n",
|
||||
" pass\n",
|
||||
" file_en.write(en[i])\n",
|
||||
" file_pl.write(sentence + '\\n')\n",
|
||||
" ctr = ctr - 1\n",
|
||||
" else:\n",
|
||||
" try:\n",
|
||||
" ctr = tlcs.pop(0) - 1\n",
|
||||
" except:\n",
|
||||
" pass\n",
|
||||
" sentence = translations.iloc[0]\n",
|
||||
" translations.drop(sentence['id'], inplace=True)\n",
|
||||
" sentence = sentence['text']\n",
|
||||
" file_en.write(en[i])\n",
|
||||
" file_pl.write(sentence + '\\n')"
|
||||
],
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"pycharm": {
|
||||
"name": "#%%\n"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 2
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython2",
|
||||
"version": "2.7.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
18
training-command.txt
Normal file
18
training-command.txt
Normal file
@ -0,0 +1,18 @@
|
||||
first iteration:
|
||||
./marian/build/marian --model mt.npz --type transformer --overwrite \
|
||||
--train-sets mt-summit-corpora/mt-summit-corpora/dev/dev.en \
|
||||
mt-summit-corpora/mt-summit-corpora/dev/dev.pl \
|
||||
--disp-freq 1000 --save-freq 1000 --optimizer adam --lr-report
|
||||
|
||||
next iterations:
|
||||
./marian/build/marian --model mt.npz --type transformer --overwrite \
|
||||
--train-sets mt-summit-corpora/mt-summit-corpora/dev/dev.en \
|
||||
mt-summit-corpora/mt-summit-corpora/dev/dev.pl \
|
||||
--disp-freq 1000 --save-freq 1000 --optimizer adam --lr-report \
|
||||
--pretrained-model mt.npz
|
||||
|
||||
./marian/build/marian --model mt.npz --type transformer --overwrite \
|
||||
--train-sets mt-summit-corpora/mt-summit-corpora/train/train.en \
|
||||
mt-summit-corpora/mt-summit-corpora/train/train.pl \
|
||||
--disp-freq 1000 --save-freq 10000 --optimizer adam --lr-report \
|
||||
--pretrained-model mt.npz
|
||||
Loading…
Reference in New Issue
Block a user