s470607/mt-summit-corpora
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

remove files

Browse Source
This commit is contained in:
Jakub Konieczny 2022-01-18 09:35:43 +00:00
parent a8190f419f
commit b7c17fbf7b
2 changed files with 0 additions and 1697 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

503
jupyter-injector.ipynb
View File

@ -1,503 +0,0 @@
{
"cells": [
{
"cell_type": "markdown",
"source": [
"## Lemmatize glossary\n",
"TODO: train test split glossary"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
}
},
{
"cell_type": "code",
"execution_count": 2,
"outputs": [
{
"data": {
"text/plain": " source \\\nsource_lem \naaofi aaofi \naca aca \nacca acca \nabacus abacus \nabandonment cost abandonment costs \n... ... \nytd ytd \nyear-end year-end \nyear-to-date year-to-date \nzog zog \nzero overhead growth zero overhead growth \n\n result \\\nsource_lem \naaofi organizacja rachunkowości i audytu dla islamsk... \naca członek stowarzyszenia dyplomowanych biegłych ... \nacca stowarzyszenie dyplomowanych biegłych rewidentów \nabacus liczydło \nabandonment cost koszty zaniechania \n... ... \nytd od początku roku \nyear-end koniec roku \nyear-to-date od początku roku \nzog zero wzrostu kosztów ogólnych \nzero overhead growth zero wzrostu kosztów ogólnych \n\n result_lem \nsource_lem \naaofi organizacja rachunkowość i audyt dla islamski ... \naca członek stowarzyszenie dyplomowany biegły rewi... \nacca stowarzyszenie dyplomowany biegły rewident \nabacus liczydło \nabandonment cost koszt zaniechanie \n... ... \nytd od początek rok \nyear-end koniec rok \nyear-to-date od początek rok \nzog zero wzrost koszt ogólny \nzero overhead growth zero wzrost koszt ogólny \n\n[1197 rows x 3 columns]",
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>source</th>\n <th>result</th>\n <th>result_lem</th>\n </tr>\n <tr>\n <th>source_lem</th>\n <th></th>\n <th></th>\n <th></th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>aaofi</th>\n <td>aaofi</td>\n <td>organizacja rachunkowości i audytu dla islamsk...</td>\n <td>organizacja rachunkowość i audyt dla islamski ...</td>\n </tr>\n <tr>\n <th>aca</th>\n <td>aca</td>\n <td>członek stowarzyszenia dyplomowanych biegłych ...</td>\n <td>członek stowarzyszenie dyplomowany biegły rewi...</td>\n </tr>\n <tr>\n <th>acca</th>\n <td>acca</td>\n <td>stowarzyszenie dyplomowanych biegłych rewidentów</td>\n <td>stowarzyszenie dyplomowany biegły rewident</td>\n </tr>\n <tr>\n <th>abacus</th>\n <td>abacus</td>\n <td>liczydło</td>\n <td>liczydło</td>\n </tr>\n <tr>\n <th>abandonment cost</th>\n <td>abandonment costs</td>\n <td>koszty zaniechania</td>\n <td>koszt zaniechanie</td>\n </tr>\n <tr>\n <th>...</th>\n <td>...</td>\n <td>...</td>\n <td>...</td>\n </tr>\n <tr>\n <th>ytd</th>\n <td>ytd</td>\n <td>od początku roku</td>\n <td>od początek rok</td>\n </tr>\n <tr>\n <th>year-end</th>\n <td>year-end</td>\n <td>koniec roku</td>\n <td>koniec rok</td>\n </tr>\n <tr>\n <th>year-to-date</th>\n <td>year-to-date</td>\n <td>od początku roku</td>\n <td>od początek rok</td>\n </tr>\n <tr>\n <th>zog</th>\n <td>zog</td>\n <td>zero wzrostu kosztów ogólnych</td>\n <td>zero wzrost koszt ogólny</td>\n </tr>\n <tr>\n <th>zero overhead growth</th>\n <td>zero overhead growth</td>\n <td>zero wzrostu kosztów ogólnych</td>\n <td>zero wzrost koszt ogólny</td>\n </tr>\n </tbody>\n</table>\n<p>1197 rows × 3 columns</p>\n</div>"
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import time\n",
"\n",
"import pandas as pd\n",
"import spacy\n",
"\n",
"\n",
"spacy_nlp_en = spacy.load('en_core_web_sm')\n",
"spacy_nlp_pl = spacy.load(\"pl_core_news_sm\")\n",
"\n",
"glossary = pd.read_csv('kompendium.tsv', sep='\\t', header=None, names=['source', 'result'])\n",
"\n",
"source_lemmatized = []\n",
"for word in glossary['source']:\n",
" temp = []\n",
" for token in spacy_nlp_en(word):\n",
" temp.append(token.lemma_)\n",
" source_lemmatized.append(' '.join(temp).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))\n",
"\n",
"result_lemmatized = []\n",
"for word in glossary['result']:\n",
" temp = []\n",
" for token in spacy_nlp_pl(word):\n",
" temp.append(token.lemma_)\n",
" result_lemmatized.append(' '.join(temp).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))\n",
"\n",
"glossary['source_lem'] = source_lemmatized\n",
"glossary['result_lem'] = result_lemmatized\n",
"glossary = glossary[['source', 'source_lem', 'result', 'result_lem']]\n",
"glossary.set_index('source_lem')\n"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 2,
"outputs": [],
"source": [
"glossary.to_csv('kompendium_lem.tsv', sep='\\t')"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "markdown",
"source": [
"## Lemmatize corpus"
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "code",
"execution_count": 3,
"outputs": [],
"source": [
"dev_path = 'mt-summit-corpora/dev/dev'\n",
"\n",
"skip_chars = ''',./!?'''\n",
"\n",
"with open(dev_path + '.en', 'r') as file:\n",
" file_lemmatized = []\n",
" for line in file:\n",
" temp = []\n",
" for token in spacy_nlp_en(line):\n",
" temp.append(token.lemma_)\n",
" file_lemmatized.append(' '.join([x for x in temp if x not in skip_chars]).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))\n",
"\n",
"with open(dev_path + '.pl', 'r') as file:\n",
" file_pl_lemmatized = []\n",
" for line in file:\n",
" temp = []\n",
" for token in spacy_nlp_pl(line):\n",
" temp.append(token.lemma_)\n",
" file_pl_lemmatized.append(' '.join([x for x in temp if x not in skip_chars]).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))\n",
"\n"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 4,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"in the course of the control the control audit firm shall fulfil the responsibility refer to in article 114 on date and in form specify by the controller \n",
"\n",
"w czas trwanie kontrola kontrolowany firma audytorski wypełnia obowiązek o których mowa w art 114 w ter-mina i forma wskazany przez osoba kontrolującą \n",
"\n"
]
}
],
"source": [
"print(file_lemmatized[2])\n",
"print(file_pl_lemmatized[2])"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "markdown",
"source": [
"## Inject glossary\n",
"# !!! Obsolete !!!"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
}
},
{
"cell_type": "code",
"execution_count": null,
"outputs": [],
"source": [
"import spacy\n",
"from spaczz.matcher import FuzzyMatcher\n",
"\n",
"\n",
"glossary = pd.read_csv('kompendium_lem.tsv', sep='\\t', header=0, index_col=0)\n",
"bad_words = ['ocf', 'toc', 'vas', 'vat']\n",
"train_glossary = glossary.iloc[[x for x in range(len(glossary)) if x % 6 != 0]]\n",
"\n",
"nlp = spacy.blank(\"en\")\n",
"matcher = FuzzyMatcher(nlp.vocab)\n",
"for word in train_glossary['source_lem']:\n",
" if word not in bad_words:\n",
" matcher.add(word, [nlp(word)])\n",
"\n",
"\n",
"en = []\n",
"translation_line_counts = []\n",
"for line_id, line in enumerate(file_lemmatized):\n",
" doc = nlp(line)\n",
" matches = matcher(doc)\n",
"\n",
" not_injected = 0\n",
" for match_id, start, end, ratio in matches:\n",
" if ratio > 90:\n",
" not_injected += 1\n",
" en.append(''.join(doc[:end].text + ' ' + train_glossary.loc[lambda df: df['source_lem'] == match_id]['result'].astype(str).values.flatten() + ' ' + doc[end:].text))\n",
"\n",
"\n",
" if not_injected == 0:\n",
" not_injected = 1\n",
" en.append(line)\n",
" translation_line_counts.append(not_injected)\n",
"\n"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n",
"is_executing": true
}
}
},
{
"cell_type": "code",
"execution_count": 6,
"outputs": [],
"source": [
"import copy\n",
"tlcs = copy.deepcopy(translation_line_counts)\n",
"\n",
"translations = pd.read_csv(dev_path + '.pl', sep='\\t', header=None, names=['text'])\n",
"with open(dev_path + '.injected.pl', 'w') as file_pl:\n",
" for trans in translations.iterrows():\n",
" try:\n",
" for _ in range(tlcs.pop(0)):\n",
" file_pl.write(trans[1]['text'] + '\\n')\n",
" except:\n",
" pass\n",
"\n",
"\n",
"with open(dev_path + '.injected.en', 'w') as file_en:\n",
" for line in en:\n",
" file_en.write(line)\n"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "markdown",
"source": [
"## Inject glossary Polish crosscheck"
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "code",
"execution_count": 7,
"outputs": [],
"source": [
"import spacy\n",
"from spaczz.matcher import FuzzyMatcher\n",
"\n",
"# glossary\n",
"glossary = pd.read_csv('kompendium_lem.tsv', sep='\\t', header=0, index_col=0)\n",
"train_glossary = glossary.iloc[[x for x in range(len(glossary)) if x % 6 != 0]]\n",
"\n",
"# add rules to English matcher\n",
"nlp = spacy.blank(\"en\")\n",
"matcher = FuzzyMatcher(nlp.vocab)\n",
"for word in train_glossary['source_lem']:\n",
" matcher.add(word, [nlp(word)])\n",
"\n",
"# add rules to Polish matcher\n",
"nlp_pl = spacy.blank(\"pl\")\n",
"matcher_pl = FuzzyMatcher(nlp_pl.vocab)\n",
"for word, word_id in zip(train_glossary['result_lem'], train_glossary['source_lem']):\n",
" matcher_pl.add(word, [nlp_pl(word)])\n",
"\n",
"en = []\n",
"translation_line_counts = []\n",
"for line_id in range(len(file_lemmatized)):\n",
"\n",
" doc = nlp(file_lemmatized[line_id])\n",
" matches = matcher(doc)\n",
"\n",
" not_injected = 0\n",
" for match_id, start, end, ratio in matches:\n",
" if ratio > 90:\n",
" doc_pl = nlp_pl(file_pl_lemmatized[line_id])\n",
" matches_pl = matcher_pl(doc_pl)\n",
"\n",
" for match_id_pl, start_pl, end_pl, ratio_pl in matches_pl:\n",
" if match_id_pl == glossary[glossary['source_lem'] == match_id].values[0][3]:\n",
" not_injected += 1\n",
" en.append(''.join(doc[:end].text + ' ' + train_glossary.loc[lambda df: df['source_lem'] == match_id]['result'].astype(str).values.flatten() + ' ' + doc[end:].text))\n",
"\n",
" if not_injected == 0:\n",
" not_injected = 1\n",
" en.append(file_lemmatized[line_id])\n",
" translation_line_counts.append(not_injected)\n"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 8,
"outputs": [],
"source": [
"import copy\n",
"\n",
"\n",
"tlcs = copy.deepcopy(translation_line_counts)\n",
"\n",
"translations = pd.read_csv(dev_path + '.pl', sep='\\t', header=None, names=['text'])\n",
"translations['id'] = [x for x in range(len(translations))]\n",
"\n",
"ctr = 0\n",
"sentence = ''\n",
"with open(dev_path + '.injected.crossvalidated.en', 'w') as file_en:\n",
" with open(dev_path + '.injected.crossvalidated.pl', 'w') as file_pl:\n",
" for i in range(len(en)):\n",
" if i > 0:\n",
" if en[i-1] != en[i]:\n",
" if ctr == 0:\n",
" sentence = translations.iloc[0]\n",
" translations.drop(sentence['id'], inplace=True)\n",
" sentence = sentence['text']\n",
" try:\n",
" ctr = tlcs.pop(0)\n",
" except:\n",
" pass\n",
" file_en.write(en[i])\n",
" file_pl.write(sentence + '\\n')\n",
" ctr = ctr - 1\n",
" else:\n",
" try:\n",
" ctr = tlcs.pop(0) - 1\n",
" except:\n",
" pass\n",
" sentence = translations.iloc[0]\n",
" translations.drop(sentence['id'], inplace=True)\n",
" sentence = sentence['text']\n",
" file_en.write(en[i])\n",
" file_pl.write(sentence + '\\n')"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "markdown",
"source": [
"# Inject glossary Polish crosscheck fast?"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
}
},
{
"cell_type": "code",
"execution_count": 49,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"took 152.213599056 injected 63 words. rate 6.569715230451229 sen/s\n"
]
}
],
"source": [
"import time\n",
"import spacy\n",
"from spaczz.matcher import FuzzyMatcher\n",
"\n",
"\n",
"# glossary\n",
"glossary = pd.read_csv('kompendium_lem.tsv', sep='\\t', header=0, index_col=0)\n",
"train_glossary = glossary.iloc[[x for x in range(len(glossary)) if x % 6 != 0]]\n",
"\n",
"# add rules to English matcher\n",
"nlp = spacy.blank(\"en\")\n",
"matcher = FuzzyMatcher(nlp.vocab)\n",
"for word in train_glossary['source_lem']:\n",
" matcher.add(word, [nlp(word)])\n",
"\n",
"# add rules to Polish matcher\n",
"nlp_pl = spacy.blank(\"pl\")\n",
"matcher_pl = FuzzyMatcher(nlp_pl.vocab)\n",
"for word, word_id in zip(train_glossary['result_lem'], train_glossary['source_lem']):\n",
" matcher_pl.add(word, [nlp_pl(word)])\n",
"\n",
"start_time = time.time_ns()\n",
"en = []\n",
"injection_counter = 0\n",
"for line_id in range(len(file_lemmatized)):\n",
"\n",
" doc = nlp(file_lemmatized[line_id])\n",
" matches = matcher(nlp(file_lemmatized[line_id]))\n",
"\n",
" not_injected = True\n",
" if len(matches) > 0:\n",
" match_id, _, end, ratio = sorted(matches, key=lambda x: len(x[0]), reverse=True)[0]\n",
" if ratio > 90:\n",
" matches_pl = matcher_pl(nlp_pl(file_pl_lemmatized[line_id]))\n",
"\n",
" for match_id_pl, _, _, _ in matches_pl:\n",
" if match_id_pl == glossary[glossary['source_lem'] == match_id].values[0][3]:\n",
" not_injected = False\n",
" injection_counter += 1\n",
" en.append(''.join(doc[:end].text + ' ' + train_glossary.loc[lambda df: df['source_lem'] == match_id]['result'].astype(str).values.flatten() + ' ' + doc[end:].text))\n",
" break\n",
"\n",
" if not_injected:\n",
" en.append(file_lemmatized[line_id])\n",
"\n",
"stop = time.time_ns()\n",
"timex = (stop - start_time) / 1000000000\n",
"print(f'took {timex} injected {injection_counter} words. rate {len(file_lemmatized)/timex} sen/s')"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 9,
"outputs": [],
"source": [
"import copy\n",
"\n",
"\n",
"tlcs = copy.deepcopy(translation_line_counts)\n",
"\n",
"translations = pd.read_csv(dev_path + '.pl', sep='\\t', header=None, names=['text'])\n",
"translations['id'] = [x for x in range(len(translations))]\n",
"\n",
"ctr = 0\n",
"sentence = ''\n",
"with open(dev_path + '.injected.crossvalidated.en', 'w') as file_en:\n",
" with open(dev_path + '.injected.crossvalidated.pl', 'w') as file_pl:\n",
" for i in range(len(en)):\n",
" if i > 0:\n",
" if en[i-1] != en[i]:\n",
" if ctr == 0:\n",
" sentence = translations.iloc[0]\n",
" translations.drop(sentence['id'], inplace=True)\n",
" sentence = sentence['text']\n",
" try:\n",
" ctr = tlcs.pop(0)\n",
" except:\n",
" pass\n",
" file_en.write(en[i])\n",
" file_pl.write(sentence + '\\n')\n",
" ctr = ctr - 1\n",
" else:\n",
" try:\n",
" ctr = tlcs.pop(0) - 1\n",
" except:\n",
" pass\n",
" sentence = translations.iloc[0]\n",
" translations.drop(sentence['id'], inplace=True)\n",
" sentence = sentence['text']\n",
" file_en.write(en[i])\n",
" file_pl.write(sentence + '\\n')\n"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

1194
kompendium_lem_cleaned.tsv
View File

File diff suppressed because it is too large Load Diff