remove files
This commit is contained in:
parent
a8190f419f
commit
b7c17fbf7b
@ -1,503 +0,0 @@
|
|||||||
{
|
|
||||||
"cells": [
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"source": [
|
|
||||||
"## Lemmatize glossary\n",
|
|
||||||
"TODO: train test split glossary"
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"collapsed": false,
|
|
||||||
"pycharm": {
|
|
||||||
"name": "#%% md\n"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 2,
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"data": {
|
|
||||||
"text/plain": " source \\\nsource_lem \naaofi aaofi \naca aca \nacca acca \nabacus abacus \nabandonment cost abandonment costs \n... ... \nytd ytd \nyear-end year-end \nyear-to-date year-to-date \nzog zog \nzero overhead growth zero overhead growth \n\n result \\\nsource_lem \naaofi organizacja rachunkowości i audytu dla islamsk... \naca członek stowarzyszenia dyplomowanych biegłych ... \nacca stowarzyszenie dyplomowanych biegłych rewidentów \nabacus liczydło \nabandonment cost koszty zaniechania \n... ... \nytd od początku roku \nyear-end koniec roku \nyear-to-date od początku roku \nzog zero wzrostu kosztów ogólnych \nzero overhead growth zero wzrostu kosztów ogólnych \n\n result_lem \nsource_lem \naaofi organizacja rachunkowość i audyt dla islamski ... \naca członek stowarzyszenie dyplomowany biegły rewi... \nacca stowarzyszenie dyplomowany biegły rewident \nabacus liczydło \nabandonment cost koszt zaniechanie \n... ... \nytd od początek rok \nyear-end koniec rok \nyear-to-date od początek rok \nzog zero wzrost koszt ogólny \nzero overhead growth zero wzrost koszt ogólny \n\n[1197 rows x 3 columns]",
|
|
||||||
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>source</th>\n <th>result</th>\n <th>result_lem</th>\n </tr>\n <tr>\n <th>source_lem</th>\n <th></th>\n <th></th>\n <th></th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>aaofi</th>\n <td>aaofi</td>\n <td>organizacja rachunkowości i audytu dla islamsk...</td>\n <td>organizacja rachunkowość i audyt dla islamski ...</td>\n </tr>\n <tr>\n <th>aca</th>\n <td>aca</td>\n <td>członek stowarzyszenia dyplomowanych biegłych ...</td>\n <td>członek stowarzyszenie dyplomowany biegły rewi...</td>\n </tr>\n <tr>\n <th>acca</th>\n <td>acca</td>\n <td>stowarzyszenie dyplomowanych biegłych rewidentów</td>\n <td>stowarzyszenie dyplomowany biegły rewident</td>\n </tr>\n <tr>\n <th>abacus</th>\n <td>abacus</td>\n <td>liczydło</td>\n <td>liczydło</td>\n </tr>\n <tr>\n <th>abandonment cost</th>\n <td>abandonment costs</td>\n <td>koszty zaniechania</td>\n <td>koszt zaniechanie</td>\n </tr>\n <tr>\n <th>...</th>\n <td>...</td>\n <td>...</td>\n <td>...</td>\n </tr>\n <tr>\n <th>ytd</th>\n <td>ytd</td>\n <td>od początku roku</td>\n <td>od początek rok</td>\n </tr>\n <tr>\n <th>year-end</th>\n <td>year-end</td>\n <td>koniec roku</td>\n <td>koniec rok</td>\n </tr>\n <tr>\n <th>year-to-date</th>\n <td>year-to-date</td>\n <td>od początku roku</td>\n <td>od początek rok</td>\n </tr>\n <tr>\n <th>zog</th>\n <td>zog</td>\n <td>zero wzrostu kosztów ogólnych</td>\n <td>zero wzrost koszt ogólny</td>\n </tr>\n <tr>\n <th>zero overhead growth</th>\n <td>zero overhead growth</td>\n <td>zero wzrostu kosztów ogólnych</td>\n <td>zero wzrost koszt ogólny</td>\n </tr>\n </tbody>\n</table>\n<p>1197 rows × 3 columns</p>\n</div>"
|
|
||||||
},
|
|
||||||
"execution_count": 2,
|
|
||||||
"metadata": {},
|
|
||||||
"output_type": "execute_result"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"import time\n",
|
|
||||||
"\n",
|
|
||||||
"import pandas as pd\n",
|
|
||||||
"import spacy\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"spacy_nlp_en = spacy.load('en_core_web_sm')\n",
|
|
||||||
"spacy_nlp_pl = spacy.load(\"pl_core_news_sm\")\n",
|
|
||||||
"\n",
|
|
||||||
"glossary = pd.read_csv('kompendium.tsv', sep='\\t', header=None, names=['source', 'result'])\n",
|
|
||||||
"\n",
|
|
||||||
"source_lemmatized = []\n",
|
|
||||||
"for word in glossary['source']:\n",
|
|
||||||
" temp = []\n",
|
|
||||||
" for token in spacy_nlp_en(word):\n",
|
|
||||||
" temp.append(token.lemma_)\n",
|
|
||||||
" source_lemmatized.append(' '.join(temp).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))\n",
|
|
||||||
"\n",
|
|
||||||
"result_lemmatized = []\n",
|
|
||||||
"for word in glossary['result']:\n",
|
|
||||||
" temp = []\n",
|
|
||||||
" for token in spacy_nlp_pl(word):\n",
|
|
||||||
" temp.append(token.lemma_)\n",
|
|
||||||
" result_lemmatized.append(' '.join(temp).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))\n",
|
|
||||||
"\n",
|
|
||||||
"glossary['source_lem'] = source_lemmatized\n",
|
|
||||||
"glossary['result_lem'] = result_lemmatized\n",
|
|
||||||
"glossary = glossary[['source', 'source_lem', 'result', 'result_lem']]\n",
|
|
||||||
"glossary.set_index('source_lem')\n"
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"collapsed": false,
|
|
||||||
"pycharm": {
|
|
||||||
"name": "#%%\n"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 2,
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"glossary.to_csv('kompendium_lem.tsv', sep='\\t')"
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"collapsed": false,
|
|
||||||
"pycharm": {
|
|
||||||
"name": "#%%\n"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"source": [
|
|
||||||
"## Lemmatize corpus"
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"collapsed": false
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 3,
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"dev_path = 'mt-summit-corpora/dev/dev'\n",
|
|
||||||
"\n",
|
|
||||||
"skip_chars = ''',./!?'''\n",
|
|
||||||
"\n",
|
|
||||||
"with open(dev_path + '.en', 'r') as file:\n",
|
|
||||||
" file_lemmatized = []\n",
|
|
||||||
" for line in file:\n",
|
|
||||||
" temp = []\n",
|
|
||||||
" for token in spacy_nlp_en(line):\n",
|
|
||||||
" temp.append(token.lemma_)\n",
|
|
||||||
" file_lemmatized.append(' '.join([x for x in temp if x not in skip_chars]).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))\n",
|
|
||||||
"\n",
|
|
||||||
"with open(dev_path + '.pl', 'r') as file:\n",
|
|
||||||
" file_pl_lemmatized = []\n",
|
|
||||||
" for line in file:\n",
|
|
||||||
" temp = []\n",
|
|
||||||
" for token in spacy_nlp_pl(line):\n",
|
|
||||||
" temp.append(token.lemma_)\n",
|
|
||||||
" file_pl_lemmatized.append(' '.join([x for x in temp if x not in skip_chars]).replace(' - ', '-').replace(' ’', '’').replace(' / ', '/').replace(' ( ', '(').replace(' ) ', ')'))\n",
|
|
||||||
"\n"
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"collapsed": false,
|
|
||||||
"pycharm": {
|
|
||||||
"name": "#%%\n"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 4,
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"in the course of the control the control audit firm shall fulfil the responsibility refer to in article 114 on date and in form specify by the controller \n",
|
|
||||||
"\n",
|
|
||||||
"w czas trwanie kontrola kontrolowany firma audytorski wypełnia obowiązek o których mowa w art 114 w ter-mina i forma wskazany przez osoba kontrolującą \n",
|
|
||||||
"\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"print(file_lemmatized[2])\n",
|
|
||||||
"print(file_pl_lemmatized[2])"
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"collapsed": false,
|
|
||||||
"pycharm": {
|
|
||||||
"name": "#%%\n"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"source": [
|
|
||||||
"## Inject glossary\n",
|
|
||||||
"# !!! Obsolete !!!"
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"collapsed": false,
|
|
||||||
"pycharm": {
|
|
||||||
"name": "#%% md\n"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"import spacy\n",
|
|
||||||
"from spaczz.matcher import FuzzyMatcher\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"glossary = pd.read_csv('kompendium_lem.tsv', sep='\\t', header=0, index_col=0)\n",
|
|
||||||
"bad_words = ['ocf', 'toc', 'vas', 'vat']\n",
|
|
||||||
"train_glossary = glossary.iloc[[x for x in range(len(glossary)) if x % 6 != 0]]\n",
|
|
||||||
"\n",
|
|
||||||
"nlp = spacy.blank(\"en\")\n",
|
|
||||||
"matcher = FuzzyMatcher(nlp.vocab)\n",
|
|
||||||
"for word in train_glossary['source_lem']:\n",
|
|
||||||
" if word not in bad_words:\n",
|
|
||||||
" matcher.add(word, [nlp(word)])\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"en = []\n",
|
|
||||||
"translation_line_counts = []\n",
|
|
||||||
"for line_id, line in enumerate(file_lemmatized):\n",
|
|
||||||
" doc = nlp(line)\n",
|
|
||||||
" matches = matcher(doc)\n",
|
|
||||||
"\n",
|
|
||||||
" not_injected = 0\n",
|
|
||||||
" for match_id, start, end, ratio in matches:\n",
|
|
||||||
" if ratio > 90:\n",
|
|
||||||
" not_injected += 1\n",
|
|
||||||
" en.append(''.join(doc[:end].text + ' ' + train_glossary.loc[lambda df: df['source_lem'] == match_id]['result'].astype(str).values.flatten() + ' ' + doc[end:].text))\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
" if not_injected == 0:\n",
|
|
||||||
" not_injected = 1\n",
|
|
||||||
" en.append(line)\n",
|
|
||||||
" translation_line_counts.append(not_injected)\n",
|
|
||||||
"\n"
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"collapsed": false,
|
|
||||||
"pycharm": {
|
|
||||||
"name": "#%%\n",
|
|
||||||
"is_executing": true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 6,
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"import copy\n",
|
|
||||||
"tlcs = copy.deepcopy(translation_line_counts)\n",
|
|
||||||
"\n",
|
|
||||||
"translations = pd.read_csv(dev_path + '.pl', sep='\\t', header=None, names=['text'])\n",
|
|
||||||
"with open(dev_path + '.injected.pl', 'w') as file_pl:\n",
|
|
||||||
" for trans in translations.iterrows():\n",
|
|
||||||
" try:\n",
|
|
||||||
" for _ in range(tlcs.pop(0)):\n",
|
|
||||||
" file_pl.write(trans[1]['text'] + '\\n')\n",
|
|
||||||
" except:\n",
|
|
||||||
" pass\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"with open(dev_path + '.injected.en', 'w') as file_en:\n",
|
|
||||||
" for line in en:\n",
|
|
||||||
" file_en.write(line)\n"
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"collapsed": false,
|
|
||||||
"pycharm": {
|
|
||||||
"name": "#%%\n"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"source": [
|
|
||||||
"## Inject glossary Polish crosscheck"
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"collapsed": false
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 7,
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"import spacy\n",
|
|
||||||
"from spaczz.matcher import FuzzyMatcher\n",
|
|
||||||
"\n",
|
|
||||||
"# glossary\n",
|
|
||||||
"glossary = pd.read_csv('kompendium_lem.tsv', sep='\\t', header=0, index_col=0)\n",
|
|
||||||
"train_glossary = glossary.iloc[[x for x in range(len(glossary)) if x % 6 != 0]]\n",
|
|
||||||
"\n",
|
|
||||||
"# add rules to English matcher\n",
|
|
||||||
"nlp = spacy.blank(\"en\")\n",
|
|
||||||
"matcher = FuzzyMatcher(nlp.vocab)\n",
|
|
||||||
"for word in train_glossary['source_lem']:\n",
|
|
||||||
" matcher.add(word, [nlp(word)])\n",
|
|
||||||
"\n",
|
|
||||||
"# add rules to Polish matcher\n",
|
|
||||||
"nlp_pl = spacy.blank(\"pl\")\n",
|
|
||||||
"matcher_pl = FuzzyMatcher(nlp_pl.vocab)\n",
|
|
||||||
"for word, word_id in zip(train_glossary['result_lem'], train_glossary['source_lem']):\n",
|
|
||||||
" matcher_pl.add(word, [nlp_pl(word)])\n",
|
|
||||||
"\n",
|
|
||||||
"en = []\n",
|
|
||||||
"translation_line_counts = []\n",
|
|
||||||
"for line_id in range(len(file_lemmatized)):\n",
|
|
||||||
"\n",
|
|
||||||
" doc = nlp(file_lemmatized[line_id])\n",
|
|
||||||
" matches = matcher(doc)\n",
|
|
||||||
"\n",
|
|
||||||
" not_injected = 0\n",
|
|
||||||
" for match_id, start, end, ratio in matches:\n",
|
|
||||||
" if ratio > 90:\n",
|
|
||||||
" doc_pl = nlp_pl(file_pl_lemmatized[line_id])\n",
|
|
||||||
" matches_pl = matcher_pl(doc_pl)\n",
|
|
||||||
"\n",
|
|
||||||
" for match_id_pl, start_pl, end_pl, ratio_pl in matches_pl:\n",
|
|
||||||
" if match_id_pl == glossary[glossary['source_lem'] == match_id].values[0][3]:\n",
|
|
||||||
" not_injected += 1\n",
|
|
||||||
" en.append(''.join(doc[:end].text + ' ' + train_glossary.loc[lambda df: df['source_lem'] == match_id]['result'].astype(str).values.flatten() + ' ' + doc[end:].text))\n",
|
|
||||||
"\n",
|
|
||||||
" if not_injected == 0:\n",
|
|
||||||
" not_injected = 1\n",
|
|
||||||
" en.append(file_lemmatized[line_id])\n",
|
|
||||||
" translation_line_counts.append(not_injected)\n"
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"collapsed": false,
|
|
||||||
"pycharm": {
|
|
||||||
"name": "#%%\n"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 8,
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"import copy\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"tlcs = copy.deepcopy(translation_line_counts)\n",
|
|
||||||
"\n",
|
|
||||||
"translations = pd.read_csv(dev_path + '.pl', sep='\\t', header=None, names=['text'])\n",
|
|
||||||
"translations['id'] = [x for x in range(len(translations))]\n",
|
|
||||||
"\n",
|
|
||||||
"ctr = 0\n",
|
|
||||||
"sentence = ''\n",
|
|
||||||
"with open(dev_path + '.injected.crossvalidated.en', 'w') as file_en:\n",
|
|
||||||
" with open(dev_path + '.injected.crossvalidated.pl', 'w') as file_pl:\n",
|
|
||||||
" for i in range(len(en)):\n",
|
|
||||||
" if i > 0:\n",
|
|
||||||
" if en[i-1] != en[i]:\n",
|
|
||||||
" if ctr == 0:\n",
|
|
||||||
" sentence = translations.iloc[0]\n",
|
|
||||||
" translations.drop(sentence['id'], inplace=True)\n",
|
|
||||||
" sentence = sentence['text']\n",
|
|
||||||
" try:\n",
|
|
||||||
" ctr = tlcs.pop(0)\n",
|
|
||||||
" except:\n",
|
|
||||||
" pass\n",
|
|
||||||
" file_en.write(en[i])\n",
|
|
||||||
" file_pl.write(sentence + '\\n')\n",
|
|
||||||
" ctr = ctr - 1\n",
|
|
||||||
" else:\n",
|
|
||||||
" try:\n",
|
|
||||||
" ctr = tlcs.pop(0) - 1\n",
|
|
||||||
" except:\n",
|
|
||||||
" pass\n",
|
|
||||||
" sentence = translations.iloc[0]\n",
|
|
||||||
" translations.drop(sentence['id'], inplace=True)\n",
|
|
||||||
" sentence = sentence['text']\n",
|
|
||||||
" file_en.write(en[i])\n",
|
|
||||||
" file_pl.write(sentence + '\\n')"
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"collapsed": false,
|
|
||||||
"pycharm": {
|
|
||||||
"name": "#%%\n"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"source": [
|
|
||||||
"# Inject glossary Polish crosscheck fast?"
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"collapsed": false,
|
|
||||||
"pycharm": {
|
|
||||||
"name": "#%% md\n"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 49,
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"took 152.213599056 injected 63 words. rate 6.569715230451229 sen/s\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"import time\n",
|
|
||||||
"import spacy\n",
|
|
||||||
"from spaczz.matcher import FuzzyMatcher\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"# glossary\n",
|
|
||||||
"glossary = pd.read_csv('kompendium_lem.tsv', sep='\\t', header=0, index_col=0)\n",
|
|
||||||
"train_glossary = glossary.iloc[[x for x in range(len(glossary)) if x % 6 != 0]]\n",
|
|
||||||
"\n",
|
|
||||||
"# add rules to English matcher\n",
|
|
||||||
"nlp = spacy.blank(\"en\")\n",
|
|
||||||
"matcher = FuzzyMatcher(nlp.vocab)\n",
|
|
||||||
"for word in train_glossary['source_lem']:\n",
|
|
||||||
" matcher.add(word, [nlp(word)])\n",
|
|
||||||
"\n",
|
|
||||||
"# add rules to Polish matcher\n",
|
|
||||||
"nlp_pl = spacy.blank(\"pl\")\n",
|
|
||||||
"matcher_pl = FuzzyMatcher(nlp_pl.vocab)\n",
|
|
||||||
"for word, word_id in zip(train_glossary['result_lem'], train_glossary['source_lem']):\n",
|
|
||||||
" matcher_pl.add(word, [nlp_pl(word)])\n",
|
|
||||||
"\n",
|
|
||||||
"start_time = time.time_ns()\n",
|
|
||||||
"en = []\n",
|
|
||||||
"injection_counter = 0\n",
|
|
||||||
"for line_id in range(len(file_lemmatized)):\n",
|
|
||||||
"\n",
|
|
||||||
" doc = nlp(file_lemmatized[line_id])\n",
|
|
||||||
" matches = matcher(nlp(file_lemmatized[line_id]))\n",
|
|
||||||
"\n",
|
|
||||||
" not_injected = True\n",
|
|
||||||
" if len(matches) > 0:\n",
|
|
||||||
" match_id, _, end, ratio = sorted(matches, key=lambda x: len(x[0]), reverse=True)[0]\n",
|
|
||||||
" if ratio > 90:\n",
|
|
||||||
" matches_pl = matcher_pl(nlp_pl(file_pl_lemmatized[line_id]))\n",
|
|
||||||
"\n",
|
|
||||||
" for match_id_pl, _, _, _ in matches_pl:\n",
|
|
||||||
" if match_id_pl == glossary[glossary['source_lem'] == match_id].values[0][3]:\n",
|
|
||||||
" not_injected = False\n",
|
|
||||||
" injection_counter += 1\n",
|
|
||||||
" en.append(''.join(doc[:end].text + ' ' + train_glossary.loc[lambda df: df['source_lem'] == match_id]['result'].astype(str).values.flatten() + ' ' + doc[end:].text))\n",
|
|
||||||
" break\n",
|
|
||||||
"\n",
|
|
||||||
" if not_injected:\n",
|
|
||||||
" en.append(file_lemmatized[line_id])\n",
|
|
||||||
"\n",
|
|
||||||
"stop = time.time_ns()\n",
|
|
||||||
"timex = (stop - start_time) / 1000000000\n",
|
|
||||||
"print(f'took {timex} injected {injection_counter} words. rate {len(file_lemmatized)/timex} sen/s')"
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"collapsed": false,
|
|
||||||
"pycharm": {
|
|
||||||
"name": "#%%\n"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 9,
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"import copy\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"tlcs = copy.deepcopy(translation_line_counts)\n",
|
|
||||||
"\n",
|
|
||||||
"translations = pd.read_csv(dev_path + '.pl', sep='\\t', header=None, names=['text'])\n",
|
|
||||||
"translations['id'] = [x for x in range(len(translations))]\n",
|
|
||||||
"\n",
|
|
||||||
"ctr = 0\n",
|
|
||||||
"sentence = ''\n",
|
|
||||||
"with open(dev_path + '.injected.crossvalidated.en', 'w') as file_en:\n",
|
|
||||||
" with open(dev_path + '.injected.crossvalidated.pl', 'w') as file_pl:\n",
|
|
||||||
" for i in range(len(en)):\n",
|
|
||||||
" if i > 0:\n",
|
|
||||||
" if en[i-1] != en[i]:\n",
|
|
||||||
" if ctr == 0:\n",
|
|
||||||
" sentence = translations.iloc[0]\n",
|
|
||||||
" translations.drop(sentence['id'], inplace=True)\n",
|
|
||||||
" sentence = sentence['text']\n",
|
|
||||||
" try:\n",
|
|
||||||
" ctr = tlcs.pop(0)\n",
|
|
||||||
" except:\n",
|
|
||||||
" pass\n",
|
|
||||||
" file_en.write(en[i])\n",
|
|
||||||
" file_pl.write(sentence + '\\n')\n",
|
|
||||||
" ctr = ctr - 1\n",
|
|
||||||
" else:\n",
|
|
||||||
" try:\n",
|
|
||||||
" ctr = tlcs.pop(0) - 1\n",
|
|
||||||
" except:\n",
|
|
||||||
" pass\n",
|
|
||||||
" sentence = translations.iloc[0]\n",
|
|
||||||
" translations.drop(sentence['id'], inplace=True)\n",
|
|
||||||
" sentence = sentence['text']\n",
|
|
||||||
" file_en.write(en[i])\n",
|
|
||||||
" file_pl.write(sentence + '\\n')\n"
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"collapsed": false,
|
|
||||||
"pycharm": {
|
|
||||||
"name": "#%%\n"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"kernelspec": {
|
|
||||||
"display_name": "Python 3",
|
|
||||||
"language": "python",
|
|
||||||
"name": "python3"
|
|
||||||
},
|
|
||||||
"language_info": {
|
|
||||||
"codemirror_mode": {
|
|
||||||
"name": "ipython",
|
|
||||||
"version": 2
|
|
||||||
},
|
|
||||||
"file_extension": ".py",
|
|
||||||
"mimetype": "text/x-python",
|
|
||||||
"name": "python",
|
|
||||||
"nbconvert_exporter": "python",
|
|
||||||
"pygments_lexer": "ipython2",
|
|
||||||
"version": "2.7.6"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 0
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user