forked from bfijalkowski/KWT-2024
Upload files to "lab"
This commit is contained in:
parent
9a804fe3dd
commit
9cc45ec99e
7543
lab/lab_04-05.ipynb
7543
lab/lab_04-05.ipynb
File diff suppressed because one or more lines are too long
|
|
@ -55,13 +55,40 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": 2,
|
||||
"id": "documented-hacker",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import re\n",
|
||||
"\n",
|
||||
"def find_tags(text):\n",
|
||||
" return []"
|
||||
" pattern = re.compile(r'</?\\w+.*?>')\n",
|
||||
" \n",
|
||||
" matches = pattern.finditer(text)\n",
|
||||
" \n",
|
||||
" results = [(match.group(), match.start(), match.end()) for match in matches]\n",
|
||||
" return results"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "8d1c8b3a-a3b0-43ca-a6c1-7b3481e58b87",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[('<note>', 0, 6), ('<to>', 6, 10), ('</to>', 14, 19), ('<from>', 19, 25), ('</from>', 29, 36), ('<heading>', 36, 45), ('</heading>', 53, 63), ('<body>', 63, 69), ('</body>', 98, 105), ('</note>', 105, 112)]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"tekst = \"<note><to>Tove</to><from>Jani</from><heading>Reminder</heading><body>Don't forget me this weekend!</body></note>\"\n",
|
||||
"tagi = find_tags(tekst)\n",
|
||||
"print(tagi)"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -74,15 +101,55 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": 8,
|
||||
"id": "unauthorized-study",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def is_translatable(text):\n",
|
||||
" if re.search(r'\\b(v?\\d+(\\.\\d+)+)\\b', text):\n",
|
||||
" return False\n",
|
||||
" \n",
|
||||
" if re.search(r'function\\s+\\w+\\s*\\(|if\\s+\\(|\\w+\\s*=\\s*\\w+', text):\n",
|
||||
" return False\n",
|
||||
" \n",
|
||||
" if re.search(r'\\b[A-Z]{2,}\\b', text):\n",
|
||||
" return False\n",
|
||||
" \n",
|
||||
" return True"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "1051142c-8170-43fc-b55a-0c367bf69e31",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"'This should be translated.': True\n",
|
||||
"'Version 4.2.1 should not be translated.': False\n",
|
||||
"'Contact API_KEY for more details.': True\n",
|
||||
"'if (x == 10) { return x; }': False\n",
|
||||
"'Welcome to New York City!': True\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"examples = [\n",
|
||||
" \"This should be translated.\",\n",
|
||||
" \"Version 4.2.1 should not be translated.\",\n",
|
||||
" \"Contact API_KEY for more details.\",\n",
|
||||
" \"if (x == 10) { return x; }\",\n",
|
||||
" \"Welcome to New York City!\"\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"for example in examples:\n",
|
||||
" print(f\"'{example}': {is_translatable(example)}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "plastic-crown",
|
||||
|
|
@ -93,13 +160,61 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": 11,
|
||||
"id": "beautiful-mathematics",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def find_dates(text):\n",
|
||||
" return []"
|
||||
" \n",
|
||||
" date_patterns = re.compile(\n",
|
||||
" r'(?:\\b(\\d{2})[-.](\\d{2})[-.](\\d{4})\\b)|' # DD-MM-YYYY DD.MM.YYYY\n",
|
||||
" r'(?:\\b(\\d{2})[/](\\d{2})[/](\\d{4})\\b)|' # MM/DD/YYYY\n",
|
||||
" r'(?:\\b(\\d{4})[/](\\d{2})[/](\\d{2})\\b)|' # YYYY/MM/DD\n",
|
||||
" r'(?:\\b(\\d{4})[-](\\d{2})[-](\\d{2})\\b)' # YYYY-MM-DD\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" results = []\n",
|
||||
" for match in date_patterns.finditer(text):\n",
|
||||
" groups = match.groups()\n",
|
||||
" if groups[:3] != (None, None, None):\n",
|
||||
" day, month, year = groups[:3]\n",
|
||||
" elif groups[3:6] != (None, None, None):\n",
|
||||
" month, day, year = groups[3:6]\n",
|
||||
" elif groups[6:9] != (None, None, None):\n",
|
||||
" year, month, day = groups[6:9]\n",
|
||||
" elif groups[9:] != (None, None, None):\n",
|
||||
" year, month, day = groups[9:]\n",
|
||||
" \n",
|
||||
" results.append({\n",
|
||||
" \"date\": f\"{day.zfill(2)}-{month.zfill(2)}-{year}\",\n",
|
||||
" \"position\": (match.start(), match.end()),\n",
|
||||
" \"day\": day,\n",
|
||||
" \"month\": month,\n",
|
||||
" \"year\": year\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" return results"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"id": "48cf2ad7-b34d-4af6-84c8-5d941f2e323c",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[{'date': '21-03-2023', 'position': (20, 30), 'day': '21', 'month': '03', 'year': '2023'}, {'date': '21-03-2023', 'position': (32, 42), 'day': '21', 'month': '03', 'year': '2023'}, {'date': '21-03-2023', 'position': (44, 54), 'day': '21', 'month': '03', 'year': '2023'}, {'date': '21-03-2023', 'position': (56, 66), 'day': '21', 'month': '03', 'year': '2023'}, {'date': '21-03-2023', 'position': (72, 82), 'day': '21', 'month': '03', 'year': '2023'}]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"example_text = \"Important dates are 21-03-2023, 03/21/2023, 2023/03/21, 21.03.2023, and 2023-03-21.\"\n",
|
||||
"found_dates = find_dates(example_text)\n",
|
||||
"print(found_dates)"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -125,13 +240,67 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": 23,
|
||||
"id": "finished-essex",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def correct_dates(source_segment, target_segment, date_format):\n",
|
||||
" return ''"
|
||||
" format_map = {\n",
|
||||
" \"Europe\": \"{2}-{1}-{0}\", # DD-MM-YYYY\n",
|
||||
" \"US\": \"{1}/{2}/{0}\", # MM/DD/YYYY\n",
|
||||
" \"digit-dot\": \"{2}.{1}.{0}\" # DD.MM.YYYY\n",
|
||||
" }\n",
|
||||
" \n",
|
||||
" if date_format not in format_map:\n",
|
||||
" raise ValueError(\"Błędny format\")\n",
|
||||
"\n",
|
||||
" source_dates = find_dates(source_segment)\n",
|
||||
" target_dates = find_dates(target_segment)\n",
|
||||
" \n",
|
||||
" if len(source_dates) != len(target_dates):\n",
|
||||
" return f\"Liczba dat w segmencie źródłowym ({len(source_dates)}) i tłumaczeniu ({len(target_dates)}) się nie zgadza.\"\n",
|
||||
"\n",
|
||||
" new_target_segment = target_segment\n",
|
||||
" for source, target in zip(source_dates, target_dates):\n",
|
||||
" if source[\"day\"] != target[\"day\"] or source[\"month\"] != target[\"month\"] or source[\"year\"] != target[\"year\"]:\n",
|
||||
" return \"Daty się nie zgadzają\"\n",
|
||||
"\n",
|
||||
" formatted_date = format_map[date_format].format(target[\"year\"], target[\"month\"].zfill(2), target[\"day\"].zfill(2))\n",
|
||||
" new_target_segment = new_target_segment[:target[\"position\"][0]] + formatted_date + new_target_segment[target[\"position\"][1]:]\n",
|
||||
"\n",
|
||||
" return new_target_segment"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 28,
|
||||
"id": "7dfb44fd-297f-41ea-8f46-a647c98341a3",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"source_text = \"The contract starts on 2023-12-21 and ends on 2023-06-21.\"\n",
|
||||
"target_text = \"Umowa zaczyna się 21.12.2023 i kończy 21.06.2023.\"\n",
|
||||
"expected_format = \"digit-dot\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 29,
|
||||
"id": "f0922192-bb51-4194-aeea-4c41c5d195a5",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Umowa zaczyna się 21.12.2023 i kończy 21.06.2023.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"text = correct_dates(source_text, target_text, expected_format)\n",
|
||||
"print(text)"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -176,13 +345,63 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": 32,
|
||||
"id": "romance-judge",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def tokenize(text):\n",
|
||||
" tokens = re.split(r'(\\s+|<[^>]+>)', text)\n",
|
||||
" return [token for token in tokens if token.strip()]\n",
|
||||
"\n",
|
||||
"def transfer_tags(source_segment, target_segment):\n",
|
||||
" return ''"
|
||||
" source_tokens = tokenize(source_segment)\n",
|
||||
" target_tokens = [token for token in tokenize(target_segment) if not re.match(r'<[^>]+>', token)]\n",
|
||||
"\n",
|
||||
" source_tags = [(token, i) for i, token in enumerate(source_tokens) if re.match(r'<[^>]+>', token)]\n",
|
||||
"\n",
|
||||
" if len(source_tags) > 0:\n",
|
||||
" target_with_tags = []\n",
|
||||
" source_word_count = len([token for token in source_tokens if not re.match(r'<[^>]+>', token)])\n",
|
||||
" target_word_count = len(target_tokens)\n",
|
||||
" tag_index = 0\n",
|
||||
" tag_positions = [int(round(tag[1] * target_word_count / source_word_count)) for tag in source_tags]\n",
|
||||
"\n",
|
||||
" for i, token in enumerate(target_tokens):\n",
|
||||
" while tag_index < len(tag_positions) and i == tag_positions[tag_index]:\n",
|
||||
" target_with_tags.append(source_tags[tag_index][0])\n",
|
||||
" tag_index += 1\n",
|
||||
" target_with_tags.append(token)\n",
|
||||
" \n",
|
||||
" while tag_index < len(tag_positions):\n",
|
||||
" target_with_tags.append(source_tags[tag_index][0])\n",
|
||||
" tag_index += 1\n",
|
||||
"\n",
|
||||
" return ' '.join(target_with_tags)\n",
|
||||
" else:\n",
|
||||
" return ' '.join(target_tokens)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 33,
|
||||
"id": "c1ebe058-1da6-43a1-8d99-78999aefca17",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"To jest <b> pogrubienie i </b> to jest kursywa. <i> </i>\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"source_segment = \"This is <b>bold</b> and this is <i>italic</i>.\"\n",
|
||||
"target_segment = \"To jest pogrubienie i to jest kursywa.\"\n",
|
||||
"\n",
|
||||
"result = transfer_tags(source_segment, target_segment)\n",
|
||||
"print(result)"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -190,7 +409,7 @@
|
|||
"author": "Rafał Jaworski",
|
||||
"email": "rjawor@amu.edu.pl",
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
|
|
@ -205,7 +424,7 @@
|
|||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.8.10"
|
||||
"version": "3.10.12"
|
||||
},
|
||||
"subtitle": "6,7. Preprocessing i postprocessing",
|
||||
"title": "Komputerowe wspomaganie tłumaczenia",
|
||||
|
|
|
|||
265
lab/lab_08.ipynb
265
lab/lab_08.ipynb
|
|
@ -57,13 +57,53 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": 18,
|
||||
"id": "moving-clothing",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def calculate_bleu():\n",
|
||||
" return 0"
|
||||
"import zipfile\n",
|
||||
"from nltk.translate.bleu_score import corpus_bleu\n",
|
||||
"import io\n",
|
||||
"\n",
|
||||
"def calculate_bleu(zip_path):\n",
|
||||
" references = []\n",
|
||||
" candidates = []\n",
|
||||
"\n",
|
||||
" with zipfile.ZipFile(zip_path, 'r') as z:\n",
|
||||
" with z.open('EMEA.en-pl.en', 'r') as file_ref, z.open('EMEA.en-pl.pl', 'r') as file_trans:\n",
|
||||
" ref_buffer = io.TextIOWrapper(file_ref, encoding='utf-8')\n",
|
||||
" trans_buffer = io.TextIOWrapper(file_trans, encoding='utf-8')\n",
|
||||
" \n",
|
||||
" for ref_line, trans_line in zip(ref_buffer, trans_buffer):\n",
|
||||
" ref_tokens = [ref_line.strip().split()]\n",
|
||||
" trans_tokens = trans_line.strip().split()\n",
|
||||
" \n",
|
||||
" references.append(ref_tokens)\n",
|
||||
" candidates.append(trans_tokens)\n",
|
||||
"\n",
|
||||
" score = corpus_bleu(references, candidates)\n",
|
||||
" return score"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 19,
|
||||
"id": "b312fc2a-8d95-4eb5-a49f-a8b0707be8bf",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Ocena BLEU: 0.05086746137866238\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"zip_file_path = 'korpusy/emea.zip'\n",
|
||||
"bleu_score = calculate_bleu(zip_file_path)\n",
|
||||
"print(f\"Ocena BLEU: {bleu_score}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -76,13 +116,59 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": 20,
|
||||
"id": "lasting-rolling",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def analyze_bleu():\n",
|
||||
" return []"
|
||||
"def analyze_bleu(zip_path, start_end_pairs):\n",
|
||||
" results = {}\n",
|
||||
" \n",
|
||||
" with zipfile.ZipFile(zip_path, 'r') as z:\n",
|
||||
" with z.open('EMEA.en-pl.en', 'r') as file_ref, z.open('EMEA.en-pl.pl', 'r') as file_trans:\n",
|
||||
" ref_buffer = io.TextIOWrapper(file_ref, encoding='utf-8')\n",
|
||||
" trans_buffer = io.TextIOWrapper(file_trans, encoding='utf-8')\n",
|
||||
" \n",
|
||||
" references_full = [line.strip().split() for line in ref_buffer]\n",
|
||||
" candidates_full = [line.strip().split() for line in trans_buffer]\n",
|
||||
"\n",
|
||||
" for label, (start, end) in start_end_pairs.items():\n",
|
||||
"\n",
|
||||
" references_segment = [references_full[i] for i in range(start, min(end, len(references_full)))]\n",
|
||||
" candidates_segment = [candidates_full[i] for i in range(start, min(end, len(candidates_full)))]\n",
|
||||
" \n",
|
||||
" references_segment = [[ref] for ref in references_segment]\n",
|
||||
"\n",
|
||||
" score = corpus_bleu(references_segment, candidates_segment)\n",
|
||||
" results[label] = score\n",
|
||||
" \n",
|
||||
" return results"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 21,
|
||||
"id": "8c218410-048d-40fb-b609-9553f8dae28b",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Ocena BLEU dla First 100 sentences: 0.03940935286156434\n",
|
||||
"Ocena BLEU dla Sentences 500-600: 1.9718207266585256e-155\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"fragments = {\n",
|
||||
" \"First 100 sentences\": (0, 100),\n",
|
||||
" \"Sentences 500-600\": (500, 600)\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"bleu_scores = analyze_bleu(zip_file_path, fragments)\n",
|
||||
"for label, score in bleu_scores.items():\n",
|
||||
" print(f\"Ocena BLEU dla {label}: {score}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -120,13 +206,91 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": 36,
|
||||
"id": "occupied-swing",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def calculate_wer():\n",
|
||||
" return 0"
|
||||
"import jiwer\n",
|
||||
"\n",
|
||||
"def calculate_wer(zip_path):\n",
|
||||
" def wer(reference, hypothesis):\n",
|
||||
" ref_words = reference.split()\n",
|
||||
" hyp_words = hypothesis.split()\n",
|
||||
" R = len(ref_words)\n",
|
||||
" H = len(hyp_words)\n",
|
||||
" cost_matrix = [[0] * (H + 1) for _ in range(R + 1)]\n",
|
||||
"\n",
|
||||
" for i in range(1, R + 1):\n",
|
||||
" cost_matrix[i][0] = i\n",
|
||||
" for j in range(1, H + 1):\n",
|
||||
" cost_matrix[0][j] = j\n",
|
||||
"\n",
|
||||
" for i in range(1, R + 1):\n",
|
||||
" for j in range(1, H + 1):\n",
|
||||
" substitution_cost = 0 if ref_words[i - 1] == hyp_words[j - 1] else 1\n",
|
||||
" cost_matrix[i][j] = min(\n",
|
||||
" cost_matrix[i - 1][j] + 1,\n",
|
||||
" cost_matrix[i][j - 1] + 1,\n",
|
||||
" cost_matrix[i - 1][j - 1] + substitution_cost\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" i, j = R, H\n",
|
||||
" substitutions = insertions = deletions = correct = 0\n",
|
||||
" while i > 0 and j > 0:\n",
|
||||
" if ref_words[i - 1] == hyp_words[j - 1]:\n",
|
||||
" correct += 1\n",
|
||||
" i -= 1\n",
|
||||
" j -= 1\n",
|
||||
" elif cost_matrix[i][j] == cost_matrix[i - 1][j - 1] + 1:\n",
|
||||
" substitutions += 1\n",
|
||||
" i -= 1\n",
|
||||
" j -= 1\n",
|
||||
" elif cost_matrix[i][j] == cost_matrix[i][j - 1] + 1:\n",
|
||||
" insertions += 1\n",
|
||||
" j -= 1\n",
|
||||
" else:\n",
|
||||
" deletions += 1\n",
|
||||
" i -= 1\n",
|
||||
"\n",
|
||||
" N = substitutions + deletions + correct\n",
|
||||
" WER = (substitutions + deletions + insertions) / N if N > 0 else 0\n",
|
||||
" return WER\n",
|
||||
"\n",
|
||||
" total_wer = 0\n",
|
||||
" num_sentences = 0\n",
|
||||
"\n",
|
||||
" with zipfile.ZipFile(zip_path, 'r') as z:\n",
|
||||
" with z.open('EMEA.en-pl.en', 'r') as file_ref, z.open('EMEA.en-pl.pl', 'r') as file_trans:\n",
|
||||
" ref_buffer = io.TextIOWrapper(file_ref, encoding='utf-8')\n",
|
||||
" trans_buffer = io.TextIOWrapper(file_trans, encoding='utf-8')\n",
|
||||
"\n",
|
||||
" for ref_line, hyp_line in zip(ref_buffer, trans_buffer):\n",
|
||||
" total_wer += wer(ref_line.strip(), hyp_line.strip())\n",
|
||||
" num_sentences += 1\n",
|
||||
"\n",
|
||||
" average_wer = total_wer / num_sentences if num_sentences > 0 else 0\n",
|
||||
" return average_wer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 37,
|
||||
"id": "ceccd005-714b-4dfb-b210-d13a9c5238c9",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"WER (Word Error Rate): 82.13%\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"zip_file_path = 'korpusy/emea.zip'\n",
|
||||
"wer_result = calculate_wer(zip_file_path)\n",
|
||||
"print(f\"WER (Word Error Rate): {wer_result:.2%}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -147,13 +311,48 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": 41,
|
||||
"id": "immediate-element",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def calculate_levenshtein():\n",
|
||||
" return 0"
|
||||
"import Levenshtein as lev\n",
|
||||
"\n",
|
||||
"def calculate_levenshtein(zip_path):\n",
|
||||
" total_distance = 0\n",
|
||||
" num_sentences = 0\n",
|
||||
"\n",
|
||||
" with zipfile.ZipFile(zip_path, 'r') as z:\n",
|
||||
" with z.open('EMEA.en-pl.en', 'r') as file_ref, z.open('EMEA.en-pl.pl', 'r') as file_trans:\n",
|
||||
" ref_buffer = io.TextIOWrapper(file_ref, encoding='utf-8')\n",
|
||||
" trans_buffer = io.TextIOWrapper(file_trans, encoding='utf-8')\n",
|
||||
"\n",
|
||||
" for ref_line, hyp_line in zip(ref_buffer, trans_buffer):\n",
|
||||
" distance = lev.distance(ref_line.strip(), hyp_line.strip())\n",
|
||||
" total_distance += distance\n",
|
||||
" num_sentences += 1\n",
|
||||
"\n",
|
||||
" average_distance = total_distance / num_sentences if num_sentences > 0 else 0\n",
|
||||
" return average_distance"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 43,
|
||||
"id": "a4581547-7219-4a8d-913d-e0e4fa4d0914",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Średnia wartość dystancu Levenshteina: 61.29\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"average_distance = calculate_levenshtein(zip_file_path)\n",
|
||||
"print(f\"Średnia wartość dystancu Levenshteina: {average_distance:.2f}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -177,28 +376,45 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "descending-easter",
|
||||
"execution_count": null,
|
||||
"id": "c0e3f109-795d-4844-b41d-9e0b570577c5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def analyze_translations():\n",
|
||||
" return []"
|
||||
"from PyDictionary import PyDictionary\n",
|
||||
"\n",
|
||||
"def translate_corpus(zip_path, filename):\n",
|
||||
" dictionary = PyDictionary()\n",
|
||||
" translations = {}\n",
|
||||
"\n",
|
||||
" with zipfile.ZipFile(zip_path, 'r') as z:\n",
|
||||
" with z.open(filename, 'r') as file:\n",
|
||||
" buffer = io.TextIOWrapper(file, encoding='utf-8')\n",
|
||||
"\n",
|
||||
" for line in buffer:\n",
|
||||
" words = line.strip().split()\n",
|
||||
" for word in words:\n",
|
||||
" if word not in translations:\n",
|
||||
" try:\n",
|
||||
" german_translation = dictionary.translate(word, \"German\")\n",
|
||||
" translations[word] = german_translation\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"Error translating {word}: {str(e)}\")\n",
|
||||
" translations[word] = None\n",
|
||||
"\n",
|
||||
" return translations"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"author": "Rafał Jaworski",
|
||||
"email": "rjawor@amu.edu.pl",
|
||||
"lang": "pl",
|
||||
"subtitle": "8. Wykorzystanie tłumaczenia automatycznego we wspomaganiu tłumaczenia",
|
||||
"title": "Komputerowe wspomaganie tłumaczenia",
|
||||
"year": "2021",
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"lang": "pl",
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
|
|
@ -209,8 +425,11 @@
|
|||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.8.10"
|
||||
}
|
||||
"version": "3.10.12"
|
||||
},
|
||||
"subtitle": "8. Wykorzystanie tłumaczenia automatycznego we wspomaganiu tłumaczenia",
|
||||
"title": "Komputerowe wspomaganie tłumaczenia",
|
||||
"year": "2021"
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
|
|
|
|||
1016
lab/lab_09-10.ipynb
1016
lab/lab_09-10.ipynb
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue