bfijalkowski/KWT-2024
1
1
Fork 8
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

merge into: bfijalkowski:main
Branches Tags
bfijalkowski:main
s495726:main
s495726:wip
...
pull from: s495726:main
Branches Tags
s495726:main
s495726:wip
bfijalkowski:main

8 Commits
main ... main

Author SHA1 Message Date
Patryk
9068e7e85d lab 15 2024-06-09 18:22:19 +02:00
Patryk
a3dca39152 lab 11-14 2024-05-28 23:44:55 +02:00
Patryk
824f7d373d Merge branch 'wip' 
# Conflicts:
#	lab/lab_02.ipynb
 2024-05-27 00:55:27 +02:00
Patryk
6a0efac373 lab 09-10 2024-05-27 00:53:56 +02:00
Patryk Osiński
78982a4f21 wip 2024-04-20 19:58:36 +02:00
Patryk
9b9e46df22 lab 3 2024-04-16 21:12:25 +02:00
Patryk
2b22583359 lab 2 2024-04-16 08:47:38 +02:00
Patryk Osiński
ddd2833663 lab 1 2024-04-13 14:22:23 +02:00
10 changed files with 906 additions and 289 deletions
Show Stats Expand all files Collapse all files

222
lab/lab_01.ipynb
View File

@ -52,9 +52,14 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 1, "execution_count": 5,
"id": "narrow-romantic", "id": "narrow-romantic",
"metadata": {}, "metadata": {
"ExecuteTime": {
"end_time": "2024-04-13T11:05:09.046685900Z",
"start_time": "2024-04-13T11:05:08.877692800Z"
}
},
"outputs": [], "outputs": [],
"source": [ "source": [
"translation_memory = [('Wciśnij przycisk Enter', 'Press the ENTER button'), \n", "translation_memory = [('Wciśnij przycisk Enter', 'Press the ENTER button'), \n",
@ -71,9 +76,14 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 2, "execution_count": 6,
"id": "indonesian-electron", "id": "indonesian-electron",
"metadata": {}, "metadata": {
"ExecuteTime": {
"end_time": "2024-04-13T11:05:09.131296300Z",
"start_time": "2024-04-13T11:05:08.893315Z"
}
},
"outputs": [], "outputs": [],
"source": [ "source": [
"def tm_lookup(sentence):\n", "def tm_lookup(sentence):\n",
@ -82,9 +92,14 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 3, "execution_count": 7,
"id": "compact-trinidad", "id": "compact-trinidad",
"metadata": {}, "metadata": {
"ExecuteTime": {
"end_time": "2024-04-13T11:05:09.162547Z",
"start_time": "2024-04-13T11:05:08.924558500Z"
}
},
"outputs": [ "outputs": [
{ {
"data": { "data": {
@ -92,7 +107,7 @@
"['Press the ENTER button']" "['Press the ENTER button']"
] ]
}, },
"execution_count": 3, "execution_count": 7,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -119,9 +134,14 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 4, "execution_count": 8,
"id": "exposed-daniel", "id": "exposed-daniel",
"metadata": {}, "metadata": {
"ExecuteTime": {
"end_time": "2024-04-13T11:05:09.162547Z",
"start_time": "2024-04-13T11:05:08.946722400Z"
}
},
"outputs": [], "outputs": [],
"source": [ "source": [
"translation_memory.append(('Drukarka jest wyłączona', 'The printer is switched off'))\n", "translation_memory.append(('Drukarka jest wyłączona', 'The printer is switched off'))\n",
@ -139,9 +159,14 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 5, "execution_count": 9,
"id": "serial-velvet", "id": "serial-velvet",
"metadata": {}, "metadata": {
"ExecuteTime": {
"end_time": "2024-04-13T11:05:09.162547Z",
"start_time": "2024-04-13T11:05:08.955053700Z"
}
},
"outputs": [ "outputs": [
{ {
"data": { "data": {
@ -149,7 +174,7 @@
"['Press the ENTER button', 'Press the ENTER key']" "['Press the ENTER button', 'Press the ENTER key']"
] ]
}, },
"execution_count": 5, "execution_count": 9,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -176,9 +201,14 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 6, "execution_count": 10,
"id": "every-gibson", "id": "every-gibson",
"metadata": {}, "metadata": {
"ExecuteTime": {
"end_time": "2024-04-13T11:05:09.178168700Z",
"start_time": "2024-04-13T11:05:08.970677700Z"
}
},
"outputs": [ "outputs": [
{ {
"data": { "data": {
@ -186,7 +216,7 @@
"[]" "[]"
] ]
}, },
"execution_count": 6, "execution_count": 10,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -213,13 +243,19 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 7, "execution_count": 21,
"id": "protected-rings", "id": "protected-rings",
"metadata": {}, "metadata": {
"ExecuteTime": {
"end_time": "2024-04-13T11:05:12.496455200Z",
"start_time": "2024-04-13T11:05:12.465209700Z"
}
},
"outputs": [], "outputs": [],
"source": [ "source": [
"def tm_lookup(sentence):\n", "def tm_lookup(sentence):\n",
" return ''" " sentence = sentence.lower()\n",
" return [entry[1] for entry in translation_memory if entry[0].lower() == sentence]"
] ]
}, },
{ {
@ -232,17 +268,22 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 18, "execution_count": 22,
"id": "severe-alloy", "id": "severe-alloy",
"metadata": {}, "metadata": {
"ExecuteTime": {
"end_time": "2024-04-13T11:05:14.153976900Z",
"start_time": "2024-04-13T11:05:14.120474700Z"
}
},
"outputs": [ "outputs": [
{ {
"data": { "data": {
"text/plain": [ "text/plain": [
"''" "[]"
] ]
}, },
"execution_count": 18, "execution_count": 22,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -261,13 +302,24 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 11, "execution_count": 23,
"id": "structural-diesel", "id": "structural-diesel",
"metadata": {}, "metadata": {
"ExecuteTime": {
"end_time": "2024-04-13T11:05:15.199517300Z",
"start_time": "2024-04-13T11:05:15.105892400Z"
}
},
"outputs": [], "outputs": [],
"source": [ "source": [
"import string\n",
"\n",
"def normalize(sentence):\n",
" return sentence.translate(str.maketrans('', '', string.punctuation)).lower()\n",
"\n",
"def tm_lookup(sentence):\n", "def tm_lookup(sentence):\n",
" return ''" " sentence = normalize(sentence)\n",
" return [entry[1] for entry in translation_memory if normalize(entry[0]) == sentence]"
] ]
}, },
{ {
@ -280,17 +332,22 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 12, "execution_count": 24,
"id": "brief-senegal", "id": "brief-senegal",
"metadata": {}, "metadata": {
"ExecuteTime": {
"end_time": "2024-04-13T11:05:17.857048100Z",
"start_time": "2024-04-13T11:05:17.825799600Z"
}
},
"outputs": [ "outputs": [
{ {
"data": { "data": {
"text/plain": [ "text/plain": [
"''" "[]"
] ]
}, },
"execution_count": 12, "execution_count": 24,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -317,13 +374,49 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 14, "execution_count": 25,
"id": "mathematical-customs", "id": "mathematical-customs",
"metadata": {}, "metadata": {
"ExecuteTime": {
"end_time": "2024-04-13T12:00:14.223561700Z",
"start_time": "2024-04-13T12:00:14.159559100Z"
}
},
"outputs": [], "outputs": [],
"source": [ "source": [
"def find_similar(sentence):\n",
" mismatches_threshold = 2\n",
" words = sentence.split()\n",
" words_count = len(words)\n",
" for entry in translation_memory:\n",
" entry_words = normalize(entry[0]).split()\n",
" if words_count != len(entry_words):\n",
" continue\n",
" mismatches = 0\n",
" i = 0\n",
" for word in words:\n",
" if word != entry_words[i]:\n",
" if mismatches < mismatches_threshold:\n",
" mismatches += 1\n",
" else:\n",
" break\n",
" i += 1\n",
" if mismatches < mismatches_threshold:\n",
" return entry[1]\n",
" return []\n",
"\n",
"\n",
"def find_exact(sentence):\n",
" return [entry[1] for entry in translation_memory if normalize(entry[0]) == sentence]\n",
"\n",
"\n",
"def tm_lookup(sentence):\n", "def tm_lookup(sentence):\n",
" return ''" " sentence = normalize(sentence)\n",
" exact_match = find_exact(sentence)\n",
" if not exact_match:\n",
" return find_similar(sentence)\n",
" else:\n",
" return exact_match"
] ]
}, },
{ {
@ -344,9 +437,14 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 15, "execution_count": 26,
"id": "humanitarian-wrong", "id": "humanitarian-wrong",
"metadata": {}, "metadata": {
"ExecuteTime": {
"end_time": "2024-04-13T12:00:18.016836500Z",
"start_time": "2024-04-13T12:00:17.992836400Z"
}
},
"outputs": [], "outputs": [],
"source": [ "source": [
"glossary = [('komputer', 'computer'), ('przycisk', 'button'), ('drukarka', 'printer')]" "glossary = [('komputer', 'computer'), ('przycisk', 'button'), ('drukarka', 'printer')]"
@ -362,9 +460,14 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 16, "execution_count": 27,
"id": "located-perception", "id": "located-perception",
"metadata": {}, "metadata": {
"ExecuteTime": {
"end_time": "2024-04-13T12:02:06.039160400Z",
"start_time": "2024-04-13T12:02:06.015160400Z"
}
},
"outputs": [], "outputs": [],
"source": [ "source": [
"def glossary_lookup(sentence):\n", "def glossary_lookup(sentence):\n",
@ -374,9 +477,14 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 17, "execution_count": 28,
"id": "advised-casting", "id": "advised-casting",
"metadata": {}, "metadata": {
"ExecuteTime": {
"end_time": "2024-04-13T12:02:06.846998600Z",
"start_time": "2024-04-13T12:02:06.823447800Z"
}
},
"outputs": [ "outputs": [
{ {
"data": { "data": {
@ -384,7 +492,7 @@
"[('przycisk', 'button'), ('drukarka', 'printer')]" "[('przycisk', 'button'), ('drukarka', 'printer')]"
] ]
}, },
"execution_count": 17, "execution_count": 28,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -406,7 +514,9 @@
"id": "defensive-fifteen", "id": "defensive-fifteen",
"metadata": {}, "metadata": {},
"source": [ "source": [
"Odpowiedź:" "Odpowiedź: \n",
"złożoność pesymistyczna: m*n\n",
"złożoność optymistyczna: m"
] ]
}, },
{ {
@ -421,11 +531,17 @@
"cell_type": "code", "cell_type": "code",
"execution_count": 19, "execution_count": 19,
"id": "original-tunisia", "id": "original-tunisia",
"metadata": {}, "metadata": {
"ExecuteTime": {
"end_time": "2024-04-13T11:05:09.247171300Z",
"start_time": "2024-04-13T11:05:09.124790700Z"
}
},
"outputs": [], "outputs": [],
"source": [ "source": [
"def glossary_lookup(sentence):\n", "def glossary_lookup(sentence):\n",
" return ''" " sentence_words = sentence.lower().split()\n",
" return [entry for entry in glossary if entry[0].lower() in sentence_words]"
] ]
}, },
{ {
@ -440,11 +556,25 @@
"cell_type": "code", "cell_type": "code",
"execution_count": 20, "execution_count": 20,
"id": "adolescent-semiconductor", "id": "adolescent-semiconductor",
"metadata": {}, "metadata": {
"ExecuteTime": {
"end_time": "2024-04-13T11:05:09.247171300Z",
"start_time": "2024-04-13T11:05:09.146924500Z"
}
},
"outputs": [], "outputs": [],
"source": [ "source": [
"def glossary_lookup(sentence):\n", "def glossary_lookup(sentence):\n",
" return ''" " sentence_words = sentence.lower().split()\n",
" entry_words = []\n",
" for entry in glossary:\n",
" entry_words.append((entry[0].lower(), entry[1]))\n",
" result = []\n",
" for word in sentence_words:\n",
" for entry_word in entry_words:\n",
" if entry_word[0] == word:\n",
" result.append(entry_word)\n",
" return result"
] ]
} }
], ],
@ -452,7 +582,7 @@
"author": "Rafał Jaworski", "author": "Rafał Jaworski",
"email": "rjawor@amu.edu.pl", "email": "rjawor@amu.edu.pl",
"kernelspec": { "kernelspec": {
"display_name": "Python 3", "display_name": "Python 3 (ipykernel)",
"language": "python", "language": "python",
"name": "python3" "name": "python3"
}, },
@ -467,7 +597,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.8.10" "version": "3.9.2"
}, },
"subtitle": "1. Podstawowe techniki wspomagania tłumaczenia", "subtitle": "1. Podstawowe techniki wspomagania tłumaczenia",
"title": "Komputerowe wspomaganie tłumaczenia", "title": "Komputerowe wspomaganie tłumaczenia",

92
lab/lab_02.ipynb
View File

@ -40,9 +40,11 @@
] ]
}, },
{ {
"cell_type": "markdown", "cell_type": "code",
"id": "existing-approval", "execution_count": null,
"id": "961796fd-4463-4a17-ac15-afe712b3959e",
"metadata": {}, "metadata": {},
"outputs": [],
"source": [ "source": [
"Jedną z funkcji dostępnych we wszystkich większych programach do wspomagania tłumaczenia jest znajdowanie bardzo pewnych dopasowań w pamięci tłumaczeń. Są one zwane **ICE** (In-Context Exact match) lub 101% match. Są to takie dopasowania z pamięci tłumaczeń, dla których nie tylko zdanie źródłowe z TM jest identyczne z tłumaczonym, ale także poprzednie zdanie źródłowe z TM zgadza się z poprzednim zdaniem tłumaczonym oraz następne z TM z następnym tłumaczonym." "Jedną z funkcji dostępnych we wszystkich większych programach do wspomagania tłumaczenia jest znajdowanie bardzo pewnych dopasowań w pamięci tłumaczeń. Są one zwane **ICE** (In-Context Exact match) lub 101% match. Są to takie dopasowania z pamięci tłumaczeń, dla których nie tylko zdanie źródłowe z TM jest identyczne z tłumaczonym, ale także poprzednie zdanie źródłowe z TM zgadza się z poprzednim zdaniem tłumaczonym oraz następne z TM z następnym tłumaczonym."
] ]
@ -85,8 +87,31 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"def exact_match(sentence):\n",
" for key, entry in enumerate(translation_memory):\n",
" if entry[0] == sentence:\n",
" return key, entry[1]\n",
" return None, None\n",
"\n",
"\n",
"def has_exact_match_on_index(index, sentence):\n",
" return translation_memory[index][0] == sentence\n",
"\n",
"\n",
"def ice_lookup(sentence, prev_sentence, next_sentence):\n", "def ice_lookup(sentence, prev_sentence, next_sentence):\n",
" return []" " index, match = exact_match(sentence)\n",
" trans_length = len(translation_memory)\n",
" if index is None:\n",
" return []\n",
" if next_sentence \\\n",
" and index < trans_length \\\n",
" and not has_exact_match_on_index(index + 1, next_sentence):\n",
" return []\n",
" if prev_sentence \\\n",
" and index > 0 \\\n",
" and not has_exact_match_on_index(index - 1, prev_sentence):\n",
" return []\n",
" return [match]"
] ]
}, },
{ {
@ -141,7 +166,7 @@
"id": "graduate-theorem", "id": "graduate-theorem",
"metadata": {}, "metadata": {},
"source": [ "source": [
"Odpowiedź:" "Odpowiedź: Nie. 1, 3, 4."
] ]
}, },
{ {
@ -179,7 +204,7 @@
"id": "metallic-leave", "id": "metallic-leave",
"metadata": {}, "metadata": {},
"source": [ "source": [
"Odpowiedź:" "Odpowiedź: Tak. 1, 2, 3, 4."
] ]
}, },
{ {
@ -206,7 +231,17 @@
"id": "bibliographic-stopping", "id": "bibliographic-stopping",
"metadata": {}, "metadata": {},
"source": [ "source": [
"Odpowiedź:" "Odpowiedź: Tak.\n",
"1. Liczba operacji wykonanych nie może być ujemna.\n",
"2. Gdy x == y, nie są wymagane żadne operacje edycyjne, więc wynik funkcji to 0.\n",
"3. Zmiana jednego łańcucha znaków w drugi, wymaga tyle samo operacji edycji, co zmiana drugiego w pierwszy.\n",
" Studia -> Studiel = 2; Studiel -> Studia = 2; 2 == 2\n",
"4. Istnieją trzy opcje\n",
" - Jeżeli x == y == z, więc 0 + 0 == 0\n",
" - Jeżeli x == y, x != z, a x -> z = n, to y -> z = n więc albo 0 + n == n, albo n + n > 0\n",
" - Jeżeli x != y != z to im z jest bliżej do x, tym jest dalej od y (jednostką odległości jest liczba przekształceń). Można by to przedstawić graficznie jako trójkąt (x, y, z). z stanowi punkt na pośredniej drodze pomiędzy x i y, która nie może być dłuższa niż droga bezpośrednia - wynika to z własności trójkąta.\n",
" Studia -> Studiel = 2; Studiel -> udia = 4; udia -> Studia = 2;\n",
" 2 + 4 > 2; 2 + 2 == 4"
] ]
}, },
{ {
@ -214,6 +249,7 @@
"id": "attended-channels", "id": "attended-channels",
"metadata": {}, "metadata": {},
"source": [ "source": [
"\n",
"W Pythonie dostępna jest biblioteka zawierająca implementację dystansu Levenshteina. Zainstaluj ją w swoim systemie przy użyciu polecenia:\n", "W Pythonie dostępna jest biblioteka zawierająca implementację dystansu Levenshteina. Zainstaluj ją w swoim systemie przy użyciu polecenia:\n",
"\n", "\n",
"`pip3 install python-Levenshtein`\n", "`pip3 install python-Levenshtein`\n",
@ -223,21 +259,10 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 5, "execution_count": null,
"id": "secondary-wrist", "id": "355e4914-08da-4bd4-b8a2-67b055831c30",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"data": {
"text/plain": [
"2"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [ "source": [
"from Levenshtein import distance as levenshtein_distance\n", "from Levenshtein import distance as levenshtein_distance\n",
"\n", "\n",
@ -314,22 +339,9 @@
] ]
}, },
{ {
"cell_type": "code", "cell_type": "raw",
"execution_count": 9, "id": "4a47854f-df2e-451f-8e09-99f59210f86f",
"id": "invisible-cambodia",
"metadata": {}, "metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.631578947368421"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [ "source": [
"levenshtein_similarity('Spróbuj wyłączyć i włączyć komputer', 'Nie próbuj wyłączać i włączać drukarki')" "levenshtein_similarity('Spróbuj wyłączyć i włączyć komputer', 'Nie próbuj wyłączać i włączać drukarki')"
] ]
@ -350,7 +362,11 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"def fuzzy_lookup(sentence, threshold):\n", "def fuzzy_lookup(sentence, threshold):\n",
" return []" " results = []\n",
" for entry in translation_memory:\n",
" if levenshtein_similarity(entry[0], sentence) >= threshold:\n",
" results.append(entry[1])\n",
" return results"
] ]
} }
], ],
@ -358,7 +374,7 @@
"author": "Rafał Jaworski", "author": "Rafał Jaworski",
"email": "rjawor@amu.edu.pl", "email": "rjawor@amu.edu.pl",
"kernelspec": { "kernelspec": {
"display_name": "Python 3", "display_name": "Python 3 (ipykernel)",
"language": "python", "language": "python",
"name": "python3" "name": "python3"
}, },
@ -373,7 +389,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.8.10" "version": "3.9.2"
}, },
"subtitle": "2. Zaawansowane użycie pamięci tłumaczeń", "subtitle": "2. Zaawansowane użycie pamięci tłumaczeń",
"title": "Komputerowe wspomaganie tłumaczenia", "title": "Komputerowe wspomaganie tłumaczenia",

189
lab/lab_03.ipynb
View File

@ -63,7 +63,7 @@
"id": "diverse-sunglasses", "id": "diverse-sunglasses",
"metadata": {}, "metadata": {},
"source": [ "source": [
"Odpowiedź:" "Odpowiedź: \"metal cabinet guides\". https://translate.google.pl/"
] ]
}, },
{ {
@ -115,7 +115,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"dictionary = ['program', 'application', 'applet' 'compile']" "dictionary = ['program', 'application', 'applet', 'compile']"
] ]
}, },
{ {
@ -133,8 +133,18 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"import re\n",
"\n",
"def terminology_lookup():\n", "def terminology_lookup():\n",
" return []" " result = []\n",
" regex = ''\n",
" for word in dictionary:\n",
" if regex != '':\n",
" regex += '|'\n",
" regex += '(' + word + ')'\n",
" for occurrence in re.finditer(regex, text, re.I):\n",
" result.append((occurrence.group(), occurrence.start(), occurrence.end()))\n",
" return result"
] ]
}, },
{ {
@ -161,116 +171,34 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 4, "execution_count": 1,
"id": "tribal-attention", "id": "tribal-attention",
"metadata": {}, "metadata": {
"ExecuteTime": {
"end_time": "2024-04-20T15:23:32.727687100Z",
"start_time": "2024-04-20T15:23:24.826454500Z"
}
},
"outputs": [ "outputs": [
{ {
"name": "stdout", "ename": "KeyboardInterrupt",
"output_type": "stream", "evalue": "",
"text": [ "output_type": "error",
" \n", "traceback": [
"for\n", "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
"all\n", "\u001B[1;31mKeyboardInterrupt\u001B[0m Traceback (most recent call last)",
"Java\n", "Cell \u001B[1;32mIn[1], line 1\u001B[0m\n\u001B[1;32m----> 1\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m \u001B[38;5;21;01mspacy\u001B[39;00m\n\u001B[0;32m 2\u001B[0m nlp \u001B[38;5;241m=\u001B[39m spacy\u001B[38;5;241m.\u001B[39mload(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124men_core_web_sm\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[0;32m 4\u001B[0m doc \u001B[38;5;241m=\u001B[39m nlp(text)\n",
"programmer\n", "File \u001B[1;32mj:\\.AppData\\Python\\Python310\\site-packages\\spacy\\__init__.py:13\u001B[0m\n\u001B[0;32m 10\u001B[0m \u001B[38;5;66;03m# These are imported as part of the API\u001B[39;00m\n\u001B[0;32m 11\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mthinc\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mapi\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m Config, prefer_gpu, require_cpu, require_gpu \u001B[38;5;66;03m# noqa: F401\u001B[39;00m\n\u001B[1;32m---> 13\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m pipeline \u001B[38;5;66;03m# noqa: F401\u001B[39;00m\n\u001B[0;32m 14\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m util\n\u001B[0;32m 15\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mabout\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m __version__ \u001B[38;5;66;03m# noqa: F401\u001B[39;00m\n",
":\n", "File \u001B[1;32mj:\\.AppData\\Python\\Python310\\site-packages\\spacy\\pipeline\\__init__.py:1\u001B[0m\n\u001B[1;32m----> 1\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mattributeruler\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m AttributeRuler\n\u001B[0;32m 2\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mdep_parser\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m DependencyParser\n\u001B[0;32m 3\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01medit_tree_lemmatizer\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m EditTreeLemmatizer\n",
"this\n", "File \u001B[1;32mj:\\.AppData\\Python\\Python310\\site-packages\\spacy\\pipeline\\attributeruler.py:8\u001B[0m\n\u001B[0;32m 6\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m util\n\u001B[0;32m 7\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01merrors\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m Errors\n\u001B[1;32m----> 8\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mlanguage\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m Language\n\u001B[0;32m 9\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mmatcher\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m Matcher\n\u001B[0;32m 10\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mscorer\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m Scorer\n",
"section\n", "File \u001B[1;32mj:\\.AppData\\Python\\Python310\\site-packages\\spacy\\language.py:43\u001B[0m\n\u001B[0;32m 41\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mlang\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mtokenizer_exceptions\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m BASE_EXCEPTIONS, URL_MATCH\n\u001B[0;32m 42\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mlookups\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m load_lookups\n\u001B[1;32m---> 43\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mpipe_analysis\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m analyze_pipes, print_pipe_analysis, validate_attrs\n\u001B[0;32m 44\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mschemas\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m (\n\u001B[0;32m 45\u001B[0m ConfigSchema,\n\u001B[0;32m 46\u001B[0m ConfigSchemaInit,\n\u001B[1;32m (...)\u001B[0m\n\u001B[0;32m 49\u001B[0m validate_init_settings,\n\u001B[0;32m 50\u001B[0m )\n\u001B[0;32m 51\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mscorer\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m Scorer\n",
"explain\n", "File \u001B[1;32mj:\\.AppData\\Python\\Python310\\site-packages\\spacy\\pipe_analysis.py:6\u001B[0m\n\u001B[0;32m 3\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mwasabi\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m msg\n\u001B[0;32m 5\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01merrors\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m Errors\n\u001B[1;32m----> 6\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mtokens\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m Doc, Span, Token\n\u001B[0;32m 7\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mutil\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m dot_to_dict\n\u001B[0;32m 9\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m TYPE_CHECKING:\n\u001B[0;32m 10\u001B[0m \u001B[38;5;66;03m# This lets us add type hints for mypy etc. without causing circular imports\u001B[39;00m\n",
"how\n", "File \u001B[1;32mj:\\.AppData\\Python\\Python310\\site-packages\\spacy\\tokens\\__init__.py:1\u001B[0m\n\u001B[1;32m----> 1\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m_serialize\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m DocBin\n\u001B[0;32m 2\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mdoc\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m Doc\n\u001B[0;32m 3\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mmorphanalysis\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m MorphAnalysis\n",
"to\n", "File \u001B[1;32mj:\\.AppData\\Python\\Python310\\site-packages\\spacy\\tokens\\_serialize.py:14\u001B[0m\n\u001B[0;32m 12\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01merrors\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m Errors\n\u001B[0;32m 13\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mutil\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m SimpleFrozenList, ensure_path\n\u001B[1;32m---> 14\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mvocab\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m Vocab\n\u001B[0;32m 15\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m_dict_proxies\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m SpanGroups\n\u001B[0;32m 16\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mdoc\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m DOCBIN_ALL_ATTRS \u001B[38;5;28;01mas\u001B[39;00m ALL_ATTRS\n",
"compile\n", "File \u001B[1;32mj:\\.AppData\\Python\\Python310\\site-packages\\spacy\\vocab.pyx:1\u001B[0m, in \u001B[0;36minit spacy.vocab\u001B[1;34m()\u001B[0m\n",
"and\n", "File \u001B[1;32mj:\\.AppData\\Python\\Python310\\site-packages\\spacy\\tokens\\doc.pyx:1\u001B[0m, in \u001B[0;36minit spacy.tokens.doc\u001B[1;34m()\u001B[0m\n",
"run\n", "File \u001B[1;32m<frozen importlib._bootstrap>:404\u001B[0m, in \u001B[0;36mparent\u001B[1;34m(self)\u001B[0m\n",
"a\n", "\u001B[1;31mKeyboardInterrupt\u001B[0m: "
"swing\n",
"application\n",
"from\n",
"the\n",
"command\n",
"line\n",
".\n",
"for\n",
"information\n",
"on\n",
"compile\n",
"and\n",
"run\n",
"a\n",
"swing\n",
"application\n",
"use\n",
"NetBeans\n",
"IDE\n",
",\n",
"see\n",
"Running\n",
"Tutorial\n",
"Examples\n",
"in\n",
"NetBeans\n",
"IDE\n",
".\n",
"the\n",
"compilation\n",
"instruction\n",
"work\n",
"for\n",
"all\n",
"swing\n",
"program\n",
"—\n",
"applet\n",
",\n",
"as\n",
"well\n",
"as\n",
"application\n",
".\n",
"here\n",
"be\n",
"the\n",
"step\n",
"-PRON-\n",
"need\n",
"to\n",
"follow\n",
":\n",
"install\n",
"the\n",
"late\n",
"release\n",
"of\n",
"the\n",
"Java\n",
"SE\n",
"platform\n",
",\n",
"if\n",
"-PRON-\n",
"have\n",
"not\n",
"already\n",
"do\n",
"so\n",
".\n",
"create\n",
"a\n",
"program\n",
"that\n",
"use\n",
"Swing\n",
"component\n",
".\n",
"compile\n",
"the\n",
"program\n",
".\n",
"run\n",
"the\n",
"program\n",
".\n"
] ]
} }
], ],
@ -308,7 +236,12 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"def terminology_lookup():\n", "def terminology_lookup():\n",
" return []" " result = []\n",
" for token in doc:\n",
" if token.lemma_ in dictionary:\n",
" result.append((token, token.idx, token.idx + len(token)))\n",
"\n",
" return result"
] ]
}, },
{ {
@ -343,7 +276,13 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"def get_nouns(text):\n", "def get_nouns(text):\n",
" return []" " result = []\n",
" doc = nlp(text)\n",
" for token in doc:\n",
" if token.pos_ == 'NOUN':\n",
" result.append(token)\n",
"\n",
" return result"
] ]
}, },
{ {
@ -380,7 +319,16 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"def extract_terms(text):\n", "def extract_terms(text):\n",
" return []" " result = {}\n",
" doc = nlp(text)\n",
" for token in doc:\n",
" if token.pos_ == 'NOUN':\n",
" if result.get(token.lemma_) is None:\n",
" result[token.lemma_] = 1\n",
" else:\n",
" result[token.lemma_] += 1\n",
"\n",
" return result"
] ]
}, },
{ {
@ -399,7 +347,16 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"def extract_terms(text):\n", "def extract_terms(text):\n",
" return []" " result = {}\n",
" doc = nlp(text)\n",
" for token in doc:\n",
" if token.pos_ in ['NOUN', 'VERB', 'ADJ']:\n",
" if result.get(token.lemma_) is None:\n",
" result[token.lemma_] = 1\n",
" else:\n",
" result[token.lemma_] += 1\n",
"\n",
" return result"
] ]
} }
], ],
@ -407,7 +364,7 @@
"author": "Rafał Jaworski", "author": "Rafał Jaworski",
"email": "rjawor@amu.edu.pl", "email": "rjawor@amu.edu.pl",
"kernelspec": { "kernelspec": {
"display_name": "Python 3", "display_name": "Python 3 (ipykernel)",
"language": "python", "language": "python",
"name": "python3" "name": "python3"
}, },
@ -422,7 +379,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.8.10" "version": "3.9.2"
}, },
"subtitle": "3. Terminologia", "subtitle": "3. Terminologia",
"title": "Komputerowe wspomaganie tłumaczenia", "title": "Komputerowe wspomaganie tłumaczenia",

130
lab/lab_04-05.ipynb
View File

File diff suppressed because one or more lines are too long

54
lab/lab_06-07.ipynb
View File

@ -60,8 +60,14 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"import regex\n",
"\n",
"\n",
"def find_tags(text):\n", "def find_tags(text):\n",
" return []" " result = []\n",
" for occurance in regex.finditer(\"(\\</?\\w+\\>)\", text, regex.IGNORECASE):\n",
" result.append(occurance.span())\n",
" return result"
] ]
}, },
{ {
@ -79,8 +85,12 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"import regex\n",
"\n",
"\n",
"# Assuming text is a single word\n",
"def is_translatable(text):\n", "def is_translatable(text):\n",
" return True" " return regex.fullmatch(\"[A-Z\\-]+\", text, regex.IGNORECASE) is not None"
] ]
}, },
{ {
@ -98,8 +108,26 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"import regex\n",
"\n",
"\n",
"def find_dates(text):\n", "def find_dates(text):\n",
" return []" " regex_format = regex.compile(\"(?P<day>[0-9]{1,2})[/.-](?P<month>[0-9]{1,2})[/.-](?P<year>[0-9]{4})\")\n",
" matches = regex.match(regex_format, text)\n",
" result = {\n",
" 'day': int(matches.group('day')),\n",
" 'month': int(matches.group('month')),\n",
" 'year': int(matches.group('year')),\n",
" }\n",
"\n",
" return result\n",
"\n",
"\n",
"print(find_dates(\"01/02/1970\"))\n",
"print(find_dates(\"01.02.1970\"))\n",
"print(find_dates(\"01-02-1970\"))\n",
"print(find_dates(\"1/2/1970\"))\n",
"print(find_dates(\"1.2.1970\"))"
] ]
}, },
{ {
@ -130,8 +158,22 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"formats = {\n",
" 'd/m/y': lambda date: f\"{date['day']}/{date['month']}/{date['year']}\",\n",
" 'y-m-d': lambda date: f\"{date['year']}-{date['month']}-{date['day']}\",\n",
"}\n",
"\n",
"\n",
"def correct_dates(source_segment, target_segment, date_format):\n", "def correct_dates(source_segment, target_segment, date_format):\n",
" return ''" " source_date = find_dates(source_segment)\n",
" target_date = find_dates(target_segment)\n",
" if target_date != source_date:\n",
" print('Dates differ')\n",
"\n",
" return formats[date_format](source_date)\n",
"\n",
"\n",
"print(correct_dates(\"1.2.1970\", \"1.2.1970\", 'y-m-d'))"
] ]
}, },
{ {
@ -190,7 +232,7 @@
"author": "Rafał Jaworski", "author": "Rafał Jaworski",
"email": "rjawor@amu.edu.pl", "email": "rjawor@amu.edu.pl",
"kernelspec": { "kernelspec": {
"display_name": "Python 3", "display_name": "Python 3 (ipykernel)",
"language": "python", "language": "python",
"name": "python3" "name": "python3"
}, },
@ -205,7 +247,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.8.10" "version": "3.9.2"
}, },
"subtitle": "6,7. Preprocessing i postprocessing", "subtitle": "6,7. Preprocessing i postprocessing",
"title": "Komputerowe wspomaganie tłumaczenia", "title": "Komputerowe wspomaganie tłumaczenia",

235
lab/lab_09-10.ipynb
View File

@ -50,37 +50,36 @@
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"Nastolatek ukradł znajomemu 4500 złotych. Wcześniej pił z nim alkohol\n", "W Poznaniu uroczyście odsłonięto monument upamiętniający cmentarz żydowski założony jeszcze w XIX wieku\n",
"Czekają nas kolejne podwyżki rachunków. Tym razem za ogrzewanie i ciepłą wodę\n", "Przez ulice Poznania przejdzie Marsz dla Życia. Będą utrudnienia\n",
"Nie żyje Piotr Ś. Czyściciel kamienic miał 47 lat\n", "Sierść psa zatopiona w żywicy? Taką biżuterię pamiątkową zlecają właściciele czworonożnych pociech\n",
"Maciej Skorża nie zmienił zdania o systemie na mecz z Rakowem. Kolejorz ma szybką okazję do rehabilitacji\n", "Nagrał film w jednej z poznańskich \"Biedronek\". Kilka spleśniałych cytryn w kartonie. \"Nikt się tym nie przejmuje\"\n",
"Kto zabił Kazimierę Kurkowiak? Poznańskie Archiwum X wraca do sprawy sprzed 30 lat\n", "Gniezno: poszkodowani po ulewie będą mogli ubiegać się o pomoc w ZUS i US. Powstała również specjalna infolinia\n",
"Mieszkańcy osiedla Kwiatowego zyskają nowy chodnik\n", "Zostawiła jedzenie dla potrzebujących. Coraz więcej głodnych osób, którym nie wystarcza pieniędzy po opłaceniu rachunków\n",
"Poznańskie ZOO ponownie się otwiera i apeluje o kupowanie biletów online\n", "Kolejne ostrzeżenie I stopnia od IMGW. Oprócz burz może wystąpić również grad\n",
"1700 zł mandatu dla motocyklisty: nie ma prawa jazdy, jechał za szybko\n", "Lech przegrał Koroną. Na trybunach marsz żałobny i 'mamy k**** dość'\n",
"Plac Wolności ma tętnić życiem. Jest koncepcja zagospodarowania\n", "Warta Poznań po przegranej z Jagielonią Białystok spada do I ligi\n",
"Dzikie wysypisko w Wielkopolskim Parku Narodowym, a w nim paczka z telefonem odbiorcy\n", "Mieszkańcy skarżą się na właściciela samochodu, w którym notorycznie włącza się alarm. \"Uprzykrza nam to życie!\"\n",
"Dobre wieści z Łazarza! \"Zielona Perła\" sprzedana!\n", "Leśne Placówki Montessori\n",
"Sokoły wędrowne w gnieździe na kominie poznańskiej elektrociepłowni! Są 4 młode\n", "Na autostradzie samochód wpadł w poślizg i stanął w poprzek. Są spore utrudnienia\n",
"720 nowych zakażeń w Wielkopolsce\n", "Wróciła plaga kradzieży katalizatorów. Zmora dla kierowców, którzy nie mogą garażować auta\n",
"Uderzył kobietę w sklepie: \"sprawca będzie rozliczony\"\n", "Nowy basen w Kiekrzu? W tunelu wody przybyło po same kolana\n",
"Zespół Szkół Geodezyjno- Drogowych. Przyszłość rysuje się w kolorowych barwach!\n", "Pierożki Dim Sum z Para Bar Rataje ze specjalną zniżką!\n",
"Tajemniczy wypadek i pożar pod Kwilczem. Auto spłonęło, w środku nikogo nie było\n", "Wielka głowa Darii Zawiałow zablokowała przez chwilę przejście dla pieszych na jednej z poznańskich ulic\n",
"Nad Jeziorem Maltańskim powstanie duży hotel? \"Ma uzupełniać infrastrukturę sportową\"\n", "Fałszywy pożar w centrum Poznania. Kłęby dymu w kamienicy?\n",
"Śmiertelny wypadek na trasie S8: samochód potrącił rowerzystę\n", "Jest kolejne ostrzeżenie pierwszego stopnia, tym razem hydrologiczne. Gwałtowny wzrost stanu wody\n",
"Specjaliści o poszukiwaniu Natalii Lick: \"niestety trop psa prowadził na Wartostradę\"\n", "Uwaga. Utrudnienia na drodze i ograniczenie prędkości. Potrwa to około 5 godzin\n",
"Korki przy skrzyżowaniu Grochowska / Grunwaldzka: ruszyły prace!\n", "Chcą pobić rekord w kręceniu lodów. Tona lodów w ciągu doby\n",
"Restauracja w Kaliszu przyjmuje klientów: sanepid i policja \"odwiedzili\" lokal\n", "Jest ostrzeżenie IMGW dla Wielkopolski. Lepiej schować przedmioty, które mogą przemieścić się pod wypływem silnego wiatru\n",
"Ile kosztuje wywóz odpadów?\n", "Nowe Centrum Medyczne Bizpark już w sprzedaży. Znajdź idealny lokal pod swoją działalność medyczną\n",
"Dachowanie auta na trasie Konin - Turek\n", "Rondo Obornickie: zderzenie samochodu z motocyklem. Poszkodowany został odwieziony do szpitala. Chwilowe utrudnienia\n",
"Kierowca BMW pod wpływem narkotyków, pasażer w ich posiadaniu. Obaj zostali zatrzymani\n", "Policjanci publikują wizerunek i szukają tego mężczyzny\n",
"Leszno: mężczyzna uderzył klientkę sklepu. Poszło o maseczkę?\n", "Grupa Stonewall będzie miała program na antenie TVP3 Poznań. \"To będzie odtrutka na lata dezinformacji\"\n",
"Od poniedziałku zapłacimy za parkowanie na kolejnych ulicach\n", "Ruszył remont ważnego mostu. Co z kłódkami zakochanych?\n",
"Włamał się do obiektu handlowego. Grozi mu nawet 15 lat więzienia\n", "Mieszkaniec spotkał wilka w Poznaniu?\n",
"Rondo Śródka: kolizja z udziałem dwóch pojazdów\n", "Włamanie do... lokomotywy\n",
"Europoseł PSL: oświadczenie Episkopatu ma wpływ na proces szczepień. \"Bardzo dużo ludzi zrezygnowało\"\n", "W nadwarciański krajobraz wpisały się... żurawie. \"Jeden jest największy na świecie\"\n",
"Bezcenna wygrana Enea Energetyka. Poznanianki zagrają w fazie play-off\n", "Robisz remont? Za to możesz słono zapłacić!\n"
"No to w drogę! Po odmienionych trasach w Wielkopolsce\n"
] ]
} }
], ],
@ -108,13 +107,51 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 1, "execution_count": 10,
"id": "moving-clothing", "id": "moving-clothing",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"def get_names(article_type):\n", "from bs4 import element\n",
" return []" "\n",
"def get_names(article_type, page_nr: int = 0):\n",
" url = 'https://www.ceneo.pl/;szukaj-' + article_type + ';0020-30-0-0-' + str(page_nr) + '.htm'\n",
" page = requests.get(url)\n",
" if page_nr != 0 and url != page.url:\n",
" return []\n",
" soup = BeautifulSoup(page.content, 'html.parser')\n",
" result = []\n",
"\n",
" def is_product_title_container(tag: element.Tag) -> bool:\n",
" if not tag.has_attr('class'):\n",
" return False\n",
"\n",
" classes = tag.attrs['class']\n",
" if len(classes) != 1:\n",
" return False\n",
"\n",
" return classes[0] == 'cat-prod-row__name'\n",
"\n",
" def is_product_title(tag: element.Tag) -> bool:\n",
" if not tag.has_attr('class'):\n",
" return True\n",
"\n",
" classes = tag.attrs['class']\n",
" if len(classes) != 1:\n",
" return False\n",
"\n",
" return classes[0] == 'font-bold'\n",
"\n",
" for tag in soup.find_all(is_product_title_container):\n",
" href = tag.find('a')\n",
" if type(href) is not element.Tag:\n",
" continue\n",
" spans = href.find_all('span')\n",
" for span in spans:\n",
" if is_product_title(span):\n",
" result.append(span.text)\n",
"\n",
" return result"
] ]
}, },
{ {
@ -135,13 +172,21 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 9,
"id": "german-dispute", "id": "german-dispute",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"def scrape_names():\n", "def scrape_names():\n",
" return []" " result = []\n",
" search = 'laptop'\n",
" page = 0\n",
" while True:\n",
" local_result = get_names(search, page)\n",
" if len(local_result) == 0:\n",
" return result\n",
" result = result + local_result\n",
" page += 1"
] ]
}, },
{ {
@ -197,13 +242,39 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 8,
"id": "regulation-sheriff", "id": "regulation-sheriff",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"def scrape_wmi():\n", "def scrape_wmi():\n",
" return []" " def get_text(soup_l: BeautifulSoup) -> str:\n",
" for trash in soup_l(['script', 'style']):\n",
" trash.extract()\n",
"\n",
" text = soup_l.get_text()\n",
"\n",
" return re.sub(r'\\s+', ' ', text)\n",
"\n",
" result = []\n",
"\n",
" base_url = 'https://wmi.amu.edu.pl/'\n",
" page = requests.get(base_url)\n",
" soup = BeautifulSoup(page.content, 'html.parser')\n",
" result.append(get_text(soup))\n",
" for href in soup.find_all('a'):\n",
" if type(href) != element.Tag:\n",
" continue\n",
"\n",
" if not href.has_attr('href'):\n",
" continue\n",
"\n",
" if base_url in href.attrs['href']:\n",
" sub_page = requests.get(href.attrs['href'])\n",
" result.append(get_text(BeautifulSoup(sub_page.content, 'html.parser')))\n",
"\n",
"\n",
" return result"
] ]
}, },
{ {
@ -222,30 +293,97 @@
"### Ćwiczenie 4: Pobierz jak najwięcej słów w języku albańskim z serwisu glosbe.com." "### Ćwiczenie 4: Pobierz jak najwięcej słów w języku albańskim z serwisu glosbe.com."
] ]
}, },
{
"cell_type": "markdown",
"id": "706d6cba-c7a7-4d1b-9c2f-eb2119f859b5",
"metadata": {},
"source": [
"Nie jest to rozwiązanie zbalansowane, ale pobierze najwięcej słów (Przy odpowiedniej rotacji adresów IP, z których korzystamy, ale założyłem, że kwestia infrastruktury i tego jak strona jest chroniona przed atakami DOS, jest poza zakresem tego zadania)"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 7,
"id": "surgical-ozone", "id": "surgical-ozone",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"def scrape_shqip():\n", "def scrape_shqip():\n",
" return []" " import string\n",
"\n",
" result = []\n",
" letters = list(string.ascii_lowercase)\n",
" letters_count = len(letters)\n",
" longest_sensible_english_word_len = 28\n",
" base_url = 'https://glosbe.com/en/sq/'\n",
"\n",
" def get_words(word_l: str) -> list[str]:\n",
" def is_translated_word(tag: element.Tag) -> bool:\n",
" if not tag.has_attr('id') or not tag.has_attr('lang'):\n",
" return False\n",
"\n",
" if not 'translation__' in tag.attrs['id'] or 'sq' != tag.attrs['lang']:\n",
" return False\n",
"\n",
" return True\n",
"\n",
" result_l = []\n",
" page = requests.get(base_url + word_l)\n",
" soup = BeautifulSoup(page.content, 'html.parser')\n",
" words_l = soup.find_all(is_translated_word)\n",
" for word_l in words_l:\n",
" text = word_l.text\n",
" result_l.append(re.sub(r'\\s+', ' ', text))\n",
"\n",
" return result_l\n",
"\n",
" def trans(word_l: list[int]) -> str:\n",
" result_l = ''\n",
" for letter_l in word_l:\n",
" result_l += letters[letter_l]\n",
"\n",
" return result_l\n",
"\n",
" def increment(word_l: list[int]) -> list[int]:\n",
" done = False\n",
" result_l = []\n",
" for letter_l in word_l:\n",
" if done:\n",
" result_l.append(letter_l)\n",
" continue\n",
" next_letter_l = letter_l + 1\n",
" if next_letter_l == letters_count:\n",
" result_l.append(0)\n",
" continue\n",
"\n",
" result_l.append(next_letter_l)\n",
" done = True\n",
"\n",
" return result_l\n",
"\n",
" for length in range(longest_sensible_english_word_len - 1):\n",
" length += 1\n",
" combos = pow(length, letters_count)\n",
" word = []\n",
" for pos in range(length):\n",
" word.append(0)\n",
" for i in range(combos):\n",
" result.append(get_words(trans(word)))\n",
" word = increment(word)\n",
"\n",
" return result"
] ]
} }
], ],
"metadata": { "metadata": {
"author": "Rafał Jaworski", "author": "Rafał Jaworski",
"email": "rjawor@amu.edu.pl", "email": "rjawor@amu.edu.pl",
"lang": "pl",
"subtitle": "9,10. Web scraping",
"title": "Komputerowe wspomaganie tłumaczenia",
"year": "2021",
"kernelspec": { "kernelspec": {
"display_name": "Python 3", "display_name": "Python 3 (ipykernel)",
"language": "python", "language": "python",
"name": "python3" "name": "python3"
}, },
"lang": "pl",
"language_info": { "language_info": {
"codemirror_mode": { "codemirror_mode": {
"name": "ipython", "name": "ipython",
@ -256,8 +394,11 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.8.10" "version": "3.10.4"
} },
"subtitle": "9,10. Web scraping",
"title": "Komputerowe wspomaganie tłumaczenia",
"year": "2021"
}, },
"nbformat": 4, "nbformat": 4,
"nbformat_minor": 5 "nbformat_minor": 5

56
lab/lab_11.ipynb
View File

@ -52,13 +52,22 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 1, "execution_count": 2,
"id": "german-dispute", "id": "german-dispute",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"def sentence_split(text):\n", "def sentence_split(text):\n",
" return []" " def purge(text_l: str) -> str:\n",
" return text_l.strip('.').strip()\n",
" index = 0\n",
" result = []\n",
" for match in regex.finditer(r'\\. \\p{Lu}|\\n', text):\n",
" result.append(purge(text[index:match.start(0)]))\n",
" index = match.start(0)\n",
" result.append(purge(text[index:len(text)]))\n",
"\n",
" return result"
] ]
}, },
{ {
@ -69,6 +78,14 @@
"### Ćwiczenie 2: Uruchom powyższy algorytm na treści wybranej przez siebie strony internetowej (do ściągnięcia treści strony wykorzystaj kod z laboratoriów nr 7). Zidentyfikuj co najmniej dwa wyjątki od ogólnej reguły podziału na segmenty i ulepsz algorytm." "### Ćwiczenie 2: Uruchom powyższy algorytm na treści wybranej przez siebie strony internetowej (do ściągnięcia treści strony wykorzystaj kod z laboratoriów nr 7). Zidentyfikuj co najmniej dwa wyjątki od ogólnej reguły podziału na segmenty i ulepsz algorytm."
] ]
}, },
{
"cell_type": "markdown",
"id": "20bc0bf7-35b7-44e5-8750-c22e6de9d048",
"metadata": {},
"source": [
"Dwa wyjatki to zdania zakończone wykrzyknikiem i zdania zakończone znakiem zapytania"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 3, "execution_count": 3,
@ -76,8 +93,17 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"def sentence_split_enhanced(text):\n", "def sentence_split(text):\n",
" return []" " def purge(text_l: str) -> str:\n",
" return text_l.strip('.').strip('?').strip('!').strip()\n",
" index = 0\n",
" result = []\n",
" for match in regex.finditer(r'(\\.|\\?|\\!) \\p{Lu}|\\n', text):\n",
" result.append(purge(text[index:match.start(0)]))\n",
" index = match.start(0)\n",
" result.append(purge(text[index:len(text)]))\n",
"\n",
" return result"
] ]
}, },
{ {
@ -117,6 +143,14 @@
"Wyjściem z Hunaligna jest plik w specjalnym formacie Hunaligna. Problem jednak w tym, że niestety nie można go w prosty sposób zaimportować do jakiegokolwiek narzędzia typu CAT. Potrzebna jest konwersja do któregoś z bardziej popularnych formatów, np. XLIFF." "Wyjściem z Hunaligna jest plik w specjalnym formacie Hunaligna. Problem jednak w tym, że niestety nie można go w prosty sposób zaimportować do jakiegokolwiek narzędzia typu CAT. Potrzebna jest konwersja do któregoś z bardziej popularnych formatów, np. XLIFF."
] ]
}, },
{
"cell_type": "markdown",
"id": "80360005-5110-4f83-bfd6-dbe22a1d5b5b",
"metadata": {},
"source": [
"## *Linki do pobrania tego progamu(ftp://ftp.mokk.bme.hu/Hunglish/src/hunalign/latest/hunalign-1.1-windows.zip), dostępne w README na githubie, nie działają.*"
]
},
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "divided-chain", "id": "divided-chain",
@ -187,15 +221,12 @@
"metadata": { "metadata": {
"author": "Rafał Jaworski", "author": "Rafał Jaworski",
"email": "rjawor@amu.edu.pl", "email": "rjawor@amu.edu.pl",
"lang": "pl",
"subtitle": "11. Urównoleglanie",
"title": "Komputerowe wspomaganie tłumaczenia",
"year": "2021",
"kernelspec": { "kernelspec": {
"display_name": "Python 3", "display_name": "Python 3 (ipykernel)",
"language": "python", "language": "python",
"name": "python3" "name": "python3"
}, },
"lang": "pl",
"language_info": { "language_info": {
"codemirror_mode": { "codemirror_mode": {
"name": "ipython", "name": "ipython",
@ -206,8 +237,11 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.8.10" "version": "3.10.4"
} },
"subtitle": "11. Urównoleglanie",
"title": "Komputerowe wspomaganie tłumaczenia",
"year": "2021"
}, },
"nbformat": 4, "nbformat": 4,
"nbformat_minor": 5 "nbformat_minor": 5

109
lab/lab_12.ipynb
View File

@ -96,6 +96,26 @@
"### Ćwiczenie 1: Wykorzystując powyższy kod napisz keylogger, który zapisuje wszystkie uderzenia w klawisze do pliku. Format pliku jest dowolny, każdy wpis musi zawierać precyzyjną godzinę uderzenia oraz uderzony klawisz. Uruchom program i przepisz paragraf dowolnie wybranego tekstu." "### Ćwiczenie 1: Wykorzystując powyższy kod napisz keylogger, który zapisuje wszystkie uderzenia w klawisze do pliku. Format pliku jest dowolny, każdy wpis musi zawierać precyzyjną godzinę uderzenia oraz uderzony klawisz. Uruchom program i przepisz paragraf dowolnie wybranego tekstu."
] ]
}, },
{
"cell_type": "code",
"execution_count": null,
"id": "8663ef15-88a0-4bb5-aff9-f19cbb3178c1",
"metadata": {},
"outputs": [],
"source": [
"import keyboard\n",
"\n",
"\n",
"def report_key(event: keyboard.KeyboardEvent):\n",
" file = open('test.txt', 'a')\n",
" file.write(f'[{event.time}] {event.name}\\n')\n",
" file.close()\n",
"\n",
"\n",
"keyboard.on_release(callback=report_key)\n",
"keyboard.wait()"
]
},
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "valuable-bearing", "id": "valuable-bearing",
@ -120,7 +140,40 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"def calculate_typing_speed():\n", "def calculate_typing_speed():\n",
" return 0" " import re\n",
" import numpy\n",
"\n",
" def parse(line_l: str) -> (float, str):\n",
" res = re.findall(r'(\\d+.\\d+)|([a-zA-Z,.]+)', ''.join(line_l.split()))\n",
" return float(res[0][0]), res[1][1]\n",
"\n",
" file = open('test.txt', 'r')\n",
" time_per_word = []\n",
" time_per_character = []\n",
" local_time_per_word = []\n",
"\n",
" prev_char_timestamp = None\n",
" for line in file:\n",
" time, key = parse(line)\n",
" if prev_char_timestamp is None or time - prev_char_timestamp > 5:\n",
" prev_char_timestamp = time\n",
" local_time_per_word = []\n",
" continue\n",
" elapsed = time - prev_char_timestamp\n",
" time_per_character.append(elapsed)\n",
" if key == 'space' or key == 'enter' or key == ',' or key == '.':\n",
" if len(local_time_per_word) > 0:\n",
" time_per_word.append(numpy.sum(local_time_per_word))\n",
" local_time_per_word = []\n",
" time_per_character.append(elapsed)\n",
" prev_char_timestamp = time\n",
" continue\n",
" local_time_per_word.append(elapsed)\n",
" prev_char_timestamp = time\n",
" file.close()\n",
" time_per_word.append(numpy.sum(local_time_per_word))\n",
" \n",
" return 60 / numpy.average(time_per_character), 60 / numpy.average(time_per_word)"
] ]
}, },
{ {
@ -147,22 +200,57 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"def find_pauses():\n", "def find_pauses():\n",
" return []" " import re\n",
"\n",
" def parse(line_l: str) -> (float, str):\n",
" res = re.findall(r'(\\d+.\\d+)|([a-zA-Z,.]+)', ''.join(line_l.split()))\n",
" return float(res[0][0]), res[1][1]\n",
"\n",
" file = open('test.txt', 'r')\n",
" stops = []\n",
" stop_reporting_time = 1\n",
"\n",
" prev_char_timestamp = None\n",
" lines = file.readlines()\n",
" file.close()\n",
" for i in range(len(lines)):\n",
" time, key = parse(lines[i])\n",
" if prev_char_timestamp is None:\n",
" prev_char_timestamp = time\n",
" continue\n",
" elapsed = time - prev_char_timestamp\n",
" if elapsed > stop_reporting_time:\n",
" context_start = max(0, i - 20)\n",
" context_end = min(len(lines), i + 20)\n",
" context_before = ''\n",
" context_after = ''\n",
" for j in range(context_start, i):\n",
" time_l, key_l = parse(lines[j])\n",
" context_before += key_l\n",
" for j in range(i, context_end):\n",
" time_l, key_l = parse(lines[j])\n",
" context_after += key_l\n",
" stops.append((elapsed, (context_before, context_after)))\n",
" prev_char_timestamp = time\n",
"\n",
" def stop_sort(record: tuple):\n",
" return record[0]\n",
"\n",
" stops.sort(reverse=True, key=stop_sort)\n",
" \n",
" return stops"
] ]
} }
], ],
"metadata": { "metadata": {
"author": "Rafał Jaworski", "author": "Rafał Jaworski",
"email": "rjawor@amu.edu.pl", "email": "rjawor@amu.edu.pl",
"lang": "pl",
"subtitle": "12. Key logging",
"title": "Komputerowe wspomaganie tłumaczenia",
"year": "2021",
"kernelspec": { "kernelspec": {
"display_name": "Python 3", "display_name": "Python 3 (ipykernel)",
"language": "python", "language": "python",
"name": "python3" "name": "python3"
}, },
"lang": "pl",
"language_info": { "language_info": {
"codemirror_mode": { "codemirror_mode": {
"name": "ipython", "name": "ipython",
@ -173,8 +261,11 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.8.10" "version": "3.10.4"
} },
"subtitle": "12. Key logging",
"title": "Komputerowe wspomaganie tłumaczenia",
"year": "2021"
}, },
"nbformat": 4, "nbformat": 4,
"nbformat_minor": 5 "nbformat_minor": 5

4
lab/lab_13-14.ipynb
View File

@ -201,7 +201,7 @@
"author": "Rafał Jaworski", "author": "Rafał Jaworski",
"email": "rjawor@amu.edu.pl", "email": "rjawor@amu.edu.pl",
"kernelspec": { "kernelspec": {
"display_name": "Python 3", "display_name": "Python 3 (ipykernel)",
"language": "python", "language": "python",
"name": "python3" "name": "python3"
}, },
@ -216,7 +216,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.8.10" "version": "3.10.4"
}, },
"subtitle": "13,14. Korekta pisowni", "subtitle": "13,14. Korekta pisowni",
"title": "Komputerowe wspomaganie tłumaczenia", "title": "Komputerowe wspomaganie tłumaczenia",

104
lab/lab_15.ipynb
View File

File diff suppressed because one or more lines are too long