s440054/challenging-america-word-gap-prediction
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

work in progress

Browse Source
This commit is contained in:
s440054 2022-04-04 16:03:59 +02:00
parent 5af6e29a07
commit a0a24dd6b5
1 changed files with 17 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
testing.ipynb
View File

@ -2,7 +2,7 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 18, "execution_count": 36,
"id": "21c9b695", "id": "21c9b695",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@ -20,7 +20,7 @@
"\n", "\n",
"def train_model(data, model):\n", "def train_model(data, model):\n",
" for _, row in data.iterrows():\n", " for _, row in data.iterrows():\n",
" words = nltk.word_tokenize(clean_text(row[\"final\"]))\n", " words = nltk.word_tokenize(clean_text(row[760]))\n",
" for w1, w2 in nltk.bigrams(words, pad_left=True, pad_right=True):\n", " for w1, w2 in nltk.bigrams(words, pad_left=True, pad_right=True):\n",
" if w1 and w2:\n", " if w1 and w2:\n",
" model[w2][w1] += 1\n", " model[w2][w1] += 1\n",
@ -36,7 +36,6 @@
"\n", "\n",
" total_prob = 0.0\n", " total_prob = 0.0\n",
" str_prediction = \"\"\n", " str_prediction = \"\"\n",
"\n",
" for word, prob in most_common.items():\n", " for word, prob in most_common.items():\n",
" total_prob += prob\n", " total_prob += prob\n",
" str_prediction += f\"{word}:{prob} \"\n", " str_prediction += f\"{word}:{prob} \"\n",
@ -129,7 +128,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 7, "execution_count": 22,
"id": "7662d802", "id": "7662d802",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@ -150,19 +149,19 @@
" on_bad_lines='skip',\n", " on_bad_lines='skip',\n",
" header=None,\n", " header=None,\n",
" # names=out_cols,\n", " # names=out_cols,\n",
" quoting=csv.QUOTE_NONE,,\n", " quoting=csv.QUOTE_NONE,\n",
" encoding=\"utf-8\"\n", " encoding=\"utf-8\"\n",
")\n", ")\n",
"\n", "\n",
"train_data = data[[7, 6]]\n", "train_data = data[[7, 6]]\n",
"train_data = pd.concat([train_data, train_words], axis=1)\n", "train_data = pd.concat([train_data, train_words], axis=1)\n",
"\n", "\n",
"train_data[\"final\"] = train_data[7] + train_data[0] + train_data[6]\n" "train_data[760] = train_data[7] + train_data[0] + train_data[6]\n"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 8, "execution_count": 23,
"id": "c3d2cfec", "id": "c3d2cfec",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@ -190,7 +189,7 @@
" <th>7</th>\n", " <th>7</th>\n",
" <th>6</th>\n", " <th>6</th>\n",
" <th>0</th>\n", " <th>0</th>\n",
" <th>final</th>\n", " <th>760</th>\n",
" </tr>\n", " </tr>\n",
" </thead>\n", " </thead>\n",
" <tbody>\n", " <tbody>\n",
@ -303,7 +302,7 @@
"432020 settlement with the department.\\nIt is also sh... for \n", "432020 settlement with the department.\\nIt is also sh... for \n",
"432021 Flour quotations—low extras at 1 R0®2 50;\\ncit... at \n", "432021 Flour quotations—low extras at 1 R0®2 50;\\ncit... at \n",
"\n", "\n",
" final \n", " 760 \n",
"0 said\\nit's all squash. The best I could get\\ni... \n", "0 said\\nit's all squash. The best I could get\\ni... \n",
"1 \\ninto a proper perspective with those\\nminor ... \n", "1 \\ninto a proper perspective with those\\nminor ... \n",
"2 all notU\\nashore and afloat arc subjects for I... \n", "2 all notU\\nashore and afloat arc subjects for I... \n",
@ -319,7 +318,7 @@
"[432022 rows x 4 columns]" "[432022 rows x 4 columns]"
] ]
}, },
"execution_count": 8, "execution_count": 23,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -330,38 +329,10 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 14, "execution_count": 26,
"id": "bd92ba07", "id": "bd92ba07",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\Norbert\\AppData\\Local\\Temp\\ipykernel_15436\\842062938.py:47: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
"\n",
"\n",
" data = pd.read_csv(\n",
"C:\\Users\\Norbert\\AppData\\Local\\Temp\\ipykernel_15436\\842062938.py:47: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
"\n",
"\n",
" data = pd.read_csv(\n"
]
},
{
"ename": "UnicodeEncodeError",
"evalue": "'charmap' codec can't encode character '\\u03b2' in position 21: character maps to <undefined>",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mUnicodeEncodeError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32mc:\\Users\\Norbert\\code\\challenging-america-word-gap-prediction\\testing.ipynb Cell 7'\u001b[0m in \u001b[0;36m<cell line: 5>\u001b[1;34m()\u001b[0m\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Norbert/code/challenging-america-word-gap-prediction/testing.ipynb#ch0000006?line=2'>3</a>\u001b[0m train_model(train_data, model)\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Norbert/code/challenging-america-word-gap-prediction/testing.ipynb#ch0000006?line=3'>4</a>\u001b[0m predict_data(\u001b[39m\"\u001b[39m\u001b[39mdev-0/in.tsv.xz\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mdev-0/out.tsv\u001b[39m\u001b[39m\"\u001b[39m, model)\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/Norbert/code/challenging-america-word-gap-prediction/testing.ipynb#ch0000006?line=4'>5</a>\u001b[0m predict_data(\u001b[39m\"\u001b[39;49m\u001b[39mtest-A/in.tsv.xz\u001b[39;49m\u001b[39m\"\u001b[39;49m, \u001b[39m\"\u001b[39;49m\u001b[39mtest-A/out.tsv\u001b[39;49m\u001b[39m\"\u001b[39;49m, model)\n",
"\u001b[1;32mc:\\Users\\Norbert\\code\\challenging-america-word-gap-prediction\\testing.ipynb Cell 1'\u001b[0m in \u001b[0;36mpredict_data\u001b[1;34m(read_path, save_path, model)\u001b[0m\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Norbert/code/challenging-america-word-gap-prediction/testing.ipynb#ch0000000?line=54'>55</a>\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Norbert/code/challenging-america-word-gap-prediction/testing.ipynb#ch0000000?line=55'>56</a>\u001b[0m prediction \u001b[39m=\u001b[39m predict(words[\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m], model)\n\u001b[1;32m---> <a href='vscode-notebook-cell:/c%3A/Users/Norbert/code/challenging-america-word-gap-prediction/testing.ipynb#ch0000000?line=56'>57</a>\u001b[0m file\u001b[39m.\u001b[39;49mwrite(prediction \u001b[39m+\u001b[39;49m \u001b[39m\"\u001b[39;49m\u001b[39m\\n\u001b[39;49;00m\u001b[39m\"\u001b[39;49m)\n",
"File \u001b[1;32mC:\\Python310\\lib\\encodings\\cp1250.py:19\u001b[0m, in \u001b[0;36mIncrementalEncoder.encode\u001b[1;34m(self, input, final)\u001b[0m\n\u001b[0;32m <a href='file:///c%3A/Python310/lib/encodings/cp1250.py?line=17'>18</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mencode\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39minput\u001b[39m, final\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m):\n\u001b[1;32m---> <a href='file:///c%3A/Python310/lib/encodings/cp1250.py?line=18'>19</a>\u001b[0m \u001b[39mreturn\u001b[39;00m codecs\u001b[39m.\u001b[39;49mcharmap_encode(\u001b[39minput\u001b[39;49m,\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49merrors,encoding_table)[\u001b[39m0\u001b[39m]\n",
"\u001b[1;31mUnicodeEncodeError\u001b[0m: 'charmap' codec can't encode character '\\u03b2' in position 21: character maps to <undefined>"
]
}
],
"source": [ "source": [
"\n", "\n",
"model = defaultdict(lambda: defaultdict(lambda: 0))\n", "model = defaultdict(lambda: defaultdict(lambda: 0))\n",
@ -371,7 +342,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 15, "execution_count": 37,
"id": "ad23240e", "id": "ad23240e",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@ -379,7 +350,7 @@
"name": "stderr", "name": "stderr",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"C:\\Users\\Norbert\\AppData\\Local\\Temp\\ipykernel_15436\\842062938.py:47: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n", "C:\\Users\\Norbert\\AppData\\Local\\Temp\\ipykernel_15436\\749044266.py:46: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
"\n", "\n",
"\n", "\n",
" data = pd.read_csv(\n" " data = pd.read_csv(\n"
@ -392,7 +363,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 19, "execution_count": 38,
"id": "195cb6cf", "id": "195cb6cf",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@ -400,7 +371,7 @@
"name": "stderr", "name": "stderr",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"C:\\Users\\Norbert\\AppData\\Local\\Temp\\ipykernel_15436\\751703071.py:47: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n", "C:\\Users\\Norbert\\AppData\\Local\\Temp\\ipykernel_15436\\749044266.py:46: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
"\n", "\n",
"\n", "\n",
" data = pd.read_csv(\n" " data = pd.read_csv(\n"