{ "cells": [ { "cell_type": "code", "execution_count": 18, "id": "21c9b695", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import csv\n", "import regex as re\n", "import nltk\n", "from collections import Counter, defaultdict\n", "import string\n", "import unicodedata\n", "\n", "def clean_text(text): \n", " return re.sub(r\"\\p{P}\", \"\", str(text).lower().replace(\"-\\\\n\", \"\").replace(\"\\\\n\", \" \"))\n", "\n", "def train_model(data, model):\n", " for _, row in data.iterrows():\n", " words = nltk.word_tokenize(clean_text(row[\"final\"]))\n", " for w1, w2 in nltk.bigrams(words, pad_left=True, pad_right=True):\n", " if w1 and w2:\n", " model[w2][w1] += 1\n", " for w2 in model:\n", " total_count = float(sum(model[w2].values()))\n", " for w1 in model[w2]:\n", " model[w2][w1] /= total_count\n", "\n", "\n", "def predict(word, model):\n", " predictions = dict(model[word])\n", " most_common = dict(Counter(predictions).most_common(5))\n", "\n", " total_prob = 0.0\n", " str_prediction = \"\"\n", "\n", " for word, prob in most_common.items():\n", " total_prob += prob\n", " str_prediction += f\"{word}:{prob} \"\n", "\n", " if not total_prob:\n", " return \"the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1\"\n", "\n", " if 1 - total_prob >= 0.01:\n", " str_prediction += f\":{1-total_prob}\"\n", " else:\n", " str_prediction += f\":0.01\"\n", "\n", " return str_prediction\n", "\n", "\n", "def predict_data(read_path, save_path, model):\n", " data = pd.read_csv(\n", " read_path,\n", " sep=\"\\t\",\n", " error_bad_lines=False,\n", " header=None,\n", " quoting=csv.QUOTE_NONE,\n", " encoding=\"utf-8\"\n", " )\n", " with open(save_path, \"w\", encoding=\"utf-8\") as f:\n", " for _, row in data.iterrows():\n", " words = nltk.word_tokenize(clean_text(row[7]))\n", " if len(words) < 3:\n", " prediction = \"the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1\"\n", " else:\n", " prediction = predict(words[-1], model)\n", " f.write(prediction + \"\\n\")\n" ] }, { "cell_type": "code", "execution_count": 11, "id": "e39473e2", "metadata": {}, "outputs": [], "source": [ "with open(\"in-header.tsv\") as f:\n", " in_cols = f.read().strip().split(\"\\t\")\n", "\n", "with open(\"out-header.tsv\") as f:\n", " out_cols = f.read().strip().split(\"\\t\")" ] }, { "cell_type": "code", "execution_count": 12, "id": "bde510c9", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['FileId', 'Year', 'LeftContext', 'RightContext']" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "in_cols" ] }, { "cell_type": "code", "execution_count": 13, "id": "0e8b31dd", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['Word']" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_cols" ] }, { "cell_type": "code", "execution_count": 7, "id": "7662d802", "metadata": {}, "outputs": [], "source": [ "data = pd.read_csv(\n", " \"train/in.tsv.xz\",\n", " sep=\"\\t\",\n", " on_bad_lines='skip',\n", " header=None,\n", " # names=in_cols,\n", " quoting=csv.QUOTE_NONE,\n", " encoding=\"utf-8\"\n", ")\n", "\n", "train_words = pd.read_csv(\n", " \"train/expected.tsv\",\n", " sep=\"\\t\",\n", " on_bad_lines='skip',\n", " header=None,\n", " # names=out_cols,\n", " quoting=csv.QUOTE_NONE,,\n", " encoding=\"utf-8\"\n", ")\n", "\n", "train_data = data[[7, 6]]\n", "train_data = pd.concat([train_data, train_words], axis=1)\n", "\n", "train_data[\"final\"] = train_data[7] + train_data[0] + train_data[6]\n" ] }, { "cell_type": "code", "execution_count": 8, "id": "c3d2cfec", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
760final
0said\\nit's all squash. The best I could get\\ni...came fiom the last place to this\\nplace, and t...liesaid\\nit's all squash. The best I could get\\ni...
1\\ninto a proper perspective with those\\nminor ...MB. BOOT'S POLITICAL OBEED\\nAttempt to imagine...himself\\ninto a proper perspective with those\\nminor ...
2all notU\\nashore and afloat arc subjects for I...\"Thera were in 1771 only aeventy-nine\\n*ub*erl...ofall notU\\nashore and afloat arc subjects for I...
3ceucju l< d no; <o waste it nud so\\nsunk it in...A gixnl man y nitereRtiiiv dii-clos-\\nur«s reg...ablyceucju l< d no; <o waste it nud so\\nsunk it in...
4ascertained w? OCt the COOltS of ibis\\nletale ...Tin: 188UB TV THF BBABBT QABJE\\nMr. Schiffs *t...jascertained w? OCt the COOltS of ibis\\nletale ...
...............
432017\\nSam was arrested.\\nThe case excited a great ...Sam Clendenin bad a fancy for Ui«\\nscience of ...and\\nSam was arrested.\\nThe case excited a great ...
432018through the alnp the »Uitors laapeeeed tia.»\\n...Wita.htt halting the party ware dilven to the ...paasliicthrough the alnp the »Uitors laapeeeed tia.»\\n...
432019Agua Negra across the line.\\nIt was a grim pla...It was the last thing that either of\\nthem exp...forAgua Negra across the line.\\nIt was a grim pla...
432020\\na note of Wood, Dialogue fc Co., for\\nc27,im...settlement with the department.\\nIt is also sh...for\\na note of Wood, Dialogue fc Co., for\\nc27,im...
4320213214c;do White at 3614c: Mixed Western at\\n331...Flour quotations—low extras at 1 R0®2 50;\\ncit...at3214c;do White at 3614c: Mixed Western at\\n331...
\n", "

432022 rows × 4 columns

\n", "
" ], "text/plain": [ " 7 \\\n", "0 said\\nit's all squash. The best I could get\\ni... \n", "1 \\ninto a proper perspective with those\\nminor ... \n", "2 all notU\\nashore and afloat arc subjects for I... \n", "3 ceucju l< d no; ", "output_type": "error", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[1;31mUnicodeEncodeError\u001b[0m Traceback (most recent call last)", "\u001b[1;32mc:\\Users\\Norbert\\code\\challenging-america-word-gap-prediction\\testing.ipynb Cell 7'\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[0;32m 3\u001b[0m train_model(train_data, model)\n\u001b[0;32m 4\u001b[0m predict_data(\u001b[39m\"\u001b[39m\u001b[39mdev-0/in.tsv.xz\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mdev-0/out.tsv\u001b[39m\u001b[39m\"\u001b[39m, model)\n\u001b[1;32m----> 5\u001b[0m predict_data(\u001b[39m\"\u001b[39;49m\u001b[39mtest-A/in.tsv.xz\u001b[39;49m\u001b[39m\"\u001b[39;49m, \u001b[39m\"\u001b[39;49m\u001b[39mtest-A/out.tsv\u001b[39;49m\u001b[39m\"\u001b[39;49m, model)\n", "\u001b[1;32mc:\\Users\\Norbert\\code\\challenging-america-word-gap-prediction\\testing.ipynb Cell 1'\u001b[0m in \u001b[0;36mpredict_data\u001b[1;34m(read_path, save_path, model)\u001b[0m\n\u001b[0;32m 55\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m 56\u001b[0m prediction \u001b[39m=\u001b[39m predict(words[\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m], model)\n\u001b[1;32m---> 57\u001b[0m file\u001b[39m.\u001b[39;49mwrite(prediction \u001b[39m+\u001b[39;49m \u001b[39m\"\u001b[39;49m\u001b[39m\\n\u001b[39;49;00m\u001b[39m\"\u001b[39;49m)\n", "File \u001b[1;32mC:\\Python310\\lib\\encodings\\cp1250.py:19\u001b[0m, in \u001b[0;36mIncrementalEncoder.encode\u001b[1;34m(self, input, final)\u001b[0m\n\u001b[0;32m 18\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mencode\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39minput\u001b[39m, final\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m):\n\u001b[1;32m---> 19\u001b[0m \u001b[39mreturn\u001b[39;00m codecs\u001b[39m.\u001b[39;49mcharmap_encode(\u001b[39minput\u001b[39;49m,\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49merrors,encoding_table)[\u001b[39m0\u001b[39m]\n", "\u001b[1;31mUnicodeEncodeError\u001b[0m: 'charmap' codec can't encode character '\\u03b2' in position 21: character maps to " ] } ], "source": [ "\n", "model = defaultdict(lambda: defaultdict(lambda: 0))\n", "\n", "train_model(train_data, model)" ] }, { "cell_type": "code", "execution_count": 15, "id": "ad23240e", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "C:\\Users\\Norbert\\AppData\\Local\\Temp\\ipykernel_15436\\842062938.py:47: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n", "\n", "\n", " data = pd.read_csv(\n" ] } ], "source": [ "predict_data(\"dev-0/in.tsv.xz\", \"dev-0/out.tsv\", model)" ] }, { "cell_type": "code", "execution_count": 19, "id": "195cb6cf", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "C:\\Users\\Norbert\\AppData\\Local\\Temp\\ipykernel_15436\\751703071.py:47: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n", "\n", "\n", " data = pd.read_csv(\n" ] } ], "source": [ "predict_data(\"test-A/in.tsv.xz\", \"test-A/out.tsv\", model)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.2" } }, "nbformat": 4, "nbformat_minor": 5 }