{ "cells": [ { "cell_type": "code", "execution_count": 36, "id": "21c9b695", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import csv\n", "import regex as re\n", "import nltk\n", "from collections import Counter, defaultdict\n", "import string\n", "import unicodedata\n", "\n", "def clean_text(text): \n", " return re.sub(r\"\\p{P}\", \"\", str(text).lower().replace(\"-\\\\n\", \"\").replace(\"\\\\n\", \" \"))\n", "\n", "def train_model(data, model):\n", " for _, row in data.iterrows():\n", " words = nltk.word_tokenize(clean_text(row[760]))\n", " for w1, w2 in nltk.bigrams(words, pad_left=True, pad_right=True):\n", " if w1 and w2:\n", " model[w2][w1] += 1\n", " for w2 in model:\n", " total_count = float(sum(model[w2].values()))\n", " for w1 in model[w2]:\n", " model[w2][w1] /= total_count\n", "\n", "\n", "def predict(word, model):\n", " predictions = dict(model[word])\n", " most_common = dict(Counter(predictions).most_common(5))\n", "\n", " total_prob = 0.0\n", " str_prediction = \"\"\n", " for word, prob in most_common.items():\n", " total_prob += prob\n", " str_prediction += f\"{word}:{prob} \"\n", "\n", " if not total_prob:\n", " return \"the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1\"\n", "\n", " if 1 - total_prob >= 0.01:\n", " str_prediction += f\":{1-total_prob}\"\n", " else:\n", " str_prediction += f\":0.01\"\n", "\n", " return str_prediction\n", "\n", "\n", "def predict_data(read_path, save_path, model):\n", " data = pd.read_csv(\n", " read_path,\n", " sep=\"\\t\",\n", " error_bad_lines=False,\n", " header=None,\n", " quoting=csv.QUOTE_NONE,\n", " encoding=\"utf-8\"\n", " )\n", " with open(save_path, \"w\", encoding=\"utf-8\") as f:\n", " for _, row in data.iterrows():\n", " words = nltk.word_tokenize(clean_text(row[7]))\n", " if len(words) < 3:\n", " prediction = \"the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1\"\n", " else:\n", " prediction = predict(words[-1], model)\n", " f.write(prediction + \"\\n\")\n" ] }, { "cell_type": "code", "execution_count": 11, "id": "e39473e2", "metadata": {}, "outputs": [], "source": [ "with open(\"in-header.tsv\") as f:\n", " in_cols = f.read().strip().split(\"\\t\")\n", "\n", "with open(\"out-header.tsv\") as f:\n", " out_cols = f.read().strip().split(\"\\t\")" ] }, { "cell_type": "code", "execution_count": 12, "id": "bde510c9", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['FileId', 'Year', 'LeftContext', 'RightContext']" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "in_cols" ] }, { "cell_type": "code", "execution_count": 13, "id": "0e8b31dd", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['Word']" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "out_cols" ] }, { "cell_type": "code", "execution_count": 22, "id": "7662d802", "metadata": {}, "outputs": [], "source": [ "data = pd.read_csv(\n", " \"train/in.tsv.xz\",\n", " sep=\"\\t\",\n", " on_bad_lines='skip',\n", " header=None,\n", " # names=in_cols,\n", " quoting=csv.QUOTE_NONE,\n", " encoding=\"utf-8\"\n", ")\n", "\n", "train_words = pd.read_csv(\n", " \"train/expected.tsv\",\n", " sep=\"\\t\",\n", " on_bad_lines='skip',\n", " header=None,\n", " # names=out_cols,\n", " quoting=csv.QUOTE_NONE,\n", " encoding=\"utf-8\"\n", ")\n", "\n", "train_data = data[[7, 6]]\n", "train_data = pd.concat([train_data, train_words], axis=1)\n", "\n", "train_data[760] = train_data[7] + train_data[0] + train_data[6]\n" ] }, { "cell_type": "code", "execution_count": 23, "id": "c3d2cfec", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
760760
0said\\nit's all squash. The best I could get\\ni...came fiom the last place to this\\nplace, and t...liesaid\\nit's all squash. The best I could get\\ni...
1\\ninto a proper perspective with those\\nminor ...MB. BOOT'S POLITICAL OBEED\\nAttempt to imagine...himself\\ninto a proper perspective with those\\nminor ...
2all notU\\nashore and afloat arc subjects for I...\"Thera were in 1771 only aeventy-nine\\n*ub*erl...ofall notU\\nashore and afloat arc subjects for I...
3ceucju l< d no; <o waste it nud so\\nsunk it in...A gixnl man y nitereRtiiiv dii-clos-\\nur«s reg...ablyceucju l< d no; <o waste it nud so\\nsunk it in...
4ascertained w? OCt the COOltS of ibis\\nletale ...Tin: 188UB TV THF BBABBT QABJE\\nMr. Schiffs *t...jascertained w? OCt the COOltS of ibis\\nletale ...
...............
432017\\nSam was arrested.\\nThe case excited a great ...Sam Clendenin bad a fancy for Ui«\\nscience of ...and\\nSam was arrested.\\nThe case excited a great ...
432018through the alnp the »Uitors laapeeeed tia.»\\n...Wita.htt halting the party ware dilven to the ...paasliicthrough the alnp the »Uitors laapeeeed tia.»\\n...
432019Agua Negra across the line.\\nIt was a grim pla...It was the last thing that either of\\nthem exp...forAgua Negra across the line.\\nIt was a grim pla...
432020\\na note of Wood, Dialogue fc Co., for\\nc27,im...settlement with the department.\\nIt is also sh...for\\na note of Wood, Dialogue fc Co., for\\nc27,im...
4320213214c;do White at 3614c: Mixed Western at\\n331...Flour quotations—low extras at 1 R0®2 50;\\ncit...at3214c;do White at 3614c: Mixed Western at\\n331...
\n", "

432022 rows × 4 columns

\n", "
" ], "text/plain": [ " 7 \\\n", "0 said\\nit's all squash. The best I could get\\ni... \n", "1 \\ninto a proper perspective with those\\nminor ... \n", "2 all notU\\nashore and afloat arc subjects for I... \n", "3 ceucju l< d no;