s434695/challenging-america-word-gap-prediction
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!

Browse Source
This commit is contained in:
Tomasz Grzybowski 2022-06-25 21:43:48 +02:00
parent f90040b458
commit 7b0fbff812
1 changed files with 150 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

150
Untitled.ipynb Normal file
View File

@ -0,0 +1,150 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 3,
"id": "b43d8178",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\grzyb\\AppData\\Local\\Temp/ipykernel_34768/887107210.py:86: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version.\n",
"\n",
"\n",
" predict_data(\"dev-0/in.tsv.xz\", \"dev-0/out.tsv\")\n"
]
},
{
"ename": "UnicodeEncodeError",
"evalue": "'charmap' codec can't encode character '\\u25a0' in position 0: character maps to <undefined>",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mUnicodeEncodeError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_34768/887107210.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m 84\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 85\u001b[0m \u001b[0mtrain_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 86\u001b[1;33m \u001b[0mpredict_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"dev-0/in.tsv.xz\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"dev-0/out.tsv\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 87\u001b[0m \u001b[0mpredict_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"test-A/in.tsv.xz\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"test-A/out.tsv\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_34768/887107210.py\u001b[0m in \u001b[0;36mpredict_data\u001b[1;34m(read_path, save_path)\u001b[0m\n\u001b[0;32m 80\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 81\u001b[0m \u001b[0mprediction\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwords\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 82\u001b[1;33m \u001b[0mfile\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprediction\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m\"\\n\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 83\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 84\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32mc:\\Users\\grzyb\\anaconda3\\lib\\encodings\\cp1250.py\u001b[0m in \u001b[0;36mencode\u001b[1;34m(self, input, final)\u001b[0m\n\u001b[0;32m 17\u001b[0m \u001b[1;32mclass\u001b[0m \u001b[0mIncrementalEncoder\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcodecs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mIncrementalEncoder\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 18\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mencode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfinal\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 19\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mcodecs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcharmap_encode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merrors\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mencoding_table\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 20\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 21\u001b[0m \u001b[1;32mclass\u001b[0m \u001b[0mIncrementalDecoder\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcodecs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mIncrementalDecoder\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;31mUnicodeEncodeError\u001b[0m: 'charmap' codec can't encode character '\\u25a0' in position 0: character maps to <undefined>"
]
}
],
"source": [
"import pandas as pd\n",
"import csv\n",
"import regex as re\n",
"from nltk import bigrams, word_tokenize\n",
"from collections import Counter, defaultdict\n",
"import string\n",
"import unicodedata\n",
"\n",
"data = pd.read_csv(\n",
" \"train/in.tsv.xz\",\n",
" sep=\"\\t\",\n",
" error_bad_lines=False,\n",
" header=None,\n",
" quoting=csv.QUOTE_NONE,\n",
" \n",
")\n",
"train_labels = pd.read_csv(\n",
" \"train/expected.tsv\",\n",
" sep=\"\\t\",\n",
" error_bad_lines=False,\n",
" header=None,\n",
" quoting=csv.QUOTE_NONE,\n",
")\n",
"\n",
"train_data = data[[6, 7]]\n",
"train_data = pd.concat([train_data, train_labels], axis=1)\n",
"\n",
"train_data[\"final\"] = train_data[6] + train_data[0] + train_data[7]\n",
"\n",
"model = defaultdict(lambda: defaultdict(lambda: 0))\n",
"\n",
"\n",
"def clean(text):\n",
" text = str(text).lower().replace(\"-\\\\n\", \"\").replace(\"\\\\n\", \" \")\n",
" return re.sub(r\"\\p{P}\", \"\", text)\n",
"\n",
"def train_model(data):\n",
" for _, row in data.iterrows():\n",
" words = word_tokenize(clean(row[\"final\"]))\n",
" for w1, w2 in bigrams(words, pad_left=True, pad_right=True):\n",
" if w1 and w2:\n",
" model[w1][w2] += 1\n",
" for w1 in model:\n",
" total_count = float(sum(model[w1].values()))\n",
" for w2 in model[w1]:\n",
" model[w1][w2] /= total_count\n",
"\n",
"\n",
"def predict(word):\n",
" predictions = dict(model[word])\n",
" most_common = dict(Counter(predictions).most_common(5))\n",
"\n",
" total_prob = 0.0\n",
" str_prediction = \"\"\n",
"\n",
" for word, prob in most_common.items():\n",
" total_prob += prob\n",
" str_prediction += f\"{word}:{prob} \"\n",
"\n",
" if not total_prob:\n",
" return \"the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1\"\n",
"\n",
" if 1 - total_prob >= 0.01:\n",
" str_prediction += f\":{1-total_prob}\"\n",
" else:\n",
" str_prediction += f\":0.01\"\n",
"\n",
" return str_prediction\n",
"\n",
"\n",
"def predict_data(read_path, save_path):\n",
" data = pd.read_csv(\n",
" read_path, sep=\"\\t\", error_bad_lines=False, header=None, quoting=csv.QUOTE_NONE\n",
" )\n",
" with open(save_path, \"w\", encoding=\"UTF-8\") as file:\n",
" for _, row in data.iterrows():\n",
" words = word_tokenize(clean(row[6]))\n",
" if len(words) < 3:\n",
" prediction = \"the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1\"\n",
" else:\n",
" prediction = predict(words[-1])\n",
" file.write(prediction + \"\\n\")\n",
"\n",
"\n",
"train_model(train_data)\n",
"predict_data(\"dev-0/in.tsv.xz\", \"dev-0/out.tsv\")\n",
"predict_data(\"test-A/in.tsv.xz\", \"test-A/out.tsv\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.9.7 ('base')",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.7"
},
"vscode": {
"interpreter": {
"hash": "754a2b6bedec8aae7cfc361a819067f3f72b778cb88f366be5c7fdc236f21674"
}
}
},
"nbformat": 4,
"nbformat_minor": 5
}