!
This commit is contained in:
parent
f90040b458
commit
7b0fbff812
150
Untitled.ipynb
Normal file
150
Untitled.ipynb
Normal file
@ -0,0 +1,150 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"id": "b43d8178",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"C:\\Users\\grzyb\\AppData\\Local\\Temp/ipykernel_34768/887107210.py:86: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version.\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
" predict_data(\"dev-0/in.tsv.xz\", \"dev-0/out.tsv\")\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ename": "UnicodeEncodeError",
|
||||||
|
"evalue": "'charmap' codec can't encode character '\\u25a0' in position 0: character maps to <undefined>",
|
||||||
|
"output_type": "error",
|
||||||
|
"traceback": [
|
||||||
|
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
||||||
|
"\u001b[1;31mUnicodeEncodeError\u001b[0m Traceback (most recent call last)",
|
||||||
|
"\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_34768/887107210.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m 84\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 85\u001b[0m \u001b[0mtrain_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 86\u001b[1;33m \u001b[0mpredict_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"dev-0/in.tsv.xz\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"dev-0/out.tsv\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 87\u001b[0m \u001b[0mpredict_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"test-A/in.tsv.xz\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"test-A/out.tsv\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
|
||||||
|
"\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_34768/887107210.py\u001b[0m in \u001b[0;36mpredict_data\u001b[1;34m(read_path, save_path)\u001b[0m\n\u001b[0;32m 80\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 81\u001b[0m \u001b[0mprediction\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwords\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 82\u001b[1;33m \u001b[0mfile\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprediction\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m\"\\n\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 83\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 84\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
|
||||||
|
"\u001b[1;32mc:\\Users\\grzyb\\anaconda3\\lib\\encodings\\cp1250.py\u001b[0m in \u001b[0;36mencode\u001b[1;34m(self, input, final)\u001b[0m\n\u001b[0;32m 17\u001b[0m \u001b[1;32mclass\u001b[0m \u001b[0mIncrementalEncoder\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcodecs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mIncrementalEncoder\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 18\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mencode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfinal\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 19\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mcodecs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcharmap_encode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merrors\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mencoding_table\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 20\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 21\u001b[0m \u001b[1;32mclass\u001b[0m \u001b[0mIncrementalDecoder\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcodecs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mIncrementalDecoder\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
|
||||||
|
"\u001b[1;31mUnicodeEncodeError\u001b[0m: 'charmap' codec can't encode character '\\u25a0' in position 0: character maps to <undefined>"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"import pandas as pd\n",
|
||||||
|
"import csv\n",
|
||||||
|
"import regex as re\n",
|
||||||
|
"from nltk import bigrams, word_tokenize\n",
|
||||||
|
"from collections import Counter, defaultdict\n",
|
||||||
|
"import string\n",
|
||||||
|
"import unicodedata\n",
|
||||||
|
"\n",
|
||||||
|
"data = pd.read_csv(\n",
|
||||||
|
" \"train/in.tsv.xz\",\n",
|
||||||
|
" sep=\"\\t\",\n",
|
||||||
|
" error_bad_lines=False,\n",
|
||||||
|
" header=None,\n",
|
||||||
|
" quoting=csv.QUOTE_NONE,\n",
|
||||||
|
" \n",
|
||||||
|
")\n",
|
||||||
|
"train_labels = pd.read_csv(\n",
|
||||||
|
" \"train/expected.tsv\",\n",
|
||||||
|
" sep=\"\\t\",\n",
|
||||||
|
" error_bad_lines=False,\n",
|
||||||
|
" header=None,\n",
|
||||||
|
" quoting=csv.QUOTE_NONE,\n",
|
||||||
|
")\n",
|
||||||
|
"\n",
|
||||||
|
"train_data = data[[6, 7]]\n",
|
||||||
|
"train_data = pd.concat([train_data, train_labels], axis=1)\n",
|
||||||
|
"\n",
|
||||||
|
"train_data[\"final\"] = train_data[6] + train_data[0] + train_data[7]\n",
|
||||||
|
"\n",
|
||||||
|
"model = defaultdict(lambda: defaultdict(lambda: 0))\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def clean(text):\n",
|
||||||
|
" text = str(text).lower().replace(\"-\\\\n\", \"\").replace(\"\\\\n\", \" \")\n",
|
||||||
|
" return re.sub(r\"\\p{P}\", \"\", text)\n",
|
||||||
|
"\n",
|
||||||
|
"def train_model(data):\n",
|
||||||
|
" for _, row in data.iterrows():\n",
|
||||||
|
" words = word_tokenize(clean(row[\"final\"]))\n",
|
||||||
|
" for w1, w2 in bigrams(words, pad_left=True, pad_right=True):\n",
|
||||||
|
" if w1 and w2:\n",
|
||||||
|
" model[w1][w2] += 1\n",
|
||||||
|
" for w1 in model:\n",
|
||||||
|
" total_count = float(sum(model[w1].values()))\n",
|
||||||
|
" for w2 in model[w1]:\n",
|
||||||
|
" model[w1][w2] /= total_count\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def predict(word):\n",
|
||||||
|
" predictions = dict(model[word])\n",
|
||||||
|
" most_common = dict(Counter(predictions).most_common(5))\n",
|
||||||
|
"\n",
|
||||||
|
" total_prob = 0.0\n",
|
||||||
|
" str_prediction = \"\"\n",
|
||||||
|
"\n",
|
||||||
|
" for word, prob in most_common.items():\n",
|
||||||
|
" total_prob += prob\n",
|
||||||
|
" str_prediction += f\"{word}:{prob} \"\n",
|
||||||
|
"\n",
|
||||||
|
" if not total_prob:\n",
|
||||||
|
" return \"the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1\"\n",
|
||||||
|
"\n",
|
||||||
|
" if 1 - total_prob >= 0.01:\n",
|
||||||
|
" str_prediction += f\":{1-total_prob}\"\n",
|
||||||
|
" else:\n",
|
||||||
|
" str_prediction += f\":0.01\"\n",
|
||||||
|
"\n",
|
||||||
|
" return str_prediction\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def predict_data(read_path, save_path):\n",
|
||||||
|
" data = pd.read_csv(\n",
|
||||||
|
" read_path, sep=\"\\t\", error_bad_lines=False, header=None, quoting=csv.QUOTE_NONE\n",
|
||||||
|
" )\n",
|
||||||
|
" with open(save_path, \"w\", encoding=\"UTF-8\") as file:\n",
|
||||||
|
" for _, row in data.iterrows():\n",
|
||||||
|
" words = word_tokenize(clean(row[6]))\n",
|
||||||
|
" if len(words) < 3:\n",
|
||||||
|
" prediction = \"the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1\"\n",
|
||||||
|
" else:\n",
|
||||||
|
" prediction = predict(words[-1])\n",
|
||||||
|
" file.write(prediction + \"\\n\")\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"train_model(train_data)\n",
|
||||||
|
"predict_data(\"dev-0/in.tsv.xz\", \"dev-0/out.tsv\")\n",
|
||||||
|
"predict_data(\"test-A/in.tsv.xz\", \"test-A/out.tsv\")"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3.9.7 ('base')",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.9.7"
|
||||||
|
},
|
||||||
|
"vscode": {
|
||||||
|
"interpreter": {
|
||||||
|
"hash": "754a2b6bedec8aae7cfc361a819067f3f72b778cb88f366be5c7fdc236f21674"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 5
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user