From 7b0fbff81247a4f93e36f489075a74b3dfc9c752 Mon Sep 17 00:00:00 2001 From: Tomasz Grzybowski Date: Sat, 25 Jun 2022 21:43:48 +0200 Subject: [PATCH] ! --- Untitled.ipynb | 150 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 Untitled.ipynb diff --git a/Untitled.ipynb b/Untitled.ipynb new file mode 100644 index 0000000..8987d88 --- /dev/null +++ b/Untitled.ipynb @@ -0,0 +1,150 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 3, + "id": "b43d8178", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\grzyb\\AppData\\Local\\Temp/ipykernel_34768/887107210.py:86: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version.\n", + "\n", + "\n", + " predict_data(\"dev-0/in.tsv.xz\", \"dev-0/out.tsv\")\n" + ] + }, + { + "ename": "UnicodeEncodeError", + "evalue": "'charmap' codec can't encode character '\\u25a0' in position 0: character maps to ", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mUnicodeEncodeError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_34768/887107210.py\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 84\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 85\u001b[0m \u001b[0mtrain_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 86\u001b[1;33m \u001b[0mpredict_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"dev-0/in.tsv.xz\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"dev-0/out.tsv\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 87\u001b[0m \u001b[0mpredict_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"test-A/in.tsv.xz\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"test-A/out.tsv\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_34768/887107210.py\u001b[0m in \u001b[0;36mpredict_data\u001b[1;34m(read_path, save_path)\u001b[0m\n\u001b[0;32m 80\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 81\u001b[0m \u001b[0mprediction\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwords\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 82\u001b[1;33m \u001b[0mfile\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprediction\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m\"\\n\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 83\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 84\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32mc:\\Users\\grzyb\\anaconda3\\lib\\encodings\\cp1250.py\u001b[0m in \u001b[0;36mencode\u001b[1;34m(self, input, final)\u001b[0m\n\u001b[0;32m 17\u001b[0m \u001b[1;32mclass\u001b[0m \u001b[0mIncrementalEncoder\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcodecs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mIncrementalEncoder\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 18\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mencode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfinal\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 19\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mcodecs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcharmap_encode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merrors\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mencoding_table\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 20\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 21\u001b[0m \u001b[1;32mclass\u001b[0m \u001b[0mIncrementalDecoder\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcodecs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mIncrementalDecoder\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;31mUnicodeEncodeError\u001b[0m: 'charmap' codec can't encode character '\\u25a0' in position 0: character maps to " + ] + } + ], + "source": [ + "import pandas as pd\n", + "import csv\n", + "import regex as re\n", + "from nltk import bigrams, word_tokenize\n", + "from collections import Counter, defaultdict\n", + "import string\n", + "import unicodedata\n", + "\n", + "data = pd.read_csv(\n", + " \"train/in.tsv.xz\",\n", + " sep=\"\\t\",\n", + " error_bad_lines=False,\n", + " header=None,\n", + " quoting=csv.QUOTE_NONE,\n", + " \n", + ")\n", + "train_labels = pd.read_csv(\n", + " \"train/expected.tsv\",\n", + " sep=\"\\t\",\n", + " error_bad_lines=False,\n", + " header=None,\n", + " quoting=csv.QUOTE_NONE,\n", + ")\n", + "\n", + "train_data = data[[6, 7]]\n", + "train_data = pd.concat([train_data, train_labels], axis=1)\n", + "\n", + "train_data[\"final\"] = train_data[6] + train_data[0] + train_data[7]\n", + "\n", + "model = defaultdict(lambda: defaultdict(lambda: 0))\n", + "\n", + "\n", + "def clean(text):\n", + " text = str(text).lower().replace(\"-\\\\n\", \"\").replace(\"\\\\n\", \" \")\n", + " return re.sub(r\"\\p{P}\", \"\", text)\n", + "\n", + "def train_model(data):\n", + " for _, row in data.iterrows():\n", + " words = word_tokenize(clean(row[\"final\"]))\n", + " for w1, w2 in bigrams(words, pad_left=True, pad_right=True):\n", + " if w1 and w2:\n", + " model[w1][w2] += 1\n", + " for w1 in model:\n", + " total_count = float(sum(model[w1].values()))\n", + " for w2 in model[w1]:\n", + " model[w1][w2] /= total_count\n", + "\n", + "\n", + "def predict(word):\n", + " predictions = dict(model[word])\n", + " most_common = dict(Counter(predictions).most_common(5))\n", + "\n", + " total_prob = 0.0\n", + " str_prediction = \"\"\n", + "\n", + " for word, prob in most_common.items():\n", + " total_prob += prob\n", + " str_prediction += f\"{word}:{prob} \"\n", + "\n", + " if not total_prob:\n", + " return \"the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1\"\n", + "\n", + " if 1 - total_prob >= 0.01:\n", + " str_prediction += f\":{1-total_prob}\"\n", + " else:\n", + " str_prediction += f\":0.01\"\n", + "\n", + " return str_prediction\n", + "\n", + "\n", + "def predict_data(read_path, save_path):\n", + " data = pd.read_csv(\n", + " read_path, sep=\"\\t\", error_bad_lines=False, header=None, quoting=csv.QUOTE_NONE\n", + " )\n", + " with open(save_path, \"w\", encoding=\"UTF-8\") as file:\n", + " for _, row in data.iterrows():\n", + " words = word_tokenize(clean(row[6]))\n", + " if len(words) < 3:\n", + " prediction = \"the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1\"\n", + " else:\n", + " prediction = predict(words[-1])\n", + " file.write(prediction + \"\\n\")\n", + "\n", + "\n", + "train_model(train_data)\n", + "predict_data(\"dev-0/in.tsv.xz\", \"dev-0/out.tsv\")\n", + "predict_data(\"test-A/in.tsv.xz\", \"test-A/out.tsv\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.9.7 ('base')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + }, + "vscode": { + "interpreter": { + "hash": "754a2b6bedec8aae7cfc361a819067f3f72b778cb88f366be5c7fdc236f21674" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}