593 lines
24 KiB
Plaintext
593 lines
24 KiB
Plaintext
|
{
|
|||
|
|
"cells": [
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 1,
|
|||
|
|
"metadata": {
|
|||
|
|
"colab": {
|
|||
|
|
"base_uri": "https://localhost:8080/"
|
|||
|
|
},
|
|||
|
|
"id": "pzHZprgFPh08",
|
|||
|
|
"outputId": "0f27cdcb-57b9-4cec-85c4-dac8497403ab"
|
|||
|
|
},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"name": "stderr",
|
|||
|
|
"text": [
|
|||
|
|
"[nltk_data] Downloading package punkt to /root/nltk_data...\n",
|
|||
|
|
"[nltk_data] Package punkt is already up-to-date!\n"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"output_type": "execute_result",
|
|||
|
|
"data": {
|
|||
|
|
"text/plain": [
|
|||
|
|
"True"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"metadata": {},
|
|||
|
|
"execution_count": 1
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"import pandas as pd\n",
|
|||
|
|
"import csv\n",
|
|||
|
|
"import regex as re\n",
|
|||
|
|
"from nltk import trigrams, word_tokenize\n",
|
|||
|
|
"from collections import Counter, defaultdict\n",
|
|||
|
|
"import nltk\n",
|
|||
|
|
"nltk.download('punkt')"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"source": [
|
|||
|
|
"from google.colab import drive\n",
|
|||
|
|
"drive.mount('/content/gdrive')\n"
|
|||
|
|
],
|
|||
|
|
"metadata": {
|
|||
|
|
"colab": {
|
|||
|
|
"base_uri": "https://localhost:8080/"
|
|||
|
|
},
|
|||
|
|
"id": "wIY8w_ZxPshw",
|
|||
|
|
"outputId": "ed7fda0c-8a7b-4aa8-9109-9804cabf4d79"
|
|||
|
|
},
|
|||
|
|
"execution_count": 2,
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"name": "stdout",
|
|||
|
|
"text": [
|
|||
|
|
"Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"source": [
|
|||
|
|
"cd '/content/gdrive/MyDrive/challenging-america-word-gap-prediction/'"
|
|||
|
|
],
|
|||
|
|
"metadata": {
|
|||
|
|
"colab": {
|
|||
|
|
"base_uri": "https://localhost:8080/"
|
|||
|
|
},
|
|||
|
|
"id": "PS4mnw-1P3cP",
|
|||
|
|
"outputId": "c86f074e-5c26-4f00-eddb-d267174a4297"
|
|||
|
|
},
|
|||
|
|
"execution_count": 3,
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"name": "stdout",
|
|||
|
|
"text": [
|
|||
|
|
"/content/gdrive/MyDrive/challenging-america-word-gap-prediction\n"
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 4,
|
|||
|
|
"metadata": {
|
|||
|
|
"colab": {
|
|||
|
|
"base_uri": "https://localhost:8080/",
|
|||
|
|
"height": 701
|
|||
|
|
},
|
|||
|
|
"id": "3r2WzoHmPh1G",
|
|||
|
|
"outputId": "1853f1c3-6e28-497e-b34b-21c193fcd6e8"
|
|||
|
|
},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"name": "stderr",
|
|||
|
|
"text": [
|
|||
|
|
"<ipython-input-4-06713320f790>:1: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
|
|||
|
|
"\n",
|
|||
|
|
"\n",
|
|||
|
|
" train_data = pd.read_csv('train/in.tsv.xz', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)\n",
|
|||
|
|
"<ipython-input-4-06713320f790>:1: FutureWarning: The warn_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
|
|||
|
|
"\n",
|
|||
|
|
"\n",
|
|||
|
|
" train_data = pd.read_csv('train/in.tsv.xz', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)\n",
|
|||
|
|
"<ipython-input-4-06713320f790>:2: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
|
|||
|
|
"\n",
|
|||
|
|
"\n",
|
|||
|
|
" train_labels = pd.read_csv('train/expected.tsv', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)\n",
|
|||
|
|
"<ipython-input-4-06713320f790>:2: FutureWarning: The warn_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
|
|||
|
|
"\n",
|
|||
|
|
"\n",
|
|||
|
|
" train_labels = pd.read_csv('train/expected.tsv', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)\n"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"output_type": "execute_result",
|
|||
|
|
"data": {
|
|||
|
|
"text/plain": [
|
|||
|
|
" 6 \\\n",
|
|||
|
|
"0 came fiom the last place to this\\nplace, and t... \n",
|
|||
|
|
"1 MB. BOOT'S POLITICAL OBEED\\nAttempt to imagine... \n",
|
|||
|
|
"2 \"Thera were in 1771 only aeventy-nine\\n*ub*erl... \n",
|
|||
|
|
"3 A gixnl man y nitereRtiiiv dii-clos-\\nur«s reg... \n",
|
|||
|
|
"4 Tin: 188UB TV THF BBABBT QABJE\\nMr. Schiffs *t... \n",
|
|||
|
|
"... ... \n",
|
|||
|
|
"432017 Sam Clendenin bad a fancy for Ui«\\nscience of ... \n",
|
|||
|
|
"432018 Wita.htt halting the party ware dilven to the ... \n",
|
|||
|
|
"432019 It was the last thing that either of\\nthem exp... \n",
|
|||
|
|
"432020 settlement with the department.\\nIt is also sh... \n",
|
|||
|
|
"432021 Flour quotations—low extras at 1 R0®2 50;\\ncit... \n",
|
|||
|
|
"\n",
|
|||
|
|
" 7 0 \n",
|
|||
|
|
"0 said\\nit's all squash. The best I could get\\ni... lie \n",
|
|||
|
|
"1 \\ninto a proper perspective with those\\nminor ... himself \n",
|
|||
|
|
"2 all notU\\nashore and afloat arc subjects for I... of \n",
|
|||
|
|
"3 ceucju l< d no; <o waste it nud so\\nsunk it in... ably \n",
|
|||
|
|
"4 ascertained w? OCt the COOltS of ibis\\nletale ... j \n",
|
|||
|
|
"... ... ... \n",
|
|||
|
|
"432017 \\nSam was arrested.\\nThe case excited a great ... and \n",
|
|||
|
|
"432018 through the alnp the »Uitors laapeeeed tia.»\\n... paasliic \n",
|
|||
|
|
"432019 Agua Negra across the line.\\nIt was a grim pla... for \n",
|
|||
|
|
"432020 \\na note of Wood, Dialogue fc Co., for\\nc27,im... for \n",
|
|||
|
|
"432021 3214c;do White at 3614c: Mixed Western at\\n331... at \n",
|
|||
|
|
"\n",
|
|||
|
|
"[432022 rows x 3 columns]"
|
|||
|
|
],
|
|||
|
|
"text/html": [
|
|||
|
|
"\n",
|
|||
|
|
" <div id=\"df-89d6ad5d-c536-4a35-8506-b69b94e55deb\">\n",
|
|||
|
|
" <div class=\"colab-df-container\">\n",
|
|||
|
|
" <div>\n",
|
|||
|
|
"<style scoped>\n",
|
|||
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|||
|
|
" vertical-align: middle;\n",
|
|||
|
|
" }\n",
|
|||
|
|
"\n",
|
|||
|
|
" .dataframe tbody tr th {\n",
|
|||
|
|
" vertical-align: top;\n",
|
|||
|
|
" }\n",
|
|||
|
|
"\n",
|
|||
|
|
" .dataframe thead th {\n",
|
|||
|
|
" text-align: right;\n",
|
|||
|
|
" }\n",
|
|||
|
|
"</style>\n",
|
|||
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|||
|
|
" <thead>\n",
|
|||
|
|
" <tr style=\"text-align: right;\">\n",
|
|||
|
|
" <th></th>\n",
|
|||
|
|
" <th>6</th>\n",
|
|||
|
|
" <th>7</th>\n",
|
|||
|
|
" <th>0</th>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" </thead>\n",
|
|||
|
|
" <tbody>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>0</th>\n",
|
|||
|
|
" <td>came fiom the last place to this\\nplace, and t...</td>\n",
|
|||
|
|
" <td>said\\nit's all squash. The best I could get\\ni...</td>\n",
|
|||
|
|
" <td>lie</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>1</th>\n",
|
|||
|
|
" <td>MB. BOOT'S POLITICAL OBEED\\nAttempt to imagine...</td>\n",
|
|||
|
|
" <td>\\ninto a proper perspective with those\\nminor ...</td>\n",
|
|||
|
|
" <td>himself</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>2</th>\n",
|
|||
|
|
" <td>\"Thera were in 1771 only aeventy-nine\\n*ub*erl...</td>\n",
|
|||
|
|
" <td>all notU\\nashore and afloat arc subjects for I...</td>\n",
|
|||
|
|
" <td>of</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>3</th>\n",
|
|||
|
|
" <td>A gixnl man y nitereRtiiiv dii-clos-\\nur«s reg...</td>\n",
|
|||
|
|
" <td>ceucju l< d no; <o waste it nud so\\nsunk it in...</td>\n",
|
|||
|
|
" <td>ably</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>4</th>\n",
|
|||
|
|
" <td>Tin: 188UB TV THF BBABBT QABJE\\nMr. Schiffs *t...</td>\n",
|
|||
|
|
" <td>ascertained w? OCt the COOltS of ibis\\nletale ...</td>\n",
|
|||
|
|
" <td>j</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>...</th>\n",
|
|||
|
|
" <td>...</td>\n",
|
|||
|
|
" <td>...</td>\n",
|
|||
|
|
" <td>...</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>432017</th>\n",
|
|||
|
|
" <td>Sam Clendenin bad a fancy for Ui«\\nscience of ...</td>\n",
|
|||
|
|
" <td>\\nSam was arrested.\\nThe case excited a great ...</td>\n",
|
|||
|
|
" <td>and</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>432018</th>\n",
|
|||
|
|
" <td>Wita.htt halting the party ware dilven to the ...</td>\n",
|
|||
|
|
" <td>through the alnp the »Uitors laapeeeed tia.»\\n...</td>\n",
|
|||
|
|
" <td>paasliic</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>432019</th>\n",
|
|||
|
|
" <td>It was the last thing that either of\\nthem exp...</td>\n",
|
|||
|
|
" <td>Agua Negra across the line.\\nIt was a grim pla...</td>\n",
|
|||
|
|
" <td>for</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>432020</th>\n",
|
|||
|
|
" <td>settlement with the department.\\nIt is also sh...</td>\n",
|
|||
|
|
" <td>\\na note of Wood, Dialogue fc Co., for\\nc27,im...</td>\n",
|
|||
|
|
" <td>for</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" <tr>\n",
|
|||
|
|
" <th>432021</th>\n",
|
|||
|
|
" <td>Flour quotations—low extras at 1 R0®2 50;\\ncit...</td>\n",
|
|||
|
|
" <td>3214c;do White at 3614c: Mixed Western at\\n331...</td>\n",
|
|||
|
|
" <td>at</td>\n",
|
|||
|
|
" </tr>\n",
|
|||
|
|
" </tbody>\n",
|
|||
|
|
"</table>\n",
|
|||
|
|
"<p>432022 rows × 3 columns</p>\n",
|
|||
|
|
"</div>\n",
|
|||
|
|
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-89d6ad5d-c536-4a35-8506-b69b94e55deb')\"\n",
|
|||
|
|
" title=\"Convert this dataframe to an interactive table.\"\n",
|
|||
|
|
" style=\"display:none;\">\n",
|
|||
|
|
" \n",
|
|||
|
|
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
|
|||
|
|
" width=\"24px\">\n",
|
|||
|
|
" <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
|
|||
|
|
" <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
|
|||
|
|
" </svg>\n",
|
|||
|
|
" </button>\n",
|
|||
|
|
" \n",
|
|||
|
|
" <style>\n",
|
|||
|
|
" .colab-df-container {\n",
|
|||
|
|
" display:flex;\n",
|
|||
|
|
" flex-wrap:wrap;\n",
|
|||
|
|
" gap: 12px;\n",
|
|||
|
|
" }\n",
|
|||
|
|
"\n",
|
|||
|
|
" .colab-df-convert {\n",
|
|||
|
|
" background-color: #E8F0FE;\n",
|
|||
|
|
" border: none;\n",
|
|||
|
|
" border-radius: 50%;\n",
|
|||
|
|
" cursor: pointer;\n",
|
|||
|
|
" display: none;\n",
|
|||
|
|
" fill: #1967D2;\n",
|
|||
|
|
" height: 32px;\n",
|
|||
|
|
" padding: 0 0 0 0;\n",
|
|||
|
|
" width: 32px;\n",
|
|||
|
|
" }\n",
|
|||
|
|
"\n",
|
|||
|
|
" .colab-df-convert:hover {\n",
|
|||
|
|
" background-color: #E2EBFA;\n",
|
|||
|
|
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
|
|||
|
|
" fill: #174EA6;\n",
|
|||
|
|
" }\n",
|
|||
|
|
"\n",
|
|||
|
|
" [theme=dark] .colab-df-convert {\n",
|
|||
|
|
" background-color: #3B4455;\n",
|
|||
|
|
" fill: #D2E3FC;\n",
|
|||
|
|
" }\n",
|
|||
|
|
"\n",
|
|||
|
|
" [theme=dark] .colab-df-convert:hover {\n",
|
|||
|
|
" background-color: #434B5C;\n",
|
|||
|
|
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
|
|||
|
|
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
|
|||
|
|
" fill: #FFFFFF;\n",
|
|||
|
|
" }\n",
|
|||
|
|
" </style>\n",
|
|||
|
|
"\n",
|
|||
|
|
" <script>\n",
|
|||
|
|
" const buttonEl =\n",
|
|||
|
|
" document.querySelector('#df-89d6ad5d-c536-4a35-8506-b69b94e55deb button.colab-df-convert');\n",
|
|||
|
|
" buttonEl.style.display =\n",
|
|||
|
|
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
|
|||
|
|
"\n",
|
|||
|
|
" async function convertToInteractive(key) {\n",
|
|||
|
|
" const element = document.querySelector('#df-89d6ad5d-c536-4a35-8506-b69b94e55deb');\n",
|
|||
|
|
" const dataTable =\n",
|
|||
|
|
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
|
|||
|
|
" [key], {});\n",
|
|||
|
|
" if (!dataTable) return;\n",
|
|||
|
|
"\n",
|
|||
|
|
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
|
|||
|
|
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
|
|||
|
|
" + ' to learn more about interactive tables.';\n",
|
|||
|
|
" element.innerHTML = '';\n",
|
|||
|
|
" dataTable['output_type'] = 'display_data';\n",
|
|||
|
|
" await google.colab.output.renderOutput(dataTable, element);\n",
|
|||
|
|
" const docLink = document.createElement('div');\n",
|
|||
|
|
" docLink.innerHTML = docLinkHtml;\n",
|
|||
|
|
" element.appendChild(docLink);\n",
|
|||
|
|
" }\n",
|
|||
|
|
" </script>\n",
|
|||
|
|
" </div>\n",
|
|||
|
|
" </div>\n",
|
|||
|
|
" "
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
"metadata": {},
|
|||
|
|
"execution_count": 4
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"train_data = pd.read_csv('train/in.tsv.xz', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)\n",
|
|||
|
|
"train_labels = pd.read_csv('train/expected.tsv', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)\n",
|
|||
|
|
"\n",
|
|||
|
|
"train_data = train_data[[6, 7]]\n",
|
|||
|
|
"train_data = pd.concat([train_data, train_labels], axis=1)\n",
|
|||
|
|
"train_data\n"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"source": [
|
|||
|
|
"train_data = train_data[:120000]"
|
|||
|
|
],
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "ifrGODxOTuK7"
|
|||
|
|
},
|
|||
|
|
"execution_count": 5,
|
|||
|
|
"outputs": []
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 6,
|
|||
|
|
"metadata": {
|
|||
|
|
"colab": {
|
|||
|
|
"base_uri": "https://localhost:8080/"
|
|||
|
|
},
|
|||
|
|
"id": "JJTvit-qPh1L",
|
|||
|
|
"outputId": "58f187c9-6561-4418-d0e0-fbaca2260b70"
|
|||
|
|
},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"name": "stderr",
|
|||
|
|
"text": [
|
|||
|
|
"<ipython-input-6-b31274590998>:1: SettingWithCopyWarning: \n",
|
|||
|
|
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
|
|||
|
|
"Try using .loc[row_indexer,col_indexer] = value instead\n",
|
|||
|
|
"\n",
|
|||
|
|
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
|
|||
|
|
" train_data['final'] = train_data[6] + train_data[0] + train_data[7]\n"
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"train_data['final'] = train_data[6] + train_data[0] + train_data[7]\n"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 7,
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "0GzBUzFkPh1M"
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"model = defaultdict(lambda: defaultdict(lambda: 0))"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 8,
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "IViVFNNzPh1O"
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"def clean_text(text):\n",
|
|||
|
|
" text = text.lower().replace('-\\\\n', '').replace('\\\\n', ' ')\n",
|
|||
|
|
" text = re.sub(r'\\p{P}', '', text)\n",
|
|||
|
|
" return text"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 8,
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "ZXkV4cLFPh1P"
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": []
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 9,
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "3Y4_y97tPh1R"
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"for index, row in train_data.iterrows():\n",
|
|||
|
|
" text = clean_text(str(row['final']))\n",
|
|||
|
|
" words = word_tokenize(text)\n",
|
|||
|
|
" for w1, w2, w3 in trigrams(words, pad_right=True, pad_left=True):\n",
|
|||
|
|
" if w1 and w2 and w3:\n",
|
|||
|
|
" model[(w2, w3)][w1] += 1"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 10,
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "V87WPI1PPh1S"
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"for w2_w3 in model:\n",
|
|||
|
|
" total_count = float(sum(model[w2_w3].values()))\n",
|
|||
|
|
" for w1 in model[w2_w3]:\n",
|
|||
|
|
" model[w2_w3][w1] /= total_count\n"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 11,
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "TP-eEc4OPh1T"
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"def predict_probs(word1, word2):\n",
|
|||
|
|
" raw_prediction = dict(model[word1, word2])\n",
|
|||
|
|
" prediction = dict(Counter(raw_prediction).most_common(6))\n",
|
|||
|
|
" \n",
|
|||
|
|
" total_prob = 0.0\n",
|
|||
|
|
" str_prediction = ''\n",
|
|||
|
|
"\n",
|
|||
|
|
" for word, prob in prediction.items():\n",
|
|||
|
|
" total_prob += prob\n",
|
|||
|
|
" str_prediction += f'{word}:{prob} '\n",
|
|||
|
|
"\n",
|
|||
|
|
" if total_prob == 0.0:\n",
|
|||
|
|
" return 'from:0.2 the:0.2 to:0.2 a:0.1 and:0.1 of:0.1 :0.1'\n",
|
|||
|
|
"\n",
|
|||
|
|
" remaining_prob = 1 - total_prob\n",
|
|||
|
|
"\n",
|
|||
|
|
" if remaining_prob < 0.01:\n",
|
|||
|
|
" remaining_prob = 0.01\n",
|
|||
|
|
" \n",
|
|||
|
|
" str_prediction += f':{remaining_prob}'\n",
|
|||
|
|
" \n",
|
|||
|
|
" return str_prediction"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 12,
|
|||
|
|
"metadata": {
|
|||
|
|
"colab": {
|
|||
|
|
"base_uri": "https://localhost:8080/"
|
|||
|
|
},
|
|||
|
|
"id": "aehup5qzPh1W",
|
|||
|
|
"outputId": "c4443682-95fb-43f0-c04d-726a65b4f6b9"
|
|||
|
|
},
|
|||
|
|
"outputs": [
|
|||
|
|
{
|
|||
|
|
"output_type": "stream",
|
|||
|
|
"name": "stderr",
|
|||
|
|
"text": [
|
|||
|
|
"<ipython-input-12-94466712d0ba>:1: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
|
|||
|
|
"\n",
|
|||
|
|
"\n",
|
|||
|
|
" dev_data = pd.read_csv('dev-0/in.tsv.xz', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)\n",
|
|||
|
|
"<ipython-input-12-94466712d0ba>:1: FutureWarning: The warn_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
|
|||
|
|
"\n",
|
|||
|
|
"\n",
|
|||
|
|
" dev_data = pd.read_csv('dev-0/in.tsv.xz', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)\n",
|
|||
|
|
"<ipython-input-12-94466712d0ba>:2: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
|
|||
|
|
"\n",
|
|||
|
|
"\n",
|
|||
|
|
" test_data = pd.read_csv('test-A/in.tsv.xz', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)\n",
|
|||
|
|
"<ipython-input-12-94466712d0ba>:2: FutureWarning: The warn_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
|
|||
|
|
"\n",
|
|||
|
|
"\n",
|
|||
|
|
" test_data = pd.read_csv('test-A/in.tsv.xz', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)\n"
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"source": [
|
|||
|
|
"dev_data = pd.read_csv('dev-0/in.tsv.xz', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)\n",
|
|||
|
|
"test_data = pd.read_csv('test-A/in.tsv.xz', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)\n"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 13,
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "bTCUDesePh1X"
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"with open('dev-0/out.tsv', 'w') as file:\n",
|
|||
|
|
" for index, row in dev_data.iterrows():\n",
|
|||
|
|
" text = clean_text(str(row[7]))\n",
|
|||
|
|
" words = word_tokenize(text)\n",
|
|||
|
|
" if len(words) < 4:\n",
|
|||
|
|
" prediction = 'from:0.2 the:0.2 to:0.2 a:0.1 and:0.1 of:0.1 :0.1'\n",
|
|||
|
|
" else:\n",
|
|||
|
|
" prediction = predict_probs(words[0], words[1])\n",
|
|||
|
|
" file.write(prediction + '\\n')\n",
|
|||
|
|
"\n"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 14,
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "kzg8J0hAPh1Y"
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": [
|
|||
|
|
"\n",
|
|||
|
|
"with open('test-A/out.tsv', 'w') as file:\n",
|
|||
|
|
" for index, row in test_data.iterrows():\n",
|
|||
|
|
" text = clean_text(str(row[7]))\n",
|
|||
|
|
" words = word_tokenize(text)\n",
|
|||
|
|
" if len(words) < 4:\n",
|
|||
|
|
" prediction = 'from:0.2 the:0.2 to:0.2 a:0.1 and:0.1 of:0.1 :0.1'\n",
|
|||
|
|
" else:\n",
|
|||
|
|
" prediction = predict_probs(words[0], words[1])\n",
|
|||
|
|
" file.write(prediction + '\\n')"
|
|||
|
|
]
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"cell_type": "code",
|
|||
|
|
"execution_count": 14,
|
|||
|
|
"metadata": {
|
|||
|
|
"id": "s01_AhIbPh1b"
|
|||
|
|
},
|
|||
|
|
"outputs": [],
|
|||
|
|
"source": []
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"metadata": {
|
|||
|
|
"kernelspec": {
|
|||
|
|
"display_name": "Python 3 (ipykernel)",
|
|||
|
|
"language": "python",
|
|||
|
|
"name": "python3"
|
|||
|
|
},
|
|||
|
|
"language_info": {
|
|||
|
|
"codemirror_mode": {
|
|||
|
|
"name": "ipython",
|
|||
|
|
"version": 3
|
|||
|
|
},
|
|||
|
|
"file_extension": ".py",
|
|||
|
|
"mimetype": "text/x-python",
|
|||
|
|
"name": "python",
|
|||
|
|
"nbconvert_exporter": "python",
|
|||
|
|
"pygments_lexer": "ipython3",
|
|||
|
|
"version": "3.10.2"
|
|||
|
|
},
|
|||
|
|
"colab": {
|
|||
|
|
"provenance": []
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"nbformat": 4,
|
|||
|
|
"nbformat_minor": 0
|
|||
|
|
}
|