plusalpha
This commit is contained in:
parent
61e88a9c8c
commit
5b91c8cf0a
1
.gitignore
vendored
1
.gitignore
vendored
@ -6,3 +6,4 @@
|
|||||||
*.o
|
*.o
|
||||||
.DS_Store
|
.DS_Store
|
||||||
.token
|
.token
|
||||||
|
.ipynb_checkpoints/
|
||||||
|
|||||||
10519
dev-0/out.tsv
Normal file
10519
dev-0/out.tsv
Normal file
File diff suppressed because it is too large
Load Diff
250
run.ipynb
Normal file
250
run.ipynb
Normal file
@ -0,0 +1,250 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Loading data...\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"\r",
|
||||||
|
"0it [00:00, ?it/s]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Training model...\n",
|
||||||
|
"1/2\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"50000it [03:35, 232.50it/s]\n",
|
||||||
|
" 0%| | 8/753550 [00:00<3:31:51, 59.28it/s]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"2/2\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"100%|██████████████████████████████████████████████████████████████████████| 753550/753550 [00:04<00:00, 176601.27it/s]\n",
|
||||||
|
" 0%| | 3/753550 [00:00<8:51:51, 23.61it/s]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Smoothing...\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"100%|██████████████████████████████████████████████████████████████████████| 753550/753550 [00:06<00:00, 117904.94it/s]\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Predicting...\n",
|
||||||
|
"Dev set\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"10519it [02:07, 82.51it/s] \n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Test set\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"7414it [01:16, 96.50it/s] \n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"import pandas as pd\n",
|
||||||
|
"import csv\n",
|
||||||
|
"import regex as re\n",
|
||||||
|
"from nltk import bigrams, word_tokenize\n",
|
||||||
|
"from collections import Counter, defaultdict\n",
|
||||||
|
"import string\n",
|
||||||
|
"import unicodedata\n",
|
||||||
|
"from tqdm import tqdm\n",
|
||||||
|
"\n",
|
||||||
|
"pd.set_option('display.max_columns', None)\n",
|
||||||
|
"pd.set_option('display.max_rows', None)\n",
|
||||||
|
"\n",
|
||||||
|
"NROWS = 50000\n",
|
||||||
|
"ALPHA = 0.1\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def etl():\n",
|
||||||
|
" data = pd.read_csv(\n",
|
||||||
|
" \"train/in.tsv.xz\",\n",
|
||||||
|
" sep=\"\\t\",\n",
|
||||||
|
" error_bad_lines=False,\n",
|
||||||
|
" header=None,\n",
|
||||||
|
" quoting=csv.QUOTE_NONE,\n",
|
||||||
|
" nrows=NROWS\n",
|
||||||
|
" )\n",
|
||||||
|
" train_labels = pd.read_csv(\n",
|
||||||
|
" \"train/expected.tsv\",\n",
|
||||||
|
" sep=\"\\t\",\n",
|
||||||
|
" error_bad_lines=False,\n",
|
||||||
|
" header=None,\n",
|
||||||
|
" quoting=csv.QUOTE_NONE,\n",
|
||||||
|
" nrows=NROWS\n",
|
||||||
|
" )\n",
|
||||||
|
" \n",
|
||||||
|
" train_data = data[[6, 7]]\n",
|
||||||
|
" train_data = pd.concat([train_data, train_labels], axis=1)\n",
|
||||||
|
"\n",
|
||||||
|
" train_data[\"final\"] = train_data[6] + train_data[0] + train_data[7]\n",
|
||||||
|
"\n",
|
||||||
|
" model = defaultdict(lambda: defaultdict(lambda: 0))\n",
|
||||||
|
" return train_data, model\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def clean(text):\n",
|
||||||
|
" text = str(text).lower().replace(\"-\\\\n\", \"\").replace(\"\\\\n\", \" \")\n",
|
||||||
|
" return re.sub(r\"\\p{P}\", \"\", text)\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def train_model(data):\n",
|
||||||
|
" print(\"1/2\")\n",
|
||||||
|
" for _, row in tqdm(data.iterrows()):\n",
|
||||||
|
" words = word_tokenize(clean(row[\"final\"]))\n",
|
||||||
|
" for word_1, word_2 in bigrams(words, pad_left=True, pad_right=True):\n",
|
||||||
|
" if word_1 and word_2:\n",
|
||||||
|
" vocab.add(word_1)\n",
|
||||||
|
" vocab.add(word_2)\n",
|
||||||
|
" model[word_1][word_2] += 1\n",
|
||||||
|
" print(\"2/2\")\n",
|
||||||
|
" for word_1 in tqdm(model):\n",
|
||||||
|
" total_count = float(sum(model[word_1].values()))\n",
|
||||||
|
" for word_2 in model[word_1]:\n",
|
||||||
|
" model[word_1][word_2] /= total_count\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def predict(word):\n",
|
||||||
|
" predictions = dict(model[word])\n",
|
||||||
|
" most_common = dict(Counter(predictions).most_common(5))\n",
|
||||||
|
"\n",
|
||||||
|
" total_prob = 0.0\n",
|
||||||
|
" str_prediction = \"\"\n",
|
||||||
|
"\n",
|
||||||
|
" for word, prob in most_common.items():\n",
|
||||||
|
" total_prob += prob\n",
|
||||||
|
" str_prediction += f\"{word}:{prob} \"\n",
|
||||||
|
"\n",
|
||||||
|
" if not total_prob:\n",
|
||||||
|
" return \"the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1\"\n",
|
||||||
|
"\n",
|
||||||
|
" if 1 - total_prob >= 0.01:\n",
|
||||||
|
" str_prediction += f\":{1-total_prob}\"\n",
|
||||||
|
" else:\n",
|
||||||
|
" str_prediction += f\":0.01\"\n",
|
||||||
|
"\n",
|
||||||
|
" return str_prediction\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def predict_data(read_path, save_path):\n",
|
||||||
|
" data = pd.read_csv(\n",
|
||||||
|
" read_path, sep=\"\\t\", error_bad_lines=False, header=None, quoting=csv.QUOTE_NONE\n",
|
||||||
|
" )\n",
|
||||||
|
" with open(save_path, \"w\", encoding=\"utf-8\") as file:\n",
|
||||||
|
" for _, row in tqdm(data.iterrows()):\n",
|
||||||
|
" words = word_tokenize(clean(row[6]))\n",
|
||||||
|
" if len(words) < 3:\n",
|
||||||
|
" prediction = \"the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1\"\n",
|
||||||
|
" else:\n",
|
||||||
|
" prediction = predict(words[-1])\n",
|
||||||
|
" file.write(prediction + \"\\n\")\n",
|
||||||
|
" \n",
|
||||||
|
"def plus_alpha_smoothing():\n",
|
||||||
|
" model_len = len(model)\n",
|
||||||
|
" for word_1 in tqdm(model):\n",
|
||||||
|
" word_1_occurrences = sum(model[word_1].values())\n",
|
||||||
|
" for word_2 in model[word_1]:\n",
|
||||||
|
" model[word_1][word_2] += ALPHA\n",
|
||||||
|
" model[word_1][word_2] /= float(word_1_occurrences + ALPHA + len(word_2))\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"print(\"Loading data...\")\n",
|
||||||
|
"train_data, model = etl()\n",
|
||||||
|
"vocab = set()\n",
|
||||||
|
"print(\"Training model...\")\n",
|
||||||
|
"train_model(train_data)\n",
|
||||||
|
"print(\"Smoothing...\")\n",
|
||||||
|
"plus_alpha_smoothing()\n",
|
||||||
|
"print(\"Predicting...\")\n",
|
||||||
|
"print(\"Dev set\")\n",
|
||||||
|
"predict_data(\"dev-0/in.tsv.xz\", \"dev-0/out.tsv\")\n",
|
||||||
|
"print(\"Test set\")\n",
|
||||||
|
"predict_data(\"test-A/in.tsv.xz\", \"test-A/out.tsv\")\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.8.5"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 4
|
||||||
|
}
|
||||||
133
run.py
Normal file
133
run.py
Normal file
@ -0,0 +1,133 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
# coding: utf-8
|
||||||
|
|
||||||
|
# In[2]:
|
||||||
|
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
import csv
|
||||||
|
import regex as re
|
||||||
|
from nltk import bigrams, word_tokenize
|
||||||
|
from collections import Counter, defaultdict
|
||||||
|
import string
|
||||||
|
import unicodedata
|
||||||
|
from tqdm import tqdm
|
||||||
|
|
||||||
|
pd.set_option('display.max_columns', None)
|
||||||
|
pd.set_option('display.max_rows', None)
|
||||||
|
|
||||||
|
NROWS = 50000
|
||||||
|
ALPHA = 0.1
|
||||||
|
|
||||||
|
|
||||||
|
def etl():
|
||||||
|
data = pd.read_csv(
|
||||||
|
"train/in.tsv.xz",
|
||||||
|
sep="\t",
|
||||||
|
error_bad_lines=False,
|
||||||
|
header=None,
|
||||||
|
quoting=csv.QUOTE_NONE,
|
||||||
|
nrows=NROWS
|
||||||
|
)
|
||||||
|
train_labels = pd.read_csv(
|
||||||
|
"train/expected.tsv",
|
||||||
|
sep="\t",
|
||||||
|
error_bad_lines=False,
|
||||||
|
header=None,
|
||||||
|
quoting=csv.QUOTE_NONE,
|
||||||
|
nrows=NROWS
|
||||||
|
)
|
||||||
|
|
||||||
|
train_data = data[[6, 7]]
|
||||||
|
train_data = pd.concat([train_data, train_labels], axis=1)
|
||||||
|
|
||||||
|
train_data["final"] = train_data[6] + train_data[0] + train_data[7]
|
||||||
|
|
||||||
|
model = defaultdict(lambda: defaultdict(lambda: 0))
|
||||||
|
return train_data, model
|
||||||
|
|
||||||
|
|
||||||
|
def clean(text):
|
||||||
|
text = str(text).lower().replace("-\\n", "").replace("\\n", " ")
|
||||||
|
return re.sub(r"\p{P}", "", text)
|
||||||
|
|
||||||
|
|
||||||
|
def train_model(data):
|
||||||
|
print("1/2")
|
||||||
|
for _, row in tqdm(data.iterrows()):
|
||||||
|
words = word_tokenize(clean(row["final"]))
|
||||||
|
for word_1, word_2 in bigrams(words, pad_left=True, pad_right=True):
|
||||||
|
if word_1 and word_2:
|
||||||
|
vocab.add(word_1)
|
||||||
|
vocab.add(word_2)
|
||||||
|
model[word_1][word_2] += 1
|
||||||
|
print("2/2")
|
||||||
|
for word_1 in tqdm(model):
|
||||||
|
total_count = float(sum(model[word_1].values()))
|
||||||
|
for word_2 in model[word_1]:
|
||||||
|
model[word_1][word_2] /= total_count
|
||||||
|
|
||||||
|
|
||||||
|
def predict(word):
|
||||||
|
predictions = dict(model[word])
|
||||||
|
most_common = dict(Counter(predictions).most_common(5))
|
||||||
|
|
||||||
|
total_prob = 0.0
|
||||||
|
str_prediction = ""
|
||||||
|
|
||||||
|
for word, prob in most_common.items():
|
||||||
|
total_prob += prob
|
||||||
|
str_prediction += f"{word}:{prob} "
|
||||||
|
|
||||||
|
if not total_prob:
|
||||||
|
return "the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1"
|
||||||
|
|
||||||
|
if 1 - total_prob >= 0.01:
|
||||||
|
str_prediction += f":{1-total_prob}"
|
||||||
|
else:
|
||||||
|
str_prediction += f":0.01"
|
||||||
|
|
||||||
|
return str_prediction
|
||||||
|
|
||||||
|
|
||||||
|
def predict_data(read_path, save_path):
|
||||||
|
data = pd.read_csv(
|
||||||
|
read_path, sep="\t", error_bad_lines=False, header=None, quoting=csv.QUOTE_NONE
|
||||||
|
)
|
||||||
|
with open(save_path, "w", encoding="utf-8") as file:
|
||||||
|
for _, row in tqdm(data.iterrows()):
|
||||||
|
words = word_tokenize(clean(row[6]))
|
||||||
|
if len(words) < 3:
|
||||||
|
prediction = "the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1"
|
||||||
|
else:
|
||||||
|
prediction = predict(words[-1])
|
||||||
|
file.write(prediction + "\n")
|
||||||
|
|
||||||
|
def plus_alpha_smoothing():
|
||||||
|
model_len = len(model)
|
||||||
|
for word_1 in tqdm(model):
|
||||||
|
word_1_occurrences = sum(model[word_1].values())
|
||||||
|
for word_2 in model[word_1]:
|
||||||
|
model[word_1][word_2] += ALPHA
|
||||||
|
model[word_1][word_2] /= float(word_1_occurrences + ALPHA + len(word_2))
|
||||||
|
|
||||||
|
|
||||||
|
print("Loading data...")
|
||||||
|
train_data, model = etl()
|
||||||
|
vocab = set()
|
||||||
|
print("Training model...")
|
||||||
|
train_model(train_data)
|
||||||
|
print("Smoothing...")
|
||||||
|
plus_alpha_smoothing()
|
||||||
|
print("Predicting...")
|
||||||
|
print("Dev set")
|
||||||
|
predict_data("dev-0/in.tsv.xz", "dev-0/out.tsv")
|
||||||
|
print("Test set")
|
||||||
|
predict_data("test-A/in.tsv.xz", "test-A/out.tsv")
|
||||||
|
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
7414
test-A/out.tsv
Normal file
7414
test-A/out.tsv
Normal file
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user