plusalpha
This commit is contained in:
parent
61e88a9c8c
commit
5b91c8cf0a
1
.gitignore
vendored
1
.gitignore
vendored
@ -6,3 +6,4 @@
|
||||
*.o
|
||||
.DS_Store
|
||||
.token
|
||||
.ipynb_checkpoints/
|
||||
|
||||
10519
dev-0/out.tsv
Normal file
10519
dev-0/out.tsv
Normal file
File diff suppressed because it is too large
Load Diff
250
run.ipynb
Normal file
250
run.ipynb
Normal file
@ -0,0 +1,250 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Loading data...\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\r",
|
||||
"0it [00:00, ?it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Training model...\n",
|
||||
"1/2\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"50000it [03:35, 232.50it/s]\n",
|
||||
" 0%| | 8/753550 [00:00<3:31:51, 59.28it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"2/2\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████████████████████████████████████████████████████████████████| 753550/753550 [00:04<00:00, 176601.27it/s]\n",
|
||||
" 0%| | 3/753550 [00:00<8:51:51, 23.61it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Smoothing...\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████████████████████████████████████████████████████████████████| 753550/753550 [00:06<00:00, 117904.94it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Predicting...\n",
|
||||
"Dev set\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"10519it [02:07, 82.51it/s] \n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Test set\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"7414it [01:16, 96.50it/s] \n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import csv\n",
|
||||
"import regex as re\n",
|
||||
"from nltk import bigrams, word_tokenize\n",
|
||||
"from collections import Counter, defaultdict\n",
|
||||
"import string\n",
|
||||
"import unicodedata\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"\n",
|
||||
"pd.set_option('display.max_columns', None)\n",
|
||||
"pd.set_option('display.max_rows', None)\n",
|
||||
"\n",
|
||||
"NROWS = 50000\n",
|
||||
"ALPHA = 0.1\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def etl():\n",
|
||||
" data = pd.read_csv(\n",
|
||||
" \"train/in.tsv.xz\",\n",
|
||||
" sep=\"\\t\",\n",
|
||||
" error_bad_lines=False,\n",
|
||||
" header=None,\n",
|
||||
" quoting=csv.QUOTE_NONE,\n",
|
||||
" nrows=NROWS\n",
|
||||
" )\n",
|
||||
" train_labels = pd.read_csv(\n",
|
||||
" \"train/expected.tsv\",\n",
|
||||
" sep=\"\\t\",\n",
|
||||
" error_bad_lines=False,\n",
|
||||
" header=None,\n",
|
||||
" quoting=csv.QUOTE_NONE,\n",
|
||||
" nrows=NROWS\n",
|
||||
" )\n",
|
||||
" \n",
|
||||
" train_data = data[[6, 7]]\n",
|
||||
" train_data = pd.concat([train_data, train_labels], axis=1)\n",
|
||||
"\n",
|
||||
" train_data[\"final\"] = train_data[6] + train_data[0] + train_data[7]\n",
|
||||
"\n",
|
||||
" model = defaultdict(lambda: defaultdict(lambda: 0))\n",
|
||||
" return train_data, model\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def clean(text):\n",
|
||||
" text = str(text).lower().replace(\"-\\\\n\", \"\").replace(\"\\\\n\", \" \")\n",
|
||||
" return re.sub(r\"\\p{P}\", \"\", text)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def train_model(data):\n",
|
||||
" print(\"1/2\")\n",
|
||||
" for _, row in tqdm(data.iterrows()):\n",
|
||||
" words = word_tokenize(clean(row[\"final\"]))\n",
|
||||
" for word_1, word_2 in bigrams(words, pad_left=True, pad_right=True):\n",
|
||||
" if word_1 and word_2:\n",
|
||||
" vocab.add(word_1)\n",
|
||||
" vocab.add(word_2)\n",
|
||||
" model[word_1][word_2] += 1\n",
|
||||
" print(\"2/2\")\n",
|
||||
" for word_1 in tqdm(model):\n",
|
||||
" total_count = float(sum(model[word_1].values()))\n",
|
||||
" for word_2 in model[word_1]:\n",
|
||||
" model[word_1][word_2] /= total_count\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def predict(word):\n",
|
||||
" predictions = dict(model[word])\n",
|
||||
" most_common = dict(Counter(predictions).most_common(5))\n",
|
||||
"\n",
|
||||
" total_prob = 0.0\n",
|
||||
" str_prediction = \"\"\n",
|
||||
"\n",
|
||||
" for word, prob in most_common.items():\n",
|
||||
" total_prob += prob\n",
|
||||
" str_prediction += f\"{word}:{prob} \"\n",
|
||||
"\n",
|
||||
" if not total_prob:\n",
|
||||
" return \"the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1\"\n",
|
||||
"\n",
|
||||
" if 1 - total_prob >= 0.01:\n",
|
||||
" str_prediction += f\":{1-total_prob}\"\n",
|
||||
" else:\n",
|
||||
" str_prediction += f\":0.01\"\n",
|
||||
"\n",
|
||||
" return str_prediction\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def predict_data(read_path, save_path):\n",
|
||||
" data = pd.read_csv(\n",
|
||||
" read_path, sep=\"\\t\", error_bad_lines=False, header=None, quoting=csv.QUOTE_NONE\n",
|
||||
" )\n",
|
||||
" with open(save_path, \"w\", encoding=\"utf-8\") as file:\n",
|
||||
" for _, row in tqdm(data.iterrows()):\n",
|
||||
" words = word_tokenize(clean(row[6]))\n",
|
||||
" if len(words) < 3:\n",
|
||||
" prediction = \"the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1\"\n",
|
||||
" else:\n",
|
||||
" prediction = predict(words[-1])\n",
|
||||
" file.write(prediction + \"\\n\")\n",
|
||||
" \n",
|
||||
"def plus_alpha_smoothing():\n",
|
||||
" model_len = len(model)\n",
|
||||
" for word_1 in tqdm(model):\n",
|
||||
" word_1_occurrences = sum(model[word_1].values())\n",
|
||||
" for word_2 in model[word_1]:\n",
|
||||
" model[word_1][word_2] += ALPHA\n",
|
||||
" model[word_1][word_2] /= float(word_1_occurrences + ALPHA + len(word_2))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"print(\"Loading data...\")\n",
|
||||
"train_data, model = etl()\n",
|
||||
"vocab = set()\n",
|
||||
"print(\"Training model...\")\n",
|
||||
"train_model(train_data)\n",
|
||||
"print(\"Smoothing...\")\n",
|
||||
"plus_alpha_smoothing()\n",
|
||||
"print(\"Predicting...\")\n",
|
||||
"print(\"Dev set\")\n",
|
||||
"predict_data(\"dev-0/in.tsv.xz\", \"dev-0/out.tsv\")\n",
|
||||
"print(\"Test set\")\n",
|
||||
"predict_data(\"test-A/in.tsv.xz\", \"test-A/out.tsv\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.8.5"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
133
run.py
Normal file
133
run.py
Normal file
@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[2]:
|
||||
|
||||
|
||||
import pandas as pd
|
||||
import csv
|
||||
import regex as re
|
||||
from nltk import bigrams, word_tokenize
|
||||
from collections import Counter, defaultdict
|
||||
import string
|
||||
import unicodedata
|
||||
from tqdm import tqdm
|
||||
|
||||
pd.set_option('display.max_columns', None)
|
||||
pd.set_option('display.max_rows', None)
|
||||
|
||||
NROWS = 50000
|
||||
ALPHA = 0.1
|
||||
|
||||
|
||||
def etl():
|
||||
data = pd.read_csv(
|
||||
"train/in.tsv.xz",
|
||||
sep="\t",
|
||||
error_bad_lines=False,
|
||||
header=None,
|
||||
quoting=csv.QUOTE_NONE,
|
||||
nrows=NROWS
|
||||
)
|
||||
train_labels = pd.read_csv(
|
||||
"train/expected.tsv",
|
||||
sep="\t",
|
||||
error_bad_lines=False,
|
||||
header=None,
|
||||
quoting=csv.QUOTE_NONE,
|
||||
nrows=NROWS
|
||||
)
|
||||
|
||||
train_data = data[[6, 7]]
|
||||
train_data = pd.concat([train_data, train_labels], axis=1)
|
||||
|
||||
train_data["final"] = train_data[6] + train_data[0] + train_data[7]
|
||||
|
||||
model = defaultdict(lambda: defaultdict(lambda: 0))
|
||||
return train_data, model
|
||||
|
||||
|
||||
def clean(text):
|
||||
text = str(text).lower().replace("-\\n", "").replace("\\n", " ")
|
||||
return re.sub(r"\p{P}", "", text)
|
||||
|
||||
|
||||
def train_model(data):
|
||||
print("1/2")
|
||||
for _, row in tqdm(data.iterrows()):
|
||||
words = word_tokenize(clean(row["final"]))
|
||||
for word_1, word_2 in bigrams(words, pad_left=True, pad_right=True):
|
||||
if word_1 and word_2:
|
||||
vocab.add(word_1)
|
||||
vocab.add(word_2)
|
||||
model[word_1][word_2] += 1
|
||||
print("2/2")
|
||||
for word_1 in tqdm(model):
|
||||
total_count = float(sum(model[word_1].values()))
|
||||
for word_2 in model[word_1]:
|
||||
model[word_1][word_2] /= total_count
|
||||
|
||||
|
||||
def predict(word):
|
||||
predictions = dict(model[word])
|
||||
most_common = dict(Counter(predictions).most_common(5))
|
||||
|
||||
total_prob = 0.0
|
||||
str_prediction = ""
|
||||
|
||||
for word, prob in most_common.items():
|
||||
total_prob += prob
|
||||
str_prediction += f"{word}:{prob} "
|
||||
|
||||
if not total_prob:
|
||||
return "the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1"
|
||||
|
||||
if 1 - total_prob >= 0.01:
|
||||
str_prediction += f":{1-total_prob}"
|
||||
else:
|
||||
str_prediction += f":0.01"
|
||||
|
||||
return str_prediction
|
||||
|
||||
|
||||
def predict_data(read_path, save_path):
|
||||
data = pd.read_csv(
|
||||
read_path, sep="\t", error_bad_lines=False, header=None, quoting=csv.QUOTE_NONE
|
||||
)
|
||||
with open(save_path, "w", encoding="utf-8") as file:
|
||||
for _, row in tqdm(data.iterrows()):
|
||||
words = word_tokenize(clean(row[6]))
|
||||
if len(words) < 3:
|
||||
prediction = "the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1"
|
||||
else:
|
||||
prediction = predict(words[-1])
|
||||
file.write(prediction + "\n")
|
||||
|
||||
def plus_alpha_smoothing():
|
||||
model_len = len(model)
|
||||
for word_1 in tqdm(model):
|
||||
word_1_occurrences = sum(model[word_1].values())
|
||||
for word_2 in model[word_1]:
|
||||
model[word_1][word_2] += ALPHA
|
||||
model[word_1][word_2] /= float(word_1_occurrences + ALPHA + len(word_2))
|
||||
|
||||
|
||||
print("Loading data...")
|
||||
train_data, model = etl()
|
||||
vocab = set()
|
||||
print("Training model...")
|
||||
train_model(train_data)
|
||||
print("Smoothing...")
|
||||
plus_alpha_smoothing()
|
||||
print("Predicting...")
|
||||
print("Dev set")
|
||||
predict_data("dev-0/in.tsv.xz", "dev-0/out.tsv")
|
||||
print("Test set")
|
||||
predict_data("test-A/in.tsv.xz", "test-A/out.tsv")
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
|
||||
|
||||
7414
test-A/out.tsv
Normal file
7414
test-A/out.tsv
Normal file
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user