8.2 KiB
8.2 KiB
import pandas as pd
import csv
import regex as re
from nltk import bigrams, word_tokenize
from collections import Counter, defaultdict
import string
import unicodedata
data = pd.read_csv(
"train/in.tsv.xz",
sep="\t",
error_bad_lines=False,
header=None,
quoting=csv.QUOTE_NONE,
)
train_labels = pd.read_csv(
"train/expected.tsv",
sep="\t",
error_bad_lines=False,
header=None,
quoting=csv.QUOTE_NONE,
)
train_data = data[[6, 7]]
train_data = pd.concat([train_data, train_labels], axis=1)
train_data["final"] = train_data[6] + train_data[0] + train_data[7]
model = defaultdict(lambda: defaultdict(lambda: 0))
def clean(text):
text = str(text).lower().replace("-\\\\n", "").replace("\\\\n", " ")
return re.sub(r"\p{P}", "", text)
def train_model(data):
for _, row in data.iterrows():
words = word_tokenize(clean(row["final"]))
for w1, w2 in bigrams(words, pad_left=True, pad_right=True):
if w1 and w2:
model[w1][w2] += 1
for w1 in model:
total_count = float(sum(model[w1].values()))
for w2 in model[w1]:
model[w1][w2] /= total_count
def predict(word):
predictions = dict(model[word])
most_common = dict(Counter(predictions).most_common(5))
total_prob = 0.0
str_prediction = ""
for word, prob in most_common.items():
total_prob += prob
str_prediction += f"{word}:{prob} "
if not total_prob:
return "the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1"
if 1 - total_prob >= 0.01:
str_prediction += f":{1-total_prob}"
else:
str_prediction += f":0.01"
return str_prediction
def predict_data(read_path, save_path):
data = pd.read_csv(
read_path, sep="\t", error_bad_lines=False, header=None, quoting=csv.QUOTE_NONE
)
with open(save_path, "w", encoding="UTF-8") as file:
for _, row in data.iterrows():
words = word_tokenize(clean(row[6]))
if len(words) < 3:
prediction = "the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1"
else:
prediction = predict(words[-1])
file.write(prediction + "\n")
train_model(train_data)
predict_data("dev-0/in.tsv.xz", "dev-0/out.tsv")
predict_data("test-A/in.tsv.xz", "test-A/out.tsv")
C:\Users\grzyb\AppData\Local\Temp/ipykernel_34768/887107210.py:86: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. predict_data("dev-0/in.tsv.xz", "dev-0/out.tsv")
[1;31m---------------------------------------------------------------------------[0m [1;31mUnicodeEncodeError[0m Traceback (most recent call last) [1;32m~\AppData\Local\Temp/ipykernel_34768/887107210.py[0m in [0;36m<module>[1;34m[0m [0;32m 84[0m [1;33m[0m[0m [0;32m 85[0m [0mtrain_model[0m[1;33m([0m[0mtrain_data[0m[1;33m)[0m[1;33m[0m[1;33m[0m[0m [1;32m---> 86[1;33m [0mpredict_data[0m[1;33m([0m[1;34m"dev-0/in.tsv.xz"[0m[1;33m,[0m [1;34m"dev-0/out.tsv"[0m[1;33m)[0m[1;33m[0m[1;33m[0m[0m [0m[0;32m 87[0m [0mpredict_data[0m[1;33m([0m[1;34m"test-A/in.tsv.xz"[0m[1;33m,[0m [1;34m"test-A/out.tsv"[0m[1;33m)[0m[1;33m[0m[1;33m[0m[0m [1;32m~\AppData\Local\Temp/ipykernel_34768/887107210.py[0m in [0;36mpredict_data[1;34m(read_path, save_path)[0m [0;32m 80[0m [1;32melse[0m[1;33m:[0m[1;33m[0m[1;33m[0m[0m [0;32m 81[0m [0mprediction[0m [1;33m=[0m [0mpredict[0m[1;33m([0m[0mwords[0m[1;33m[[0m[1;33m-[0m[1;36m1[0m[1;33m][0m[1;33m)[0m[1;33m[0m[1;33m[0m[0m [1;32m---> 82[1;33m [0mfile[0m[1;33m.[0m[0mwrite[0m[1;33m([0m[0mprediction[0m [1;33m+[0m [1;34m"\n"[0m[1;33m)[0m[1;33m[0m[1;33m[0m[0m [0m[0;32m 83[0m [1;33m[0m[0m [0;32m 84[0m [1;33m[0m[0m [1;32mc:\Users\grzyb\anaconda3\lib\encodings\cp1250.py[0m in [0;36mencode[1;34m(self, input, final)[0m [0;32m 17[0m [1;32mclass[0m [0mIncrementalEncoder[0m[1;33m([0m[0mcodecs[0m[1;33m.[0m[0mIncrementalEncoder[0m[1;33m)[0m[1;33m:[0m[1;33m[0m[1;33m[0m[0m [0;32m 18[0m [1;32mdef[0m [0mencode[0m[1;33m([0m[0mself[0m[1;33m,[0m [0minput[0m[1;33m,[0m [0mfinal[0m[1;33m=[0m[1;32mFalse[0m[1;33m)[0m[1;33m:[0m[1;33m[0m[1;33m[0m[0m [1;32m---> 19[1;33m [1;32mreturn[0m [0mcodecs[0m[1;33m.[0m[0mcharmap_encode[0m[1;33m([0m[0minput[0m[1;33m,[0m[0mself[0m[1;33m.[0m[0merrors[0m[1;33m,[0m[0mencoding_table[0m[1;33m)[0m[1;33m[[0m[1;36m0[0m[1;33m][0m[1;33m[0m[1;33m[0m[0m [0m[0;32m 20[0m [1;33m[0m[0m [0;32m 21[0m [1;32mclass[0m [0mIncrementalDecoder[0m[1;33m([0m[0mcodecs[0m[1;33m.[0m[0mIncrementalDecoder[0m[1;33m)[0m[1;33m:[0m[1;33m[0m[1;33m[0m[0m [1;31mUnicodeEncodeError[0m: 'charmap' codec can't encode character '\u25a0' in position 0: character maps to <undefined>