s434760/challenging-america-word-gap-prediction
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

test3

Browse Source
This commit is contained in:
adnovac 2022-04-12 11:54:38 +02:00
parent 9351a7163c
commit 1673116cf1
3 changed files with 17939 additions and 17940 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21038
dev-0/out.tsv
View File

File diff suppressed because it is too large Load Diff

13
run.py
View File

@ -11,18 +11,19 @@ class Model:
def __init__(self): def __init__(self):
self.model = defaultdict(lambda: defaultdict(lambda: 0)) self.model = defaultdict(lambda: defaultdict(lambda: 0))
self.model_bi = defaultdict(lambda: defaultdict(lambda: 0)) self.model_bi = defaultdict(lambda: defaultdict(lambda: 0))
train_in = pd.read_csv("train/in.tsv.xz", sep='\t', header=None, encoding="UTF-8", on_bad_lines="skip", quoting=csv.QUOTE_NONE, nrows=30000)[[6, 7]] train_in = pd.read_csv("train/in.tsv.xz", sep='\t', header=None, encoding="UTF-8", on_bad_lines="skip", quoting=csv.QUOTE_NONE, nrows=300000)[[6, 7]]
train_expected = pd.read_csv("train/expected.tsv", sep='\t', header=None, encoding="UTF-8", on_bad_lines="skip", quoting=csv.QUOTE_NONE, nrows=30000) train_expected = pd.read_csv("train/expected.tsv", sep='\t', header=None, encoding="UTF-8", on_bad_lines="skip", quoting=csv.QUOTE_NONE, nrows=300000)
data = pd.concat([train_in, train_expected], axis=1) data = pd.concat([train_in, train_expected], axis=1)
self.data = data[6] + data[0] + data[7] self.data = data[6] + data[0] + data[7]
self.data = self.data.apply(self.clean) self.data = self.data.apply(self.clean)
def clean(self, text): def clean(self, text):
text = str(text).lower().strip().replace("’", "'").replace('\\n', " ").replace("'t", " not").replace("'s", " is").replace("'ll", " will").replace("'m", " am").replace("'ve", " have").replace(",", "").replace("-", "") text = str(text).lower().strip().replace("’", "'").replace('\\n', " ").replace("'t", " not").replace("'s", " is").replace("'ll", " will").replace("'m", " am").replace("'ve", " have").replace(",", "").replace("-", "")
return text return text
def train(self): def train(self):
alpha = 0.4 alpha = 0.6
vocab = set() vocab = set()
for text in model.data: for text in model.data:
words = word_tokenize(text) words = word_tokenize(text)
@ -35,14 +36,12 @@ class Model:
self.model_bi[w1][w2] +=1 self.model_bi[w1][w2] +=1
for w1, w2 in self.model: for w1, w2 in self.model:
total_count = float(sum(self.model[w1, w2].values())) total_count = float(sum(self.model[w1, w2].values()))
denominator = total_count * len(vocab)
for w in self.model[w1, w2]: for w in self.model[w1, w2]:
self.model[w1, w2][w] = self.model[w1, w2][w] / denominator * alpha self.model[w1, w2][w] = (self.model[w1, w2][w] / total_count) * alpha
for w1 in self.model_bi: for w1 in self.model_bi:
total_count = float(sum(self.model_bi[w1].values())) total_count = float(sum(self.model_bi[w1].values()))
denominator = total_count * len(vocab)
for w in self.model_bi[w1]: for w in self.model_bi[w1]:
self.model_bi[w1][w] = self.model_bi[w1][w] / denominator * (1-alpha) self.model_bi[w1][w] = (self.model_bi[w1][w] / total_count) * (1-alpha)
def predict(self, words): def predict(self, words):
trigrams = Counter(dict(self.model[words])) trigrams = Counter(dict(self.model[words]))

14828
test-A/out.tsv
View File

File diff suppressed because it is too large Load Diff