s434760/challenging-america-word-gap-prediction
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

434760

Browse Source
This commit is contained in:
adnovac 2022-04-12 10:30:37 +02:00
parent ed3af7d037
commit b6dd9396b3
3 changed files with 17937 additions and 3734 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14249
dev-0/out.tsv
View File

File diff suppressed because it is too large Load Diff

8
run.py
View File

@ -11,8 +11,8 @@ class Model:
def __init__(self): def __init__(self):
self.model = defaultdict(lambda: defaultdict(lambda: 0)) self.model = defaultdict(lambda: defaultdict(lambda: 0))
self.model_bi = defaultdict(lambda: defaultdict(lambda: 0)) self.model_bi = defaultdict(lambda: defaultdict(lambda: 0))
train_in = pd.read_csv("train/in.tsv.xz", sep='\t', header=None, encoding="UTF-8", on_bad_lines="skip", quoting=csv.QUOTE_NONE)[[6, 7]] train_in = pd.read_csv("train/in.tsv.xz", sep='\t', header=None, encoding="UTF-8", on_bad_lines="skip", quoting=csv.QUOTE_NONE, nrows=20500)[[6, 7]]
train_expected = pd.read_csv("train/expected.tsv", sep='\t', header=None, encoding="UTF-8", on_bad_lines="skip", quoting=csv.QUOTE_NONE) train_expected = pd.read_csv("train/expected.tsv", sep='\t', header=None, encoding="UTF-8", on_bad_lines="skip", quoting=csv.QUOTE_NONE, nrows=20500)
data = pd.concat([train_in, train_expected], axis=1) data = pd.concat([train_in, train_expected], axis=1)
self.data = data[6] + data[0] + data[7] self.data = data[6] + data[0] + data[7]
self.data = self.data.apply(self.clean) self.data = self.data.apply(self.clean)
@ -26,12 +26,12 @@ class Model:
vocab = set() vocab = set()
for text in model.data: for text in model.data:
words = word_tokenize(text) words = word_tokenize(text)
for w1, w2, w3 in trigrams(words, pad_left=True, pad_right=True): for w1, w2, w3 in trigrams(words):
self.model[w1, w2][w3] += 1 self.model[w1, w2][w3] += 1
vocab.add(w1) vocab.add(w1)
vocab.add(w2) vocab.add(w2)
vocab.add(w3) vocab.add(w3)
for w1, w2 in bigrams(words, pad_left=True, pad_right=True): for w1, w2 in bigrams(words):
self.model_bi[w1][w2] +=1 self.model_bi[w1][w2] +=1
for w1, w2 in self.model: for w1, w2 in self.model:
total_count = float(sum(self.model[w1, w2].values())) total_count = float(sum(self.model[w1, w2].values()))

7414
test-A/out.tsv Normal file
View File

File diff suppressed because it is too large Load Diff