s434784
This commit is contained in:
Maciej(Linux)
parent
ac33d50cec
commit
96f3dafd86
21038
dev-0/out.tsv
21038
dev-0/out.tsv
File diff suppressed because it is too large
Load Diff
BIN
kenlm_model.arpa
Normal file
BIN
kenlm_model.arpa
Normal file
Binary file not shown.
271
run.ipynb
Normal file
271
run.ipynb
Normal file
@ -0,0 +1,271 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from nltk import trigrams, word_tokenize\n",
|
||||||
|
"import pandas as pd\n",
|
||||||
|
"import csv\n",
|
||||||
|
"import regex as re\n",
|
||||||
|
"from collections import Counter, defaultdict\n",
|
||||||
|
"import kenlm\n",
|
||||||
|
"from english_words import english_words_alpha_set\n",
|
||||||
|
"from math import log10"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"train_set = pd.read_csv(\n",
|
||||||
|
" 'train/in.tsv.xz',\n",
|
||||||
|
" sep='\\t',\n",
|
||||||
|
" header=None,\n",
|
||||||
|
" quoting=csv.QUOTE_NONE,\n",
|
||||||
|
" nrows=35000)\n",
|
||||||
|
"\n",
|
||||||
|
"train_labels = pd.read_csv(\n",
|
||||||
|
" 'train/expected.tsv',\n",
|
||||||
|
" sep='\\t',\n",
|
||||||
|
" header=None,\n",
|
||||||
|
" quoting=csv.QUOTE_NONE,\n",
|
||||||
|
" nrows=35000)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"data = pd.concat([train_set, train_labels], axis=1)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"data = train_set[6] + train_set[0] + train_set[7]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 6,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"def data_preprocessing(text):\n",
|
||||||
|
" return re.sub(r'\\p{P}', '', text.lower().replace('-\\\\n', '').replace('\\\\n', ' ').replace(\"'ll\", \" will\").replace(\"-\", \"\").replace(\"'ve\", \" have\").replace(\"'s\", \" is\"))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 8,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"data = data.apply(data_preprocessing)\n",
|
||||||
|
"prediction = 'the:0.03 be:0.03 to:0.03 of:0.025 and:0.025 a:0.025 in:0.020 that:0.020 have:0.015 I:0.010 it:0.010 for:0.010 not:0.010 on:0.010 with:0.010 he:0.010 as:0.010 you:0.010 do:0.010 at:0.010 :0.77'"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 25,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"with open(\"train_file.txt\", \"w+\") as f:\n",
|
||||||
|
" for text in data:\n",
|
||||||
|
" f.write(text + \"\\n\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 27,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"KENLM_BUILD_PATH='../kenlm/build/bin/lmplz'"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 28,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"=== 1/5 Counting and sorting n-grams ===\n",
|
||||||
|
"Reading /home/maciej/challenging-america-word-gap-prediction/train_file.txt\n",
|
||||||
|
"----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
|
||||||
|
"****************************************************************************************************\n",
|
||||||
|
"Unigram tokens 11040226 types 580506\n",
|
||||||
|
"=== 2/5 Calculating and sorting adjusted counts ===\n",
|
||||||
|
"Chain sizes: 1:6966072 2:4100520192 3:7688475136 4:12301560832\n",
|
||||||
|
"Statistics:\n",
|
||||||
|
"1 580506 D1=0.841976 D2=0.938008 D3+=1.10537\n",
|
||||||
|
"2 3583875 D1=0.83057 D2=1.0296 D3+=1.2275\n",
|
||||||
|
"3 7705610 D1=0.899462 D2=1.16366 D3+=1.32181\n",
|
||||||
|
"4 9865473 D1=0.942374 D2=1.27613 D3+=1.35073\n",
|
||||||
|
"Memory estimate for binary LM:\n",
|
||||||
|
"type MB\n",
|
||||||
|
"probing 442 assuming -p 1.5\n",
|
||||||
|
"probing 508 assuming -r models -p 1.5\n",
|
||||||
|
"trie 216 without quantization\n",
|
||||||
|
"trie 126 assuming -q 8 -b 8 quantization \n",
|
||||||
|
"trie 195 assuming -a 22 array pointer compression\n",
|
||||||
|
"trie 104 assuming -a 22 -q 8 -b 8 array pointer compression and quantization\n",
|
||||||
|
"=== 3/5 Calculating and sorting initial probabilities ===\n",
|
||||||
|
"Chain sizes: 1:6966072 2:57342000 3:154112200 4:236771352\n",
|
||||||
|
"----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
|
||||||
|
"####################################################################################################\n",
|
||||||
|
"=== 4/5 Calculating and writing order-interpolated probabilities ===\n",
|
||||||
|
"Chain sizes: 1:6966072 2:57342000 3:154112200 4:236771352\n",
|
||||||
|
"----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
|
||||||
|
"####################################################################################################\n",
|
||||||
|
"=== 5/5 Writing ARPA model ===\n",
|
||||||
|
"----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
|
||||||
|
"****************************************************************************************************\n",
|
||||||
|
"Name:lmplz\tVmPeak:23697780 kB\tVmRSS:21496 kB\tRSSMax:4963084 kB\tuser:39.0693\tsys:17.6943\tCPU:56.7637\treal:43.821\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"!$KENLM_BUILD_PATH -o 4 < train_file.txt > kenlm_model.arpa"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 29,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"/home/maciej/challenging-america-word-gap-prediction\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Loading the LM will be faster if you build a binary file.\n",
|
||||||
|
"Reading /home/maciej/challenging-america-word-gap-prediction/kenlm_model.arpa\n",
|
||||||
|
"----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
|
||||||
|
"****************************************************************************************************\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"import os\n",
|
||||||
|
"print(os.getcwd())\n",
|
||||||
|
"model = kenlm.Model('kenlm_model.arpa')\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 30,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"def predict(before, after):\n",
|
||||||
|
" result = ''\n",
|
||||||
|
" prob = 0.0\n",
|
||||||
|
" best = []\n",
|
||||||
|
" for word in english_words_alpha_set:\n",
|
||||||
|
" text = ' '.join([before, word, after])\n",
|
||||||
|
" text_score = model.score(text, bos=False, eos=False)\n",
|
||||||
|
" if len(best) < 12:\n",
|
||||||
|
" best.append((word, text_score))\n",
|
||||||
|
" else:\n",
|
||||||
|
" is_better = False\n",
|
||||||
|
" worst_score = None\n",
|
||||||
|
" for score in best:\n",
|
||||||
|
" if not worst_score:\n",
|
||||||
|
" worst_score = score\n",
|
||||||
|
" else:\n",
|
||||||
|
" if worst_score[1] > score[1]:\n",
|
||||||
|
" worst_score = score\n",
|
||||||
|
" if worst_score[1] < text_score:\n",
|
||||||
|
" best.remove(worst_score)\n",
|
||||||
|
" best.append((word, text_score))\n",
|
||||||
|
" probs = sorted(best, key=lambda tup: tup[1], reverse=True)\n",
|
||||||
|
" pred_str = ''\n",
|
||||||
|
" for word, prob in probs:\n",
|
||||||
|
" pred_str += f'{word}:{prob} '\n",
|
||||||
|
" pred_str += f':{log10(0.99)}'\n",
|
||||||
|
" return pred_str\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 31,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"def make_prediction(path, result_path):\n",
|
||||||
|
" data = pd.read_csv(path, sep='\\t', header=None, quoting=csv.QUOTE_NONE)\n",
|
||||||
|
" with open(result_path, 'w', encoding='utf-8') as file_out:\n",
|
||||||
|
" for _, row in data.iterrows():\n",
|
||||||
|
" before, after = word_tokenize(data_preprocessing(str(row[6]))), word_tokenize(data_preprocessing(str(row[7])))\n",
|
||||||
|
" if len(before) < 2 or len(after) < 2:\n",
|
||||||
|
" pred = prediction\n",
|
||||||
|
" else:\n",
|
||||||
|
" pred = predict(before[-1], after[0])\n",
|
||||||
|
" file_out.write(pred + '\\n')"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 32,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"make_prediction(\"dev-0/in.tsv.xz\", \"dev-0/out.tsv\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 33,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"make_prediction(\"test-A/in.tsv.xz\", \"test-A/out.tsv\")"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"interpreter": {
|
||||||
|
"hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
|
||||||
|
},
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3 (ipykernel)",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.8.10"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 2
|
||||||
|
}
|
||||||
194
run.py
Normal file → Executable file
194
run.py
Normal file → Executable file
@ -1,80 +1,146 @@
|
|||||||
from re import T
|
#!/usr/bin/env python
|
||||||
|
# coding: utf-8
|
||||||
|
|
||||||
|
# In[2]:
|
||||||
|
|
||||||
|
|
||||||
|
from nltk import trigrams, word_tokenize
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import csv
|
import csv
|
||||||
from collections import Counter, defaultdict
|
|
||||||
from nltk.tokenize import RegexpTokenizer
|
|
||||||
from nltk import trigrams
|
|
||||||
import regex as re
|
import regex as re
|
||||||
import lzma
|
from collections import Counter, defaultdict
|
||||||
|
import kenlm
|
||||||
|
from english_words import english_words_alpha_set
|
||||||
|
from math import log10
|
||||||
|
|
||||||
|
|
||||||
class GapEssa:
|
# In[3]:
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
self.alpha = 0.0001
|
|
||||||
self.vocab = set()
|
|
||||||
self.model = defaultdict(lambda: defaultdict(lambda: 0))
|
|
||||||
self.tokenizer = RegexpTokenizer(r"\w+")
|
|
||||||
|
|
||||||
def read_file(self, f, mode=0):
|
train_set = pd.read_csv(
|
||||||
for line in f:
|
'train/in.tsv.xz',
|
||||||
text = line.split("\t")
|
sep='\t',
|
||||||
if(mode==0):
|
header=None,
|
||||||
yield re.sub(r"[^\w\d'\s]+", '', re.sub(' +', ' ', ' '.join([text[6], text[7]]).replace("\\n"," ").replace("\n","").lower()))
|
quoting=csv.QUOTE_NONE,
|
||||||
|
nrows=35000)
|
||||||
|
|
||||||
|
train_labels = pd.read_csv(
|
||||||
|
'train/expected.tsv',
|
||||||
|
sep='\t',
|
||||||
|
header=None,
|
||||||
|
quoting=csv.QUOTE_NONE,
|
||||||
|
nrows=35000)
|
||||||
|
|
||||||
|
|
||||||
|
# In[4]:
|
||||||
|
|
||||||
|
|
||||||
|
data = pd.concat([train_set, train_labels], axis=1)
|
||||||
|
|
||||||
|
|
||||||
|
# In[5]:
|
||||||
|
|
||||||
|
|
||||||
|
data = train_set[6] + train_set[0] + train_set[7]
|
||||||
|
|
||||||
|
|
||||||
|
# In[6]:
|
||||||
|
|
||||||
|
|
||||||
|
def data_preprocessing(text):
|
||||||
|
return re.sub(r'\p{P}', '', text.lower().replace('-\\n', '').replace('\\n', ' ').replace("'ll", " will").replace("-", "").replace("'ve", " have").replace("'s", " is"))
|
||||||
|
|
||||||
|
|
||||||
|
# In[8]:
|
||||||
|
|
||||||
|
|
||||||
|
data = data.apply(data_preprocessing)
|
||||||
|
prediction = 'the:0.03 be:0.03 to:0.03 of:0.025 and:0.025 a:0.025 in:0.020 that:0.020 have:0.015 I:0.010 it:0.010 for:0.010 not:0.010 on:0.010 with:0.010 he:0.010 as:0.010 you:0.010 do:0.010 at:0.010 :0.77'
|
||||||
|
|
||||||
|
|
||||||
|
# In[25]:
|
||||||
|
|
||||||
|
|
||||||
|
with open("train_file.txt", "w+") as f:
|
||||||
|
for text in data:
|
||||||
|
f.write(text + "\n")
|
||||||
|
|
||||||
|
|
||||||
|
# In[27]:
|
||||||
|
|
||||||
|
|
||||||
|
KENLM_BUILD_PATH='../kenlm/build/bin/lmplz'
|
||||||
|
|
||||||
|
|
||||||
|
# In[28]:
|
||||||
|
|
||||||
|
|
||||||
|
get_ipython().system('$KENLM_BUILD_PATH -o 4 < train_file.txt > kenlm_model.arpa')
|
||||||
|
|
||||||
|
|
||||||
|
# In[29]:
|
||||||
|
|
||||||
|
|
||||||
|
import os
|
||||||
|
print(os.getcwd())
|
||||||
|
model = kenlm.Model('kenlm_model.arpa')
|
||||||
|
|
||||||
|
|
||||||
|
# In[30]:
|
||||||
|
|
||||||
|
|
||||||
|
def predict(before, after):
|
||||||
|
result = ''
|
||||||
|
prob = 0.0
|
||||||
|
best = []
|
||||||
|
for word in english_words_alpha_set:
|
||||||
|
text = ' '.join([before, word, after])
|
||||||
|
text_score = model.score(text, bos=False, eos=False)
|
||||||
|
if len(best) < 12:
|
||||||
|
best.append((word, text_score))
|
||||||
else:
|
else:
|
||||||
yield re.sub(r"[^\w\d'\s]+", '', re.sub(' +', ' ', text[7].replace("\\n"," ").replace("\n","").lower()))
|
is_better = False
|
||||||
|
worst_score = None
|
||||||
def train(self, f):
|
for score in best:
|
||||||
with lzma.open(f, mode='rt') as file:
|
if not worst_score:
|
||||||
for index, text in enumerate(self.read_file(file)):
|
worst_score = score
|
||||||
tokens = self.tokenizer.tokenize(text)
|
|
||||||
for w1, w2, w3 in trigrams(tokens, pad_right=True, pad_left=True):
|
|
||||||
if w1 and w2 and w3:
|
|
||||||
self.model[(w2, w3)][w1] += 1
|
|
||||||
self.vocab.add(w1)
|
|
||||||
self.vocab.add(w2)
|
|
||||||
self.vocab.add(w3)
|
|
||||||
if index == 40000:
|
|
||||||
break
|
|
||||||
|
|
||||||
for pair in self.model:
|
|
||||||
num_n_grams = float(sum(self.model[pair].values()))
|
|
||||||
for word in self.model[pair]:
|
|
||||||
self.model[pair][word] = (self.model[pair][word] + self.alpha) / (num_n_grams + self.alpha*len(self.vocab))
|
|
||||||
|
|
||||||
def out(self, input_f, output_f):
|
|
||||||
with open(output_f, 'w') as out_f:
|
|
||||||
with lzma.open(input_f, mode='rt') as in_f:
|
|
||||||
for _, text in enumerate(self.read_file(in_f, mode=1)):
|
|
||||||
t = self.tokenizer.tokenize(text)
|
|
||||||
if len(t) < 4:
|
|
||||||
# p = 'the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1'
|
|
||||||
p = 'the:0.03 be:0.03 to:0.03 of:0.025 and:0.025 a:0.025 in:0.020 that:0.020 have:0.015 I:0.010 it:0.010 for:0.010 not:0.010 on:0.010 with:0.010 he:0.010 as:0.010 you:0.010 do:0.010 at:0.010 :0.77'
|
|
||||||
else:
|
else:
|
||||||
p = self.pred(t[0], t[1])
|
if worst_score[1] > score[1]:
|
||||||
out_f.write(p + '\n')
|
worst_score = score
|
||||||
|
if worst_score[1] < text_score:
|
||||||
|
best.remove(worst_score)
|
||||||
|
best.append((word, text_score))
|
||||||
|
probs = sorted(best, key=lambda tup: tup[1], reverse=True)
|
||||||
|
pred_str = ''
|
||||||
|
for word, prob in probs:
|
||||||
|
pred_str += f'{word}:{prob} '
|
||||||
|
pred_str += f':{log10(0.99)}'
|
||||||
|
return pred_str
|
||||||
|
|
||||||
def pred(self, w1, w2):
|
|
||||||
total = 0.0
|
|
||||||
line = ''
|
|
||||||
|
|
||||||
p = dict(self.model[w1, w2])
|
# In[31]:
|
||||||
m = dict(Counter(p).most_common(6))
|
|
||||||
|
|
||||||
for word, prob in m.items():
|
|
||||||
total += prob
|
|
||||||
line += f'{word}:{prob} '
|
|
||||||
|
|
||||||
if total == 0.0:
|
def make_prediction(path, result_path):
|
||||||
return 'the:0.03 be:0.03 to:0.03 of:0.025 and:0.025 a:0.025 in:0.020 that:0.020 have:0.015 I:0.010 it:0.010 for:0.010 not:0.010 on:0.010 with:0.010 he:0.010 as:0.010 you:0.010 do:0.010 at:0.010 :0.77'
|
data = pd.read_csv(path, sep='\t', header=None, quoting=csv.QUOTE_NONE)
|
||||||
if 1 - total >= 0.01:
|
with open(result_path, 'w', encoding='utf-8') as file_out:
|
||||||
line += f":{1-total}"
|
for _, row in data.iterrows():
|
||||||
|
before, after = word_tokenize(data_preprocessing(str(row[6]))), word_tokenize(data_preprocessing(str(row[7])))
|
||||||
|
if len(before) < 2 or len(after) < 2:
|
||||||
|
pred = prediction
|
||||||
else:
|
else:
|
||||||
line += f":0.01"
|
pred = predict(before[-1], after[0])
|
||||||
|
file_out.write(pred + '\n')
|
||||||
|
|
||||||
return line
|
|
||||||
|
|
||||||
wp = GapEssa()
|
# In[32]:
|
||||||
wp.train('train/in.tsv.xz')
|
|
||||||
wp.out('dev-0/in.tsv.xz', 'dev-0/out.tsv')
|
|
||||||
wp.out('test-A/in.tsv.xz', 'test-A/out.tsv')
|
make_prediction("dev-0/in.tsv.xz", "dev-0/out.tsv")
|
||||||
|
|
||||||
|
|
||||||
|
# In[33]:
|
||||||
|
|
||||||
|
|
||||||
|
make_prediction("test-A/in.tsv.xz", "test-A/out.tsv")
|
||||||
|
|
||||||
|
|||||||
80
run2.py
Normal file
80
run2.py
Normal file
@ -0,0 +1,80 @@
|
|||||||
|
from re import T
|
||||||
|
import pandas as pd
|
||||||
|
import csv
|
||||||
|
from collections import Counter, defaultdict
|
||||||
|
from nltk.tokenize import RegexpTokenizer
|
||||||
|
from nltk import trigrams
|
||||||
|
import regex as re
|
||||||
|
import lzma
|
||||||
|
|
||||||
|
|
||||||
|
class GapEssa:
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.alpha = 0.0001
|
||||||
|
self.vocab = set()
|
||||||
|
self.model = defaultdict(lambda: defaultdict(lambda: 0))
|
||||||
|
self.tokenizer = RegexpTokenizer(r"\w+")
|
||||||
|
|
||||||
|
def read_file(self, f, mode=0):
|
||||||
|
for line in f:
|
||||||
|
text = line.split("\t")
|
||||||
|
if(mode==0):
|
||||||
|
yield re.sub(r"[^\w\d'\s]+", '', re.sub(' +', ' ', ' '.join([text[6], text[7]]).replace("\\n"," ").replace("\n","").lower()))
|
||||||
|
else:
|
||||||
|
yield re.sub(r"[^\w\d'\s]+", '', re.sub(' +', ' ', text[7].replace("\\n"," ").replace("\n","").lower()))
|
||||||
|
|
||||||
|
def train(self, f):
|
||||||
|
with lzma.open(f, mode='rt') as file:
|
||||||
|
for index, text in enumerate(self.read_file(file)):
|
||||||
|
tokens = self.tokenizer.tokenize(text)
|
||||||
|
for w1, w2, w3 in trigrams(tokens, pad_right=True, pad_left=True):
|
||||||
|
if w1 and w2 and w3:
|
||||||
|
self.model[(w2, w3)][w1] += 1
|
||||||
|
self.vocab.add(w1)
|
||||||
|
self.vocab.add(w2)
|
||||||
|
self.vocab.add(w3)
|
||||||
|
if index == 40000:
|
||||||
|
break
|
||||||
|
|
||||||
|
for pair in self.model:
|
||||||
|
num_n_grams = float(sum(self.model[pair].values()))
|
||||||
|
for word in self.model[pair]:
|
||||||
|
self.model[pair][word] = (self.model[pair][word] + self.alpha) / (num_n_grams + self.alpha*len(self.vocab))
|
||||||
|
|
||||||
|
def out(self, input_f, output_f):
|
||||||
|
with open(output_f, 'w') as out_f:
|
||||||
|
with lzma.open(input_f, mode='rt') as in_f:
|
||||||
|
for _, text in enumerate(self.read_file(in_f, mode=1)):
|
||||||
|
t = self.tokenizer.tokenize(text)
|
||||||
|
if len(t) < 4:
|
||||||
|
# p = 'the:0.2 be:0.2 to:0.2 of:0.1 and:0.1 a:0.1 :0.1'
|
||||||
|
p = 'the:0.03 be:0.03 to:0.03 of:0.025 and:0.025 a:0.025 in:0.020 that:0.020 have:0.015 I:0.010 it:0.010 for:0.010 not:0.010 on:0.010 with:0.010 he:0.010 as:0.010 you:0.010 do:0.010 at:0.010 :0.77'
|
||||||
|
else:
|
||||||
|
p = self.pred(t[0], t[1])
|
||||||
|
out_f.write(p + '\n')
|
||||||
|
|
||||||
|
def pred(self, w1, w2):
|
||||||
|
total = 0.0
|
||||||
|
line = ''
|
||||||
|
|
||||||
|
p = dict(self.model[w1, w2])
|
||||||
|
m = dict(Counter(p).most_common(6))
|
||||||
|
|
||||||
|
for word, prob in m.items():
|
||||||
|
total += prob
|
||||||
|
line += f'{word}:{prob} '
|
||||||
|
|
||||||
|
if total == 0.0:
|
||||||
|
return 'the:0.03 be:0.03 to:0.03 of:0.025 and:0.025 a:0.025 in:0.020 that:0.020 have:0.015 I:0.010 it:0.010 for:0.010 not:0.010 on:0.010 with:0.010 he:0.010 as:0.010 you:0.010 do:0.010 at:0.010 :0.77'
|
||||||
|
if 1 - total >= 0.01:
|
||||||
|
line += f":{1-total}"
|
||||||
|
else:
|
||||||
|
line += f":0.01"
|
||||||
|
|
||||||
|
return line
|
||||||
|
|
||||||
|
wp = GapEssa()
|
||||||
|
wp.train('train/in.tsv.xz')
|
||||||
|
wp.out('dev-0/in.tsv.xz', 'dev-0/out.tsv')
|
||||||
|
wp.out('test-A/in.tsv.xz', 'test-A/out.tsv')
|
||||||
14828
test-A/out.tsv
14828
test-A/out.tsv
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user