Compare commits
No commits in common. "kenlm" and "master" have entirely different histories.
21036
dev-0/out.tsv
21036
dev-0/out.tsv
File diff suppressed because it is too large
Load Diff
278
run.ipynb
278
run.ipynb
@ -1,278 +0,0 @@
|
|||||||
{
|
|
||||||
"cells": [
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"id": "1da94494-ccbd-4f3c-9ca0-2241cfd9d361",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": []
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 2,
|
|
||||||
"id": "2f51e23a-93a0-4bf6-9c87-19da220e11bd",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"Collecting english_words\n",
|
|
||||||
" Downloading english-words-1.1.0.tar.gz (1.1 MB)\n",
|
|
||||||
"\u001b[K |████████████████████████████████| 1.1 MB 1.5 MB/s eta 0:00:01\n",
|
|
||||||
"\u001b[?25hBuilding wheels for collected packages: english-words\n",
|
|
||||||
" Building wheel for english-words (setup.py) ... \u001b[?25ldone\n",
|
|
||||||
"\u001b[?25h Created wheel for english-words: filename=english_words-1.1.0-py3-none-any.whl size=1106680 sha256=ddaf5f4288a2022c2ce712aad0ba022e7b25d4d7e73c5637d6154abc5a899662\n",
|
|
||||||
" Stored in directory: /home/asadursk/.cache/pip/wheels/0e/24/52/b4989db82a438482aa65b3c6c0537e988fd40546b792747b1a\n",
|
|
||||||
"Successfully built english-words\n",
|
|
||||||
"Installing collected packages: english-words\n",
|
|
||||||
"Successfully installed english-words-1.1.0\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"!pip install english_words"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 5,
|
|
||||||
"id": "d99975a7-aebe-4e26-b330-4be7f32204c5",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"Collecting pypi-kenlm\n",
|
|
||||||
" Downloading pypi-kenlm-0.1.20210121.tar.gz (253 kB)\n",
|
|
||||||
"\u001b[K |████████████████████████████████| 253 kB 1.6 MB/s eta 0:00:01\n",
|
|
||||||
"\u001b[?25hBuilding wheels for collected packages: pypi-kenlm\n",
|
|
||||||
" Building wheel for pypi-kenlm (setup.py) ... \u001b[?25ldone\n",
|
|
||||||
"\u001b[?25h Created wheel for pypi-kenlm: filename=pypi_kenlm-0.1.20210121-cp39-cp39-linux_x86_64.whl size=311921 sha256=2fcde1a0b569c5d5aef6c61014559b38efc45ed4ae90357c1219816d9a5bbe9b\n",
|
|
||||||
" Stored in directory: /home/asadursk/.cache/pip/wheels/14/f0/7a/97db71356d1dc1b0c14bf48e0d01e5561d5d67ba869e4406d0\n",
|
|
||||||
"Successfully built pypi-kenlm\n",
|
|
||||||
"Installing collected packages: pypi-kenlm\n",
|
|
||||||
"Successfully installed pypi-kenlm-0.1.20210121\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"!python -m pip install pypi-kenlm"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 27,
|
|
||||||
"id": "84560801-85f1-409b-a9c8-c209928276cc",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from collections import defaultdict, Counter\n",
|
|
||||||
"from nltk import trigrams, word_tokenize\n",
|
|
||||||
"from english_words import english_words_alpha_set\n",
|
|
||||||
"import csv\n",
|
|
||||||
"import regex as re\n",
|
|
||||||
"import pandas as pd\n",
|
|
||||||
"import kenlm\n",
|
|
||||||
"from math import log10"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 29,
|
|
||||||
"id": "7a39272c-7929-42d8-98ba-8304570439af",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"def preprocess(row):\n",
|
|
||||||
" return re.sub(r'\\p{P}', '', row.lower().replace('-\\\\\\\\n', '').replace('\\\\\\\\n', ' '))"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 30,
|
|
||||||
"id": "2a330ad2-9b88-4fdd-bc04-635b5cb42c0d",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"def kenlm_model():\n",
|
|
||||||
" with open(\"train_file.txt\", \"w+\") as f:\n",
|
|
||||||
" for text in X_train:\n",
|
|
||||||
" f.write(str(text) + \"\\n\")\n",
|
|
||||||
"\n",
|
|
||||||
" #%%\n",
|
|
||||||
" KENLM_BUILD_PATH='/home/asadursk/kenlm/build'\n",
|
|
||||||
" !$KENLM_BUILD_PATH/bin/lmplz -o 4 < train_file.txt > model.arpa\n",
|
|
||||||
" !$KENLM_BUILD_PATH/bin/build_binary model.arpa model.binary\n",
|
|
||||||
" !rm train_file.txt\n",
|
|
||||||
" \n",
|
|
||||||
" model = kenlm.Model(\"model.binary\")\n",
|
|
||||||
" return model"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 31,
|
|
||||||
"id": "e848ba36-f4eb-4bd6-9b19-fffea177bfa1",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"def predict_word(w1, w3):\n",
|
|
||||||
" best_scores = []\n",
|
|
||||||
" for word in english_words_alpha_set:\n",
|
|
||||||
" text = ' '.join([w1, word, w3])\n",
|
|
||||||
" text_score = model.score(text, bos=False, eos=False)\n",
|
|
||||||
" if len(best_scores) < 12:\n",
|
|
||||||
" best_scores.append((word, text_score))\n",
|
|
||||||
" else:\n",
|
|
||||||
" is_better = False\n",
|
|
||||||
" worst_score = None\n",
|
|
||||||
" for score in best_scores:\n",
|
|
||||||
" if not worst_score:\n",
|
|
||||||
" worst_score = score\n",
|
|
||||||
" else:\n",
|
|
||||||
" if worst_score[1] > score[1]:\n",
|
|
||||||
" worst_score = score\n",
|
|
||||||
" if worst_score[1] < text_score:\n",
|
|
||||||
" best_scores.remove(worst_score)\n",
|
|
||||||
" best_scores.append((word, text_score))\n",
|
|
||||||
" probs = sorted(best_scores, key=lambda tup: tup[1], reverse=True)\n",
|
|
||||||
" pred_str = ''\n",
|
|
||||||
" for word, prob in probs:\n",
|
|
||||||
" pred_str += f'{word}:{prob} '\n",
|
|
||||||
" pred_str += f':{log10(0.99)}'\n",
|
|
||||||
" return pred_str"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 32,
|
|
||||||
"id": "6babeba5-af91-4e9c-a235-781525594f45",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"def word_gap_prediction(file, model):\n",
|
|
||||||
" X_test = pd.read_csv(f'{file}/in.tsv.xz', sep='\\t', header=None, quoting=csv.QUOTE_NONE, on_bad_lines=\"skip\")\n",
|
|
||||||
" with open(f'{file}/out.tsv', 'w', encoding='utf-8') as output_file:\n",
|
|
||||||
" for _, row in X_test.iterrows():\n",
|
|
||||||
" before, after = word_tokenize(preprocess(str(row[6]))), word_tokenize(preprocess(str(row[7])))\n",
|
|
||||||
" if len(before) < 2 or len(after) < 2:\n",
|
|
||||||
" output = 'to:0.015 be:0.015 the:0.015 not:0.01 and:0.02 a:0.02 :0.9'\n",
|
|
||||||
" else:\n",
|
|
||||||
" output = predict_word(before[-1], after[0])\n",
|
|
||||||
" output_file.write(output + '\\n')"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 33,
|
|
||||||
"id": "8df4a04c-ae0d-46d7-8b76-1bcf6b424d7a",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"=== 1/5 Counting and sorting n-grams ===\n",
|
|
||||||
"Reading /home/asadursk/challenging-america-word-gap-prediction-kenlm/train_file.txt\n",
|
|
||||||
"----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
|
|
||||||
"****************************************************************************************************\n",
|
|
||||||
"Unigram tokens 2787545 types 548500\n",
|
|
||||||
"=== 2/5 Calculating and sorting adjusted counts ===\n",
|
|
||||||
"Chain sizes: 1:6582000 2:865198656 3:1622247552 4:2595596032\n",
|
|
||||||
"Statistics:\n",
|
|
||||||
"1 548500 D1=0.85065 D2=1.01013 D3+=1.14959\n",
|
|
||||||
"2 1743634 D1=0.900957 D2=1.09827 D3+=1.20014\n",
|
|
||||||
"3 2511917 D1=0.957313 D2=1.22283 D3+=1.33724\n",
|
|
||||||
"4 2719775 D1=0.982576 D2=1.4205 D3+=1.65074\n",
|
|
||||||
"Memory estimate for binary LM:\n",
|
|
||||||
"type MB\n",
|
|
||||||
"probing 157 assuming -p 1.5\n",
|
|
||||||
"probing 184 assuming -r models -p 1.5\n",
|
|
||||||
"trie 82 without quantization\n",
|
|
||||||
"trie 51 assuming -q 8 -b 8 quantization \n",
|
|
||||||
"trie 74 assuming -a 22 array pointer compression\n",
|
|
||||||
"trie 43 assuming -a 22 -q 8 -b 8 array pointer compression and quantization\n",
|
|
||||||
"=== 3/5 Calculating and sorting initial probabilities ===\n",
|
|
||||||
"Chain sizes: 1:6582000 2:27898144 3:50238340 4:65274600\n",
|
|
||||||
"----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
|
|
||||||
"####################################################################################################\n",
|
|
||||||
"=== 4/5 Calculating and writing order-interpolated probabilities ===\n",
|
|
||||||
"Chain sizes: 1:6582000 2:27898144 3:50238340 4:65274600\n",
|
|
||||||
"----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
|
|
||||||
"####################################################################################################\n",
|
|
||||||
"=== 5/5 Writing ARPA model ===\n",
|
|
||||||
"----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
|
|
||||||
"****************************************************************************************************\n",
|
|
||||||
"Name:lmplz\tVmPeak:5126188 kB\tVmRSS:54384 kB\tRSSMax:1084112 kB\tuser:9.18382\tsys:2.72419\tCPU:11.9081\treal:9.09119\n",
|
|
||||||
"Reading model.arpa\n",
|
|
||||||
"----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
|
|
||||||
"****************************************************************************************************\n",
|
|
||||||
"SUCCESS\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"X_train = pd.read_csv('train/in.tsv.xz', sep='\\t', header=None, quoting=csv.QUOTE_NONE, nrows=10000, on_bad_lines=\"skip\")\n",
|
|
||||||
"Y_train = pd.read_csv('train/expected.tsv', sep='\\t', header=None, quoting=csv.QUOTE_NONE, nrows=10000, on_bad_lines=\"skip\")\n",
|
|
||||||
"\n",
|
|
||||||
"X_train = X_train[[6, 7]]\n",
|
|
||||||
"X_train = pd.concat([X_train, Y_train], axis=1)\n",
|
|
||||||
"X_train = X_train[6] + X_train[0] + X_train[7]\n",
|
|
||||||
"\n",
|
|
||||||
"model = kenlm_model()"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 34,
|
|
||||||
"id": "5f9b4351-54b6-42de-8653-597b17c42766",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"word_gap_prediction(\"dev-0/\", model)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 35,
|
|
||||||
"id": "71076162-473b-40f2-93ab-0536a2172780",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"word_gap_prediction(\"test-A/\", model)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"id": "2481727e-94b5-49a0-9c21-0e105af6ef5b",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": []
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"kernelspec": {
|
|
||||||
"display_name": "Python 3 (ipykernel)",
|
|
||||||
"language": "python",
|
|
||||||
"name": "python3"
|
|
||||||
},
|
|
||||||
"language_info": {
|
|
||||||
"codemirror_mode": {
|
|
||||||
"name": "ipython",
|
|
||||||
"version": 3
|
|
||||||
},
|
|
||||||
"file_extension": ".py",
|
|
||||||
"mimetype": "text/x-python",
|
|
||||||
"name": "python",
|
|
||||||
"nbconvert_exporter": "python",
|
|
||||||
"pygments_lexer": "ipython3",
|
|
||||||
"version": "3.9.7"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 5
|
|
||||||
}
|
|
||||||
358
run.py
358
run.py
@ -1,278 +1,80 @@
|
|||||||
{
|
from nltk import trigrams, word_tokenize
|
||||||
"cells": [
|
import pandas as pd
|
||||||
{
|
import csv
|
||||||
"cell_type": "code",
|
import regex as re
|
||||||
"execution_count": null,
|
from collections import Counter, defaultdict
|
||||||
"id": "1da94494-ccbd-4f3c-9ca0-2241cfd9d361",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
train_set = pd.read_csv(
|
||||||
"source": []
|
'train/in.tsv.xz',
|
||||||
},
|
sep='\t',
|
||||||
{
|
on_bad_lines='skip',
|
||||||
"cell_type": "code",
|
header=None,
|
||||||
"execution_count": 2,
|
quoting=csv.QUOTE_NONE,
|
||||||
"id": "2f51e23a-93a0-4bf6-9c87-19da220e11bd",
|
nrows=50000)
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
train_labels = pd.read_csv(
|
||||||
"name": "stdout",
|
'train/expected.tsv',
|
||||||
"output_type": "stream",
|
sep='\t',
|
||||||
"text": [
|
on_bad_lines='skip',
|
||||||
"Collecting english_words\n",
|
header=None,
|
||||||
" Downloading english-words-1.1.0.tar.gz (1.1 MB)\n",
|
quoting=csv.QUOTE_NONE,
|
||||||
"\u001b[K |████████████████████████████████| 1.1 MB 1.5 MB/s eta 0:00:01\n",
|
nrows=50000)
|
||||||
"\u001b[?25hBuilding wheels for collected packages: english-words\n",
|
|
||||||
" Building wheel for english-words (setup.py) ... \u001b[?25ldone\n",
|
|
||||||
"\u001b[?25h Created wheel for english-words: filename=english_words-1.1.0-py3-none-any.whl size=1106680 sha256=ddaf5f4288a2022c2ce712aad0ba022e7b25d4d7e73c5637d6154abc5a899662\n",
|
def data_preprocessing(text):
|
||||||
" Stored in directory: /home/asadursk/.cache/pip/wheels/0e/24/52/b4989db82a438482aa65b3c6c0537e988fd40546b792747b1a\n",
|
return re.sub(r'\p{P}', '', text.lower().replace('-\\n', '').replace('\\n', ' '))
|
||||||
"Successfully built english-words\n",
|
|
||||||
"Installing collected packages: english-words\n",
|
|
||||||
"Successfully installed english-words-1.1.0\n"
|
def predict(before, after):
|
||||||
]
|
prediction = dict(Counter(dict(trigram[before, after])).most_common(5))
|
||||||
}
|
result = ''
|
||||||
],
|
prob = 0.0
|
||||||
"source": [
|
for key, value in prediction.items():
|
||||||
"!pip install english_words"
|
prob += value
|
||||||
]
|
result += f'{key}:{value} '
|
||||||
},
|
if prob == 0.0:
|
||||||
{
|
return 'to:0.015 be:0.015 the:0.015 not:0.01 and:0.02 a:0.02 :0.9'
|
||||||
"cell_type": "code",
|
result += f':{max(1 - prob, 0.01)}'
|
||||||
"execution_count": 5,
|
return result
|
||||||
"id": "d99975a7-aebe-4e26-b330-4be7f32204c5",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
def make_prediction(file):
|
||||||
{
|
data = pd.read_csv(f'{file}/in.tsv.xz', sep='\t', on_bad_lines='skip', header=None, quoting=csv.QUOTE_NONE)
|
||||||
"name": "stdout",
|
with open(f'{file}/out.tsv', 'w', encoding='utf-8') as file_out:
|
||||||
"output_type": "stream",
|
for _, row in data.iterrows():
|
||||||
"text": [
|
before, after = word_tokenize(data_preprocessing(str(row[6]))), word_tokenize(data_preprocessing(str(row[7])))
|
||||||
"Collecting pypi-kenlm\n",
|
if len(before) < 3 or len(after) < 3:
|
||||||
" Downloading pypi-kenlm-0.1.20210121.tar.gz (253 kB)\n",
|
prediction = 'to:0.015 be:0.015 the:0.015 not:0.01 and:0.02 a:0.02 :0.9'
|
||||||
"\u001b[K |████████████████████████████████| 253 kB 1.6 MB/s eta 0:00:01\n",
|
else:
|
||||||
"\u001b[?25hBuilding wheels for collected packages: pypi-kenlm\n",
|
prediction = predict(before[-1], after[0])
|
||||||
" Building wheel for pypi-kenlm (setup.py) ... \u001b[?25ldone\n",
|
file_out.write(prediction + '\n')
|
||||||
"\u001b[?25h Created wheel for pypi-kenlm: filename=pypi_kenlm-0.1.20210121-cp39-cp39-linux_x86_64.whl size=311921 sha256=2fcde1a0b569c5d5aef6c61014559b38efc45ed4ae90357c1219816d9a5bbe9b\n",
|
|
||||||
" Stored in directory: /home/asadursk/.cache/pip/wheels/14/f0/7a/97db71356d1dc1b0c14bf48e0d01e5561d5d67ba869e4406d0\n",
|
|
||||||
"Successfully built pypi-kenlm\n",
|
train_set = train_set[[6, 7]]
|
||||||
"Installing collected packages: pypi-kenlm\n",
|
train_set = pd.concat([train_set, train_labels], axis=1)
|
||||||
"Successfully installed pypi-kenlm-0.1.20210121\n"
|
train_set['line'] = train_set[6] + train_set[0] + train_set[7]
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
trigram = defaultdict(lambda: defaultdict(lambda: 0))
|
||||||
"source": [
|
|
||||||
"!python -m pip install pypi-kenlm"
|
rows = train_set.iterrows()
|
||||||
]
|
rows_len = len(train_set)
|
||||||
},
|
for index, (_, row) in enumerate(rows):
|
||||||
{
|
text = data_preprocessing(str(row['line']))
|
||||||
"cell_type": "code",
|
words = word_tokenize(text)
|
||||||
"execution_count": 27,
|
for word_1, word_2, word_3 in trigrams(words, pad_right=True, pad_left=True):
|
||||||
"id": "84560801-85f1-409b-a9c8-c209928276cc",
|
if word_1 and word_2 and word_3:
|
||||||
"metadata": {},
|
trigram[(word_1, word_3)][word_2] += 1
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
model_len = len(trigram)
|
||||||
"from collections import defaultdict, Counter\n",
|
for index, words_1_3 in enumerate(trigram):
|
||||||
"from nltk import trigrams, word_tokenize\n",
|
count = sum(trigram[words_1_3].values())
|
||||||
"from english_words import english_words_alpha_set\n",
|
for word_2 in trigram[words_1_3]:
|
||||||
"import csv\n",
|
trigram[words_1_3][word_2] += 0.25
|
||||||
"import regex as re\n",
|
trigram[words_1_3][word_2] /= float(count + 0.25 + len(word_2))
|
||||||
"import pandas as pd\n",
|
|
||||||
"import kenlm\n",
|
|
||||||
"from math import log10"
|
make_prediction('test-A')
|
||||||
]
|
make_prediction('dev-0')
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 29,
|
|
||||||
"id": "7a39272c-7929-42d8-98ba-8304570439af",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"def preprocess(row):\n",
|
|
||||||
" return re.sub(r'\\p{P}', '', row.lower().replace('-\\\\\\\\n', '').replace('\\\\\\\\n', ' '))"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 30,
|
|
||||||
"id": "2a330ad2-9b88-4fdd-bc04-635b5cb42c0d",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"def kenlm_model():\n",
|
|
||||||
" with open(\"train_file.txt\", \"w+\") as f:\n",
|
|
||||||
" for text in X_train:\n",
|
|
||||||
" f.write(str(text) + \"\\n\")\n",
|
|
||||||
"\n",
|
|
||||||
" #%%\n",
|
|
||||||
" KENLM_BUILD_PATH='/home/asadursk/kenlm/build'\n",
|
|
||||||
" !$KENLM_BUILD_PATH/bin/lmplz -o 4 < train_file.txt > model.arpa\n",
|
|
||||||
" !$KENLM_BUILD_PATH/bin/build_binary model.arpa model.binary\n",
|
|
||||||
" !rm train_file.txt\n",
|
|
||||||
" \n",
|
|
||||||
" model = kenlm.Model(\"model.binary\")\n",
|
|
||||||
" return model"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 31,
|
|
||||||
"id": "e848ba36-f4eb-4bd6-9b19-fffea177bfa1",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"def predict_word(w1, w3):\n",
|
|
||||||
" best_scores = []\n",
|
|
||||||
" for word in english_words_alpha_set:\n",
|
|
||||||
" text = ' '.join([w1, word, w3])\n",
|
|
||||||
" text_score = model.score(text, bos=False, eos=False)\n",
|
|
||||||
" if len(best_scores) < 12:\n",
|
|
||||||
" best_scores.append((word, text_score))\n",
|
|
||||||
" else:\n",
|
|
||||||
" is_better = False\n",
|
|
||||||
" worst_score = None\n",
|
|
||||||
" for score in best_scores:\n",
|
|
||||||
" if not worst_score:\n",
|
|
||||||
" worst_score = score\n",
|
|
||||||
" else:\n",
|
|
||||||
" if worst_score[1] > score[1]:\n",
|
|
||||||
" worst_score = score\n",
|
|
||||||
" if worst_score[1] < text_score:\n",
|
|
||||||
" best_scores.remove(worst_score)\n",
|
|
||||||
" best_scores.append((word, text_score))\n",
|
|
||||||
" probs = sorted(best_scores, key=lambda tup: tup[1], reverse=True)\n",
|
|
||||||
" pred_str = ''\n",
|
|
||||||
" for word, prob in probs:\n",
|
|
||||||
" pred_str += f'{word}:{prob} '\n",
|
|
||||||
" pred_str += f':{log10(0.99)}'\n",
|
|
||||||
" return pred_str"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 32,
|
|
||||||
"id": "6babeba5-af91-4e9c-a235-781525594f45",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"def word_gap_prediction(file, model):\n",
|
|
||||||
" X_test = pd.read_csv(f'{file}/in.tsv.xz', sep='\\t', header=None, quoting=csv.QUOTE_NONE, on_bad_lines=\"skip\")\n",
|
|
||||||
" with open(f'{file}/out.tsv', 'w', encoding='utf-8') as output_file:\n",
|
|
||||||
" for _, row in X_test.iterrows():\n",
|
|
||||||
" before, after = word_tokenize(preprocess(str(row[6]))), word_tokenize(preprocess(str(row[7])))\n",
|
|
||||||
" if len(before) < 2 or len(after) < 2:\n",
|
|
||||||
" output = 'to:0.015 be:0.015 the:0.015 not:0.01 and:0.02 a:0.02 :0.9'\n",
|
|
||||||
" else:\n",
|
|
||||||
" output = predict_word(before[-1], after[0])\n",
|
|
||||||
" output_file.write(output + '\\n')"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 33,
|
|
||||||
"id": "8df4a04c-ae0d-46d7-8b76-1bcf6b424d7a",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"=== 1/5 Counting and sorting n-grams ===\n",
|
|
||||||
"Reading /home/asadursk/challenging-america-word-gap-prediction-kenlm/train_file.txt\n",
|
|
||||||
"----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
|
|
||||||
"****************************************************************************************************\n",
|
|
||||||
"Unigram tokens 2787545 types 548500\n",
|
|
||||||
"=== 2/5 Calculating and sorting adjusted counts ===\n",
|
|
||||||
"Chain sizes: 1:6582000 2:865198656 3:1622247552 4:2595596032\n",
|
|
||||||
"Statistics:\n",
|
|
||||||
"1 548500 D1=0.85065 D2=1.01013 D3+=1.14959\n",
|
|
||||||
"2 1743634 D1=0.900957 D2=1.09827 D3+=1.20014\n",
|
|
||||||
"3 2511917 D1=0.957313 D2=1.22283 D3+=1.33724\n",
|
|
||||||
"4 2719775 D1=0.982576 D2=1.4205 D3+=1.65074\n",
|
|
||||||
"Memory estimate for binary LM:\n",
|
|
||||||
"type MB\n",
|
|
||||||
"probing 157 assuming -p 1.5\n",
|
|
||||||
"probing 184 assuming -r models -p 1.5\n",
|
|
||||||
"trie 82 without quantization\n",
|
|
||||||
"trie 51 assuming -q 8 -b 8 quantization \n",
|
|
||||||
"trie 74 assuming -a 22 array pointer compression\n",
|
|
||||||
"trie 43 assuming -a 22 -q 8 -b 8 array pointer compression and quantization\n",
|
|
||||||
"=== 3/5 Calculating and sorting initial probabilities ===\n",
|
|
||||||
"Chain sizes: 1:6582000 2:27898144 3:50238340 4:65274600\n",
|
|
||||||
"----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
|
|
||||||
"####################################################################################################\n",
|
|
||||||
"=== 4/5 Calculating and writing order-interpolated probabilities ===\n",
|
|
||||||
"Chain sizes: 1:6582000 2:27898144 3:50238340 4:65274600\n",
|
|
||||||
"----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
|
|
||||||
"####################################################################################################\n",
|
|
||||||
"=== 5/5 Writing ARPA model ===\n",
|
|
||||||
"----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
|
|
||||||
"****************************************************************************************************\n",
|
|
||||||
"Name:lmplz\tVmPeak:5126188 kB\tVmRSS:54384 kB\tRSSMax:1084112 kB\tuser:9.18382\tsys:2.72419\tCPU:11.9081\treal:9.09119\n",
|
|
||||||
"Reading model.arpa\n",
|
|
||||||
"----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
|
|
||||||
"****************************************************************************************************\n",
|
|
||||||
"SUCCESS\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"X_train = pd.read_csv('train/in.tsv.xz', sep='\\t', header=None, quoting=csv.QUOTE_NONE, nrows=10000, on_bad_lines=\"skip\")\n",
|
|
||||||
"Y_train = pd.read_csv('train/expected.tsv', sep='\\t', header=None, quoting=csv.QUOTE_NONE, nrows=10000, on_bad_lines=\"skip\")\n",
|
|
||||||
"\n",
|
|
||||||
"X_train = X_train[[6, 7]]\n",
|
|
||||||
"X_train = pd.concat([X_train, Y_train], axis=1)\n",
|
|
||||||
"X_train = X_train[6] + X_train[0] + X_train[7]\n",
|
|
||||||
"\n",
|
|
||||||
"model = kenlm_model()"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 34,
|
|
||||||
"id": "5f9b4351-54b6-42de-8653-597b17c42766",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"word_gap_prediction(\"dev-0/\", model)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 35,
|
|
||||||
"id": "71076162-473b-40f2-93ab-0536a2172780",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"word_gap_prediction(\"test-A/\", model)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"id": "2481727e-94b5-49a0-9c21-0e105af6ef5b",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": []
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"kernelspec": {
|
|
||||||
"display_name": "Python 3 (ipykernel)",
|
|
||||||
"language": "python",
|
|
||||||
"name": "python3"
|
|
||||||
},
|
|
||||||
"language_info": {
|
|
||||||
"codemirror_mode": {
|
|
||||||
"name": "ipython",
|
|
||||||
"version": 3
|
|
||||||
},
|
|
||||||
"file_extension": ".py",
|
|
||||||
"mimetype": "text/x-python",
|
|
||||||
"name": "python",
|
|
||||||
"nbconvert_exporter": "python",
|
|
||||||
"pygments_lexer": "ipython3",
|
|
||||||
"version": "3.9.7"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 5
|
|
||||||
}
|
|
||||||
|
|||||||
14828
test-A/out.tsv
14828
test-A/out.tsv
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user