final solution
This commit is contained in:
parent
3dd8f4130a
commit
dbc2815e28
4
.gitignore
vendored
Normal file
4
.gitignore
vendored
Normal file
@ -0,0 +1,4 @@
|
||||
.ipynb_checkpoints/
|
||||
model/
|
||||
geval
|
||||
processed_train.txt
|
||||
@ -2,12 +2,12 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": 1,
|
||||
"id": "f73a28ea",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"KENLM_BUILD_PATH='/home/students/s434708/kenlm/build'"
|
||||
"KENLM_BUILD_PATH='/home/haskell/kenlm/build'"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -20,7 +20,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 2,
|
||||
"id": "d42ddd87",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@ -32,7 +32,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 3,
|
||||
"id": "f84be210",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@ -46,7 +46,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 4,
|
||||
"id": "de0c12d6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@ -85,13 +85,13 @@
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"=== 1/5 Counting and sorting n-grams ===\n",
|
||||
"Reading /home/students/s434708/Desktop/Modelowanie Języka/challenging-america-word-gap-prediction-kenlm/processed_train.txt\n",
|
||||
"Reading /home/haskell/Desktop/challenging-america-word-gap-prediction-kenlm/processed_train.txt\n",
|
||||
"----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
|
||||
"********************************Warning: <s> appears in the input. All instances of <s>, </s>, and <unk> will be interpreted as whitespace.\n",
|
||||
"********************************************************************\n",
|
||||
"Unigram tokens 135911223 types 4381594\n",
|
||||
"=== 2/5 Calculating and sorting adjusted counts ===\n",
|
||||
"Chain sizes: 1:52579128 2:1295655936 3:2429355008 4:3886967808 5:5668495360\n",
|
||||
"Chain sizes: 1:52579128 2:896866240 3:1681624320 4:2690598656 5:3923790080\n",
|
||||
"Statistics:\n",
|
||||
"1 4381594 D1=0.841838 D2=1.01787 D3+=1.21057\n",
|
||||
"2 26800631 D1=0.836734 D2=1.01657 D3+=1.19437\n",
|
||||
@ -116,21 +116,18 @@
|
||||
"####################################################################################################\n",
|
||||
"=== 5/5 Writing ARPA model ===\n",
|
||||
"----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
|
||||
"----------------------------------------------------------------------------------------------------Last input should have been poison. The program should end soon with an error. If it doesn't, there's a bug.\n",
|
||||
"terminate called after throwing an instance of 'util::FDException'\n",
|
||||
" what(): /home/students/s434708/kenlm/util/file.cc:228 in void util::WriteOrThrow(int, const void*, std::size_t) threw FDException because `ret < 1'.\n",
|
||||
"No space left on device in /home/students/s434708/Desktop/Modelowanie Języka/challenging-america-word-gap-prediction-kenlm/model.arpa while writing 8189 bytes\n",
|
||||
"/bin/bash: line 1: 26725 Aborted /home/students/s434708/kenlm/build/bin/lmplz -o 5 --skip_symbols < processed_train.txt > model.arpa\n"
|
||||
"****************************************************************************************************\n",
|
||||
"Name:lmplz\tVmPeak:9201752 kB\tVmRSS:2564 kB\tRSSMax:7648448 kB\tuser:506.342\tsys:106.578\tCPU:612.92\treal:1564.6\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"!$KENLM_BUILD_PATH/bin/lmplz -o 5 --skip_symbols < processed_train.txt > model.arpa"
|
||||
"!$KENLM_BUILD_PATH/bin/lmplz -o 5 --skip_symbols < processed_train.txt > model/model.arpa"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": 5,
|
||||
"id": "dc65780b",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
@ -138,22 +135,20 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Reading model.arpa\n",
|
||||
"Reading model/model.arpa\n",
|
||||
"----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
|
||||
"****************************************************************************************************\n",
|
||||
"/home/students/s434708/kenlm/util/file.cc:86 in int util::CreateOrThrow(const char*) threw ErrnoException because `-1 == (ret = open(name, 0100 | 01000 | 02, 0400 | 0200 | (0400 >> 3) | ((0400 >> 3) >> 3)))'.\n",
|
||||
"No space left on device while creating model.binary Byte: 94\n",
|
||||
"ERROR\n"
|
||||
"SUCCESS\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"!$KENLM_BUILD_PATH/bin/build_binary model.arpa model.binary"
|
||||
"!$KENLM_BUILD_PATH/bin/build_binary model/model.arpa model/model.binary"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"execution_count": 6,
|
||||
"id": "2087eb80",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@ -163,12 +158,12 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": 7,
|
||||
"id": "4ba1e592",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!rm model.arpa"
|
||||
"!rm model/model.arpa"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -181,36 +176,127 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 32,
|
||||
"id": "6865301b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import kenlm"
|
||||
"import kenlm\n",
|
||||
"import csv\n",
|
||||
"import pandas as pd\n",
|
||||
"import regex as re\n",
|
||||
"from math import log10\n",
|
||||
"from nltk import word_tokenize\n",
|
||||
"from english_words import english_words_alpha_set"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 4,
|
||||
"id": "e32de662",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"test_str = 'really good'\n",
|
||||
"\n",
|
||||
"model = kenlm.Model('model.binary')\n",
|
||||
"print(model.score(test_str, bos = True, eos = True))"
|
||||
"model = kenlm.Model('model/model.binary')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 28,
|
||||
"id": "c2535482",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def clean_text(text):\n",
|
||||
" text = text.lower().replace('-\\\\n', '').replace('\\\\n', ' ')\n",
|
||||
" text = re.sub(r'\\p{P}', '', text)\n",
|
||||
"\n",
|
||||
" return text"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 29,
|
||||
"id": "2308ccad",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def predict_probs(w1, w2, w4):\n",
|
||||
" best_scores = []\n",
|
||||
" for word in english_words_alpha_set:\n",
|
||||
" text = ' '.join([w1, w2, word, w4])\n",
|
||||
" text_score = model.score(text, bos=False, eos=False)\n",
|
||||
" if len(best_scores) < 20:\n",
|
||||
" best_scores.append((word, text_score))\n",
|
||||
" else:\n",
|
||||
" is_better = False\n",
|
||||
" worst_score = None\n",
|
||||
" for score in best_scores:\n",
|
||||
" if not worst_score:\n",
|
||||
" worst_score = score\n",
|
||||
" else:\n",
|
||||
" if worst_score[1] > score[1]:\n",
|
||||
" worst_score = score\n",
|
||||
" if worst_score[1] < text_score:\n",
|
||||
" best_scores.remove(worst_score)\n",
|
||||
" best_scores.append((word, text_score))\n",
|
||||
" probs = sorted(best_scores, key=lambda tup: tup[1], reverse=True)\n",
|
||||
" pred_str = ''\n",
|
||||
" for word, prob in probs:\n",
|
||||
" pred_str += f'{word}:{prob} '\n",
|
||||
" pred_str += f':{log10(0.99)}'\n",
|
||||
" return pred_str"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 30,
|
||||
"id": "7245cf38",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"dev_data = pd.read_csv('dev-0/in.tsv.xz', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)\n",
|
||||
"test_data = pd.read_csv('test-A/in.tsv.xz', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 35,
|
||||
"id": "ac24ff37",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"with open('dev-0/out.tsv', 'w') as file:\n",
|
||||
" for index, row in dev_data.iterrows():\n",
|
||||
" left_text = clean_text(str(row[6]))\n",
|
||||
" right_text = clean_text(str(row[7]))\n",
|
||||
" left_words = word_tokenize(left_text)\n",
|
||||
" right_words = word_tokenize(right_text)\n",
|
||||
" if len(left_words) < 2 or len(right_words) < 2:\n",
|
||||
" prediction = ':1.0'\n",
|
||||
" else:\n",
|
||||
" prediction = predict_probs(left_words[len(left_words) - 2], left_words[len(left_words) - 1], right_words[0])\n",
|
||||
" file.write(prediction + '\\n')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 37,
|
||||
"id": "a18b6ebd",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"for i in model.full_scores(test_str):\n",
|
||||
" print(i)"
|
||||
"with open('test-A/out.tsv', 'w') as file:\n",
|
||||
" for index, row in test_data.iterrows():\n",
|
||||
" left_text = clean_text(str(row[6]))\n",
|
||||
" right_text = clean_text(str(row[7]))\n",
|
||||
" left_words = word_tokenize(left_text)\n",
|
||||
" right_words = word_tokenize(right_text)\n",
|
||||
" if len(left_words) < 2 or len(right_words) < 2:\n",
|
||||
" prediction = ':1.0'\n",
|
||||
" else:\n",
|
||||
" prediction = predict_probs(left_words[len(left_words) - 2], left_words[len(left_words) - 1], right_words[0])\n",
|
||||
" file.write(prediction + '\\n')"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
10519
dev-0/out.tsv
Normal file
10519
dev-0/out.tsv
Normal file
File diff suppressed because it is too large
Load Diff
@ -1,25 +0,0 @@
|
||||
import pandas as pd
|
||||
import csv
|
||||
import regex as re
|
||||
|
||||
def clean_text(text):
|
||||
text = text.lower().replace('-\\n', '').replace('\\n', ' ')
|
||||
text = re.sub(r'\p{P}', '', text)
|
||||
|
||||
return text
|
||||
|
||||
|
||||
train_data = pd.read_csv('train/in.tsv.xz', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
|
||||
train_labels = pd.read_csv('train/expected.tsv', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
|
||||
|
||||
train_data = train_data[[6, 7]]
|
||||
train_data = pd.concat([train_data, train_labels], axis=1)
|
||||
|
||||
train_data['text'] = train_data[6] + train_data[0] + train_data[7]
|
||||
train_data = train_data[['text']]
|
||||
|
||||
with open('processed_train.txt', 'w') as file:
|
||||
for _, row in train_data.iterrows():
|
||||
text = clean_text(str(row['text']))
|
||||
file.write(text + '\n')
|
||||
|
||||
150
run.ipynb
150
run.ipynb
@ -2,12 +2,12 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": 1,
|
||||
"id": "f73a28ea",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"KENLM_BUILD_PATH='/home/students/s434708/kenlm/build'"
|
||||
"KENLM_BUILD_PATH='/home/haskell/kenlm/build'"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -20,7 +20,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 2,
|
||||
"id": "d42ddd87",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@ -32,7 +32,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 3,
|
||||
"id": "f84be210",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@ -46,7 +46,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 4,
|
||||
"id": "de0c12d6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@ -85,13 +85,13 @@
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"=== 1/5 Counting and sorting n-grams ===\n",
|
||||
"Reading /home/students/s434708/Desktop/Modelowanie Języka/challenging-america-word-gap-prediction-kenlm/processed_train.txt\n",
|
||||
"Reading /home/haskell/Desktop/challenging-america-word-gap-prediction-kenlm/processed_train.txt\n",
|
||||
"----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
|
||||
"********************************Warning: <s> appears in the input. All instances of <s>, </s>, and <unk> will be interpreted as whitespace.\n",
|
||||
"********************************************************************\n",
|
||||
"Unigram tokens 135911223 types 4381594\n",
|
||||
"=== 2/5 Calculating and sorting adjusted counts ===\n",
|
||||
"Chain sizes: 1:52579128 2:1295655936 3:2429355008 4:3886967808 5:5668495360\n",
|
||||
"Chain sizes: 1:52579128 2:896866240 3:1681624320 4:2690598656 5:3923790080\n",
|
||||
"Statistics:\n",
|
||||
"1 4381594 D1=0.841838 D2=1.01787 D3+=1.21057\n",
|
||||
"2 26800631 D1=0.836734 D2=1.01657 D3+=1.19437\n",
|
||||
@ -116,21 +116,18 @@
|
||||
"####################################################################################################\n",
|
||||
"=== 5/5 Writing ARPA model ===\n",
|
||||
"----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
|
||||
"----------------------------------------------------------------------------------------------------Last input should have been poison. The program should end soon with an error. If it doesn't, there's a bug.\n",
|
||||
"terminate called after throwing an instance of 'util::FDException'\n",
|
||||
" what(): /home/students/s434708/kenlm/util/file.cc:228 in void util::WriteOrThrow(int, const void*, std::size_t) threw FDException because `ret < 1'.\n",
|
||||
"No space left on device in /home/students/s434708/Desktop/Modelowanie Języka/challenging-america-word-gap-prediction-kenlm/model.arpa while writing 8189 bytes\n",
|
||||
"/bin/bash: line 1: 26725 Aborted /home/students/s434708/kenlm/build/bin/lmplz -o 5 --skip_symbols < processed_train.txt > model.arpa\n"
|
||||
"****************************************************************************************************\n",
|
||||
"Name:lmplz\tVmPeak:9201752 kB\tVmRSS:2564 kB\tRSSMax:7648448 kB\tuser:506.342\tsys:106.578\tCPU:612.92\treal:1564.6\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"!$KENLM_BUILD_PATH/bin/lmplz -o 5 --skip_symbols < processed_train.txt > model.arpa"
|
||||
"!$KENLM_BUILD_PATH/bin/lmplz -o 5 --skip_symbols < processed_train.txt > model/model.arpa"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": 5,
|
||||
"id": "dc65780b",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
@ -138,22 +135,20 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Reading model.arpa\n",
|
||||
"Reading model/model.arpa\n",
|
||||
"----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
|
||||
"****************************************************************************************************\n",
|
||||
"/home/students/s434708/kenlm/util/file.cc:86 in int util::CreateOrThrow(const char*) threw ErrnoException because `-1 == (ret = open(name, 0100 | 01000 | 02, 0400 | 0200 | (0400 >> 3) | ((0400 >> 3) >> 3)))'.\n",
|
||||
"No space left on device while creating model.binary Byte: 94\n",
|
||||
"ERROR\n"
|
||||
"SUCCESS\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"!$KENLM_BUILD_PATH/bin/build_binary model.arpa model.binary"
|
||||
"!$KENLM_BUILD_PATH/bin/build_binary model/model.arpa model/model.binary"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"execution_count": 6,
|
||||
"id": "2087eb80",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@ -163,12 +158,12 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": 7,
|
||||
"id": "4ba1e592",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!rm model.arpa"
|
||||
"!rm model/model.arpa"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -181,36 +176,127 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 32,
|
||||
"id": "6865301b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import kenlm"
|
||||
"import kenlm\n",
|
||||
"import csv\n",
|
||||
"import pandas as pd\n",
|
||||
"import regex as re\n",
|
||||
"from math import log10\n",
|
||||
"from nltk import word_tokenize\n",
|
||||
"from english_words import english_words_alpha_set"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 4,
|
||||
"id": "e32de662",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"test_str = 'really good'\n",
|
||||
"\n",
|
||||
"model = kenlm.Model('model.binary')\n",
|
||||
"print(model.score(test_str, bos = True, eos = True))"
|
||||
"model = kenlm.Model('model/model.binary')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 28,
|
||||
"id": "c2535482",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def clean_text(text):\n",
|
||||
" text = text.lower().replace('-\\\\n', '').replace('\\\\n', ' ')\n",
|
||||
" text = re.sub(r'\\p{P}', '', text)\n",
|
||||
"\n",
|
||||
" return text"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 29,
|
||||
"id": "2308ccad",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def predict_probs(w1, w2, w4):\n",
|
||||
" best_scores = []\n",
|
||||
" for word in english_words_alpha_set:\n",
|
||||
" text = ' '.join([w1, w2, word, w4])\n",
|
||||
" text_score = model.score(text, bos=False, eos=False)\n",
|
||||
" if len(best_scores) < 20:\n",
|
||||
" best_scores.append((word, text_score))\n",
|
||||
" else:\n",
|
||||
" is_better = False\n",
|
||||
" worst_score = None\n",
|
||||
" for score in best_scores:\n",
|
||||
" if not worst_score:\n",
|
||||
" worst_score = score\n",
|
||||
" else:\n",
|
||||
" if worst_score[1] > score[1]:\n",
|
||||
" worst_score = score\n",
|
||||
" if worst_score[1] < text_score:\n",
|
||||
" best_scores.remove(worst_score)\n",
|
||||
" best_scores.append((word, text_score))\n",
|
||||
" probs = sorted(best_scores, key=lambda tup: tup[1], reverse=True)\n",
|
||||
" pred_str = ''\n",
|
||||
" for word, prob in probs:\n",
|
||||
" pred_str += f'{word}:{prob} '\n",
|
||||
" pred_str += f':{log10(0.99)}'\n",
|
||||
" return pred_str"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 30,
|
||||
"id": "7245cf38",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"dev_data = pd.read_csv('dev-0/in.tsv.xz', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)\n",
|
||||
"test_data = pd.read_csv('test-A/in.tsv.xz', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 35,
|
||||
"id": "ac24ff37",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"with open('dev-0/out.tsv', 'w') as file:\n",
|
||||
" for index, row in dev_data.iterrows():\n",
|
||||
" left_text = clean_text(str(row[6]))\n",
|
||||
" right_text = clean_text(str(row[7]))\n",
|
||||
" left_words = word_tokenize(left_text)\n",
|
||||
" right_words = word_tokenize(right_text)\n",
|
||||
" if len(left_words) < 2 or len(right_words) < 2:\n",
|
||||
" prediction = ':1.0'\n",
|
||||
" else:\n",
|
||||
" prediction = predict_probs(left_words[len(left_words) - 2], left_words[len(left_words) - 1], right_words[0])\n",
|
||||
" file.write(prediction + '\\n')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 37,
|
||||
"id": "a18b6ebd",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"for i in model.full_scores(test_str):\n",
|
||||
" print(i)"
|
||||
"with open('test-A/out.tsv', 'w') as file:\n",
|
||||
" for index, row in test_data.iterrows():\n",
|
||||
" left_text = clean_text(str(row[6]))\n",
|
||||
" right_text = clean_text(str(row[7]))\n",
|
||||
" left_words = word_tokenize(left_text)\n",
|
||||
" right_words = word_tokenize(right_text)\n",
|
||||
" if len(left_words) < 2 or len(right_words) < 2:\n",
|
||||
" prediction = ':1.0'\n",
|
||||
" else:\n",
|
||||
" prediction = predict_probs(left_words[len(left_words) - 2], left_words[len(left_words) - 1], right_words[0])\n",
|
||||
" file.write(prediction + '\\n')"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
171
run.py
Normal file
171
run.py
Normal file
@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[1]:
|
||||
|
||||
|
||||
KENLM_BUILD_PATH='/home/haskell/kenlm/build'
|
||||
|
||||
|
||||
# ### Preprocessing danych
|
||||
|
||||
# In[2]:
|
||||
|
||||
|
||||
import pandas as pd
|
||||
import csv
|
||||
import regex as re
|
||||
|
||||
|
||||
# In[3]:
|
||||
|
||||
|
||||
def clean_text(text):
|
||||
text = text.lower().replace('-\\n', '').replace('\\n', ' ')
|
||||
text = re.sub(r'\p{P}', '', text)
|
||||
|
||||
return text
|
||||
|
||||
|
||||
# In[4]:
|
||||
|
||||
|
||||
train_data = pd.read_csv('train/in.tsv.xz', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
|
||||
train_labels = pd.read_csv('train/expected.tsv', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
|
||||
|
||||
train_data = train_data[[6, 7]]
|
||||
train_data = pd.concat([train_data, train_labels], axis=1)
|
||||
|
||||
train_data['text'] = train_data[6] + train_data[0] + train_data[7]
|
||||
train_data = train_data[['text']]
|
||||
|
||||
with open('processed_train.txt', 'w') as file:
|
||||
for _, row in train_data.iterrows():
|
||||
text = clean_text(str(row['text']))
|
||||
file.write(text + '\n')
|
||||
|
||||
|
||||
# ### Model kenLM
|
||||
|
||||
# In[4]:
|
||||
|
||||
|
||||
get_ipython().system('$KENLM_BUILD_PATH/bin/lmplz -o 5 --skip_symbols < processed_train.txt > model/model.arpa')
|
||||
|
||||
|
||||
# In[5]:
|
||||
|
||||
|
||||
get_ipython().system('$KENLM_BUILD_PATH/bin/build_binary model/model.arpa model/model.binary')
|
||||
|
||||
|
||||
# In[6]:
|
||||
|
||||
|
||||
get_ipython().system('rm processed_train.txt')
|
||||
|
||||
|
||||
# In[7]:
|
||||
|
||||
|
||||
get_ipython().system('rm model/model.arpa')
|
||||
|
||||
|
||||
# ### Predykcje
|
||||
|
||||
# In[32]:
|
||||
|
||||
|
||||
import kenlm
|
||||
import csv
|
||||
import pandas as pd
|
||||
import regex as re
|
||||
from math import log10
|
||||
from nltk import word_tokenize
|
||||
from english_words import english_words_alpha_set
|
||||
|
||||
|
||||
# In[4]:
|
||||
|
||||
|
||||
model = kenlm.Model('model/model.binary')
|
||||
|
||||
|
||||
# In[28]:
|
||||
|
||||
|
||||
def clean_text(text):
|
||||
text = text.lower().replace('-\\n', '').replace('\\n', ' ')
|
||||
text = re.sub(r'\p{P}', '', text)
|
||||
|
||||
return text
|
||||
|
||||
|
||||
# In[29]:
|
||||
|
||||
|
||||
def predict_probs(w1, w2, w4):
|
||||
best_scores = []
|
||||
for word in english_words_alpha_set:
|
||||
text = ' '.join([w1, w2, word, w4])
|
||||
text_score = model.score(text, bos=False, eos=False)
|
||||
if len(best_scores) < 20:
|
||||
best_scores.append((word, text_score))
|
||||
else:
|
||||
is_better = False
|
||||
worst_score = None
|
||||
for score in best_scores:
|
||||
if not worst_score:
|
||||
worst_score = score
|
||||
else:
|
||||
if worst_score[1] > score[1]:
|
||||
worst_score = score
|
||||
if worst_score[1] < text_score:
|
||||
best_scores.remove(worst_score)
|
||||
best_scores.append((word, text_score))
|
||||
probs = sorted(best_scores, key=lambda tup: tup[1], reverse=True)
|
||||
pred_str = ''
|
||||
for word, prob in probs:
|
||||
pred_str += f'{word}:{prob} '
|
||||
pred_str += f':{log10(0.99)}'
|
||||
return pred_str
|
||||
|
||||
|
||||
# In[30]:
|
||||
|
||||
|
||||
dev_data = pd.read_csv('dev-0/in.tsv.xz', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
|
||||
test_data = pd.read_csv('test-A/in.tsv.xz', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
|
||||
|
||||
|
||||
# In[35]:
|
||||
|
||||
|
||||
with open('dev-0/out.tsv', 'w') as file:
|
||||
for index, row in dev_data.iterrows():
|
||||
left_text = clean_text(str(row[6]))
|
||||
right_text = clean_text(str(row[7]))
|
||||
left_words = word_tokenize(left_text)
|
||||
right_words = word_tokenize(right_text)
|
||||
if len(left_words) < 2 or len(right_words) < 2:
|
||||
prediction = ':1.0'
|
||||
else:
|
||||
prediction = predict_probs(left_words[len(left_words) - 2], left_words[len(left_words) - 1], right_words[0])
|
||||
file.write(prediction + '\n')
|
||||
|
||||
|
||||
# In[37]:
|
||||
|
||||
|
||||
with open('test-A/out.tsv', 'w') as file:
|
||||
for index, row in test_data.iterrows():
|
||||
left_text = clean_text(str(row[6]))
|
||||
right_text = clean_text(str(row[7]))
|
||||
left_words = word_tokenize(left_text)
|
||||
right_words = word_tokenize(right_text)
|
||||
if len(left_words) < 2 or len(right_words) < 2:
|
||||
prediction = ':1.0'
|
||||
else:
|
||||
prediction = predict_probs(left_words[len(left_words) - 2], left_words[len(left_words) - 1], right_words[0])
|
||||
file.write(prediction + '\n')
|
||||
|
||||
7414
test-A/out.tsv
Normal file
7414
test-A/out.tsv
Normal file
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user