{ "cells": [ { "cell_type": "code", "execution_count": 2, "id": "f73a28ea", "metadata": {}, "outputs": [], "source": [ "KENLM_BUILD_PATH='/home/students/s434708/kenlm/build'" ] }, { "cell_type": "markdown", "id": "9fc5cda3", "metadata": {}, "source": [ "### Preprocessing danych" ] }, { "cell_type": "code", "execution_count": null, "id": "d42ddd87", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import csv\n", "import regex as re" ] }, { "cell_type": "code", "execution_count": null, "id": "f84be210", "metadata": {}, "outputs": [], "source": [ "def clean_text(text):\n", " text = text.lower().replace('-\\\\n', '').replace('\\\\n', ' ')\n", " text = re.sub(r'\\p{P}', '', text)\n", "\n", " return text" ] }, { "cell_type": "code", "execution_count": null, "id": "de0c12d6", "metadata": {}, "outputs": [], "source": [ "train_data = pd.read_csv('train/in.tsv.xz', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)\n", "train_labels = pd.read_csv('train/expected.tsv', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)\n", "\n", "train_data = train_data[[6, 7]]\n", "train_data = pd.concat([train_data, train_labels], axis=1)\n", "\n", "train_data['text'] = train_data[6] + train_data[0] + train_data[7]\n", "train_data = train_data[['text']]\n", "\n", "with open('processed_train.txt', 'w') as file:\n", " for _, row in train_data.iterrows():\n", " text = clean_text(str(row['text']))\n", " file.write(text + '\\n')" ] }, { "cell_type": "markdown", "id": "846b6b42", "metadata": {}, "source": [ "### Model kenLM" ] }, { "cell_type": "code", "execution_count": 4, "id": "3c74d4be", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "=== 1/5 Counting and sorting n-grams ===\n", "Reading /home/students/s434708/Desktop/Modelowanie Języka/challenging-america-word-gap-prediction-kenlm/processed_train.txt\n", "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n", "********************************Warning: appears in the input. All instances of , , and will be interpreted as whitespace.\n", "********************************************************************\n", "Unigram tokens 135911223 types 4381594\n", "=== 2/5 Calculating and sorting adjusted counts ===\n", "Chain sizes: 1:52579128 2:1295655936 3:2429355008 4:3886967808 5:5668495360\n", "Statistics:\n", "1 4381594 D1=0.841838 D2=1.01787 D3+=1.21057\n", "2 26800631 D1=0.836734 D2=1.01657 D3+=1.19437\n", "3 69811700 D1=0.878562 D2=1.11227 D3+=1.27889\n", "4 104063034 D1=0.931257 D2=1.23707 D3+=1.36664\n", "5 119487533 D1=0.938146 D2=1.3058 D3+=1.41614\n", "Memory estimate for binary LM:\n", "type MB\n", "probing 6752 assuming -p 1.5\n", "probing 7917 assuming -r models -p 1.5\n", "trie 3572 without quantization\n", "trie 2120 assuming -q 8 -b 8 quantization \n", "trie 3104 assuming -a 22 array pointer compression\n", "trie 1652 assuming -a 22 -q 8 -b 8 array pointer compression and quantization\n", "=== 3/5 Calculating and sorting initial probabilities ===\n", "Chain sizes: 1:52579128 2:428810096 3:1396234000 4:2497512816 5:3345650924\n", "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n", "####################################################################################################\n", "=== 4/5 Calculating and writing order-interpolated probabilities ===\n", "Chain sizes: 1:52579128 2:428810096 3:1396234000 4:2497512816 5:3345650924\n", "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n", "####################################################################################################\n", "=== 5/5 Writing ARPA model ===\n", "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n", "----------------------------------------------------------------------------------------------------Last input should have been poison. The program should end soon with an error. If it doesn't, there's a bug.\n", "terminate called after throwing an instance of 'util::FDException'\n", " what(): /home/students/s434708/kenlm/util/file.cc:228 in void util::WriteOrThrow(int, const void*, std::size_t) threw FDException because `ret < 1'.\n", "No space left on device in /home/students/s434708/Desktop/Modelowanie Języka/challenging-america-word-gap-prediction-kenlm/model.arpa while writing 8189 bytes\n", "/bin/bash: line 1: 26725 Aborted /home/students/s434708/kenlm/build/bin/lmplz -o 5 --skip_symbols < processed_train.txt > model.arpa\n" ] } ], "source": [ "!$KENLM_BUILD_PATH/bin/lmplz -o 5 --skip_symbols < processed_train.txt > model.arpa" ] }, { "cell_type": "code", "execution_count": 6, "id": "dc65780b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Reading model.arpa\n", "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n", "****************************************************************************************************\n", "/home/students/s434708/kenlm/util/file.cc:86 in int util::CreateOrThrow(const char*) threw ErrnoException because `-1 == (ret = open(name, 0100 | 01000 | 02, 0400 | 0200 | (0400 >> 3) | ((0400 >> 3) >> 3)))'.\n", "No space left on device while creating model.binary Byte: 94\n", "ERROR\n" ] } ], "source": [ "!$KENLM_BUILD_PATH/bin/build_binary model.arpa model.binary" ] }, { "cell_type": "code", "execution_count": 7, "id": "2087eb80", "metadata": {}, "outputs": [], "source": [ "!rm processed_train.txt" ] }, { "cell_type": "code", "execution_count": 8, "id": "4ba1e592", "metadata": {}, "outputs": [], "source": [ "!rm model.arpa" ] }, { "cell_type": "markdown", "id": "e41f7951", "metadata": {}, "source": [ "### Predykcje" ] }, { "cell_type": "code", "execution_count": null, "id": "6865301b", "metadata": {}, "outputs": [], "source": [ "import kenlm" ] }, { "cell_type": "code", "execution_count": null, "id": "e32de662", "metadata": {}, "outputs": [], "source": [ "test_str = 'really good'\n", "\n", "model = kenlm.Model('model.binary')\n", "print(model.score(test_str, bos = True, eos = True))" ] }, { "cell_type": "code", "execution_count": null, "id": "a18b6ebd", "metadata": {}, "outputs": [], "source": [ "for i in model.full_scores(test_str):\n", " print(i)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }