init solution

.ipynb_checkpoints/Untitled-checkpoint.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

.ipynb_checkpoints/run-checkpoint.ipynb

@@ -0,0 +1,238 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "f73a28ea",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "KENLM_BUILD_PATH='/home/students/s434708/kenlm/build'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9fc5cda3",
+   "metadata": {},
+   "source": [
+    "### Preprocessing danych"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d42ddd87",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import csv\n",
+    "import regex as re"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f84be210",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def clean_text(text):\n",
+    "    text = text.lower().replace('-\\\\n', '').replace('\\\\n', ' ')\n",
+    "    text = re.sub(r'\\p{P}', '', text)\n",
+    "\n",
+    "    return text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "de0c12d6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_data = pd.read_csv('train/in.tsv.xz', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)\n",
+    "train_labels = pd.read_csv('train/expected.tsv', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)\n",
+    "\n",
+    "train_data = train_data[[6, 7]]\n",
+    "train_data = pd.concat([train_data, train_labels], axis=1)\n",
+    "\n",
+    "train_data['text'] = train_data[6] + train_data[0] + train_data[7]\n",
+    "train_data = train_data[['text']]\n",
+    "\n",
+    "with open('processed_train.txt', 'w') as file:\n",
+    "    for _, row in train_data.iterrows():\n",
+    "        text = clean_text(str(row['text']))\n",
+    "        file.write(text + '\\n')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "846b6b42",
+   "metadata": {},
+   "source": [
+    "### Model kenLM"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "3c74d4be",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== 1/5 Counting and sorting n-grams ===\n",
+      "Reading /home/students/s434708/Desktop/Modelowanie Języka/challenging-america-word-gap-prediction-kenlm/processed_train.txt\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "********************************Warning: <s> appears in the input.  All instances of <s>, </s>, and <unk> will be interpreted as whitespace.\n",
+      "********************************************************************\n",
+      "Unigram tokens 135911223 types 4381594\n",
+      "=== 2/5 Calculating and sorting adjusted counts ===\n",
+      "Chain sizes: 1:52579128 2:1295655936 3:2429355008 4:3886967808 5:5668495360\n",
+      "Statistics:\n",
+      "1 4381594 D1=0.841838 D2=1.01787 D3+=1.21057\n",
+      "2 26800631 D1=0.836734 D2=1.01657 D3+=1.19437\n",
+      "3 69811700 D1=0.878562 D2=1.11227 D3+=1.27889\n",
+      "4 104063034 D1=0.931257 D2=1.23707 D3+=1.36664\n",
+      "5 119487533 D1=0.938146 D2=1.3058 D3+=1.41614\n",
+      "Memory estimate for binary LM:\n",
+      "type      MB\n",
+      "probing 6752 assuming -p 1.5\n",
+      "probing 7917 assuming -r models -p 1.5\n",
+      "trie    3572 without quantization\n",
+      "trie    2120 assuming -q 8 -b 8 quantization \n",
+      "trie    3104 assuming -a 22 array pointer compression\n",
+      "trie    1652 assuming -a 22 -q 8 -b 8 array pointer compression and quantization\n",
+      "=== 3/5 Calculating and sorting initial probabilities ===\n",
+      "Chain sizes: 1:52579128 2:428810096 3:1396234000 4:2497512816 5:3345650924\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "####################################################################################################\n",
+      "=== 4/5 Calculating and writing order-interpolated probabilities ===\n",
+      "Chain sizes: 1:52579128 2:428810096 3:1396234000 4:2497512816 5:3345650924\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "####################################################################################################\n",
+      "=== 5/5 Writing ARPA model ===\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "----------------------------------------------------------------------------------------------------Last input should have been poison.  The program should end soon with an error.  If it doesn't, there's a bug.\n",
+      "terminate called after throwing an instance of 'util::FDException'\n",
+      "  what():  /home/students/s434708/kenlm/util/file.cc:228 in void util::WriteOrThrow(int, const void*, std::size_t) threw FDException because `ret < 1'.\n",
+      "No space left on device in /home/students/s434708/Desktop/Modelowanie Języka/challenging-america-word-gap-prediction-kenlm/model.arpa while writing 8189 bytes\n",
+      "/bin/bash: line 1: 26725 Aborted                 /home/students/s434708/kenlm/build/bin/lmplz -o 5 --skip_symbols < processed_train.txt > model.arpa\n"
+     ]
+    }
+   ],
+   "source": [
+    "!$KENLM_BUILD_PATH/bin/lmplz -o 5 --skip_symbols < processed_train.txt > model.arpa"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "dc65780b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Reading model.arpa\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "****************************************************************************************************\n",
+      "/home/students/s434708/kenlm/util/file.cc:86 in int util::CreateOrThrow(const char*) threw ErrnoException because `-1 == (ret = open(name, 0100 | 01000 | 02, 0400 | 0200 | (0400 >> 3) | ((0400 >> 3) >> 3)))'.\n",
+      "No space left on device while creating model.binary Byte: 94\n",
+      "ERROR\n"
+     ]
+    }
+   ],
+   "source": [
+    "!$KENLM_BUILD_PATH/bin/build_binary model.arpa model.binary"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "2087eb80",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!rm processed_train.txt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "4ba1e592",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!rm model.arpa"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e41f7951",
+   "metadata": {},
+   "source": [
+    "### Predykcje"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6865301b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import kenlm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e32de662",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_str = 'really good'\n",
+    "\n",
+    "model = kenlm.Model('model.binary')\n",
+    "print(model.score(test_str, bos = True, eos = True))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a18b6ebd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for i in model.full_scores(test_str):\n",
+    "    print(i)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

README.md

@@ -1,2 +1,9 @@
-# challenging-america-word-gap-prediction-kenlm
+Challenging America word-gap prediction - s434708
+===================================
 
+Guess a word in a gap.
+
+Evaluation metric
+-----------------
+
+LikelihoodHashed is the metric

config.txt

@@ -0,0 +1 @@
+--metric PerplexityHashed --precision 2  --in-header in-header.tsv  --out-header out-header.tsv

in-header.tsv

@@ -0,0 +1 @@
+FileId	Year	LeftContext	RightContext

out-header.tsv

@@ -0,0 +1 @@
+Word

process_test.py

@@ -0,0 +1,25 @@
+import pandas as pd
+import csv
+import regex as re
+
+def clean_text(text):
+    text = text.lower().replace('-\\n', '').replace('\\n', ' ')
+    text = re.sub(r'\p{P}', '', text)
+
+    return text
+
+
+train_data = pd.read_csv('train/in.tsv.xz', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
+train_labels = pd.read_csv('train/expected.tsv', sep='\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)
+
+train_data = train_data[[6, 7]]
+train_data = pd.concat([train_data, train_labels], axis=1)
+
+train_data['text'] = train_data[6] + train_data[0] + train_data[7]
+train_data = train_data[['text']]
+
+with open('processed_train.txt', 'w') as file:
+    for _, row in train_data.iterrows():
+        text = clean_text(str(row['text']))
+        file.write(text + '\n')
+

run.ipynb

@@ -0,0 +1,238 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "f73a28ea",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "KENLM_BUILD_PATH='/home/students/s434708/kenlm/build'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9fc5cda3",
+   "metadata": {},
+   "source": [
+    "### Preprocessing danych"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d42ddd87",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import csv\n",
+    "import regex as re"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f84be210",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def clean_text(text):\n",
+    "    text = text.lower().replace('-\\\\n', '').replace('\\\\n', ' ')\n",
+    "    text = re.sub(r'\\p{P}', '', text)\n",
+    "\n",
+    "    return text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "de0c12d6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_data = pd.read_csv('train/in.tsv.xz', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)\n",
+    "train_labels = pd.read_csv('train/expected.tsv', sep='\\t', error_bad_lines=False, warn_bad_lines=False, header=None, quoting=csv.QUOTE_NONE)\n",
+    "\n",
+    "train_data = train_data[[6, 7]]\n",
+    "train_data = pd.concat([train_data, train_labels], axis=1)\n",
+    "\n",
+    "train_data['text'] = train_data[6] + train_data[0] + train_data[7]\n",
+    "train_data = train_data[['text']]\n",
+    "\n",
+    "with open('processed_train.txt', 'w') as file:\n",
+    "    for _, row in train_data.iterrows():\n",
+    "        text = clean_text(str(row['text']))\n",
+    "        file.write(text + '\\n')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "846b6b42",
+   "metadata": {},
+   "source": [
+    "### Model kenLM"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "3c74d4be",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== 1/5 Counting and sorting n-grams ===\n",
+      "Reading /home/students/s434708/Desktop/Modelowanie Języka/challenging-america-word-gap-prediction-kenlm/processed_train.txt\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "********************************Warning: <s> appears in the input.  All instances of <s>, </s>, and <unk> will be interpreted as whitespace.\n",
+      "********************************************************************\n",
+      "Unigram tokens 135911223 types 4381594\n",
+      "=== 2/5 Calculating and sorting adjusted counts ===\n",
+      "Chain sizes: 1:52579128 2:1295655936 3:2429355008 4:3886967808 5:5668495360\n",
+      "Statistics:\n",
+      "1 4381594 D1=0.841838 D2=1.01787 D3+=1.21057\n",
+      "2 26800631 D1=0.836734 D2=1.01657 D3+=1.19437\n",
+      "3 69811700 D1=0.878562 D2=1.11227 D3+=1.27889\n",
+      "4 104063034 D1=0.931257 D2=1.23707 D3+=1.36664\n",
+      "5 119487533 D1=0.938146 D2=1.3058 D3+=1.41614\n",
+      "Memory estimate for binary LM:\n",
+      "type      MB\n",
+      "probing 6752 assuming -p 1.5\n",
+      "probing 7917 assuming -r models -p 1.5\n",
+      "trie    3572 without quantization\n",
+      "trie    2120 assuming -q 8 -b 8 quantization \n",
+      "trie    3104 assuming -a 22 array pointer compression\n",
+      "trie    1652 assuming -a 22 -q 8 -b 8 array pointer compression and quantization\n",
+      "=== 3/5 Calculating and sorting initial probabilities ===\n",
+      "Chain sizes: 1:52579128 2:428810096 3:1396234000 4:2497512816 5:3345650924\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "####################################################################################################\n",
+      "=== 4/5 Calculating and writing order-interpolated probabilities ===\n",
+      "Chain sizes: 1:52579128 2:428810096 3:1396234000 4:2497512816 5:3345650924\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "####################################################################################################\n",
+      "=== 5/5 Writing ARPA model ===\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "----------------------------------------------------------------------------------------------------Last input should have been poison.  The program should end soon with an error.  If it doesn't, there's a bug.\n",
+      "terminate called after throwing an instance of 'util::FDException'\n",
+      "  what():  /home/students/s434708/kenlm/util/file.cc:228 in void util::WriteOrThrow(int, const void*, std::size_t) threw FDException because `ret < 1'.\n",
+      "No space left on device in /home/students/s434708/Desktop/Modelowanie Języka/challenging-america-word-gap-prediction-kenlm/model.arpa while writing 8189 bytes\n",
+      "/bin/bash: line 1: 26725 Aborted                 /home/students/s434708/kenlm/build/bin/lmplz -o 5 --skip_symbols < processed_train.txt > model.arpa\n"
+     ]
+    }
+   ],
+   "source": [
+    "!$KENLM_BUILD_PATH/bin/lmplz -o 5 --skip_symbols < processed_train.txt > model.arpa"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "dc65780b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Reading model.arpa\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "****************************************************************************************************\n",
+      "/home/students/s434708/kenlm/util/file.cc:86 in int util::CreateOrThrow(const char*) threw ErrnoException because `-1 == (ret = open(name, 0100 | 01000 | 02, 0400 | 0200 | (0400 >> 3) | ((0400 >> 3) >> 3)))'.\n",
+      "No space left on device while creating model.binary Byte: 94\n",
+      "ERROR\n"
+     ]
+    }
+   ],
+   "source": [
+    "!$KENLM_BUILD_PATH/bin/build_binary model.arpa model.binary"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "2087eb80",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!rm processed_train.txt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "4ba1e592",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!rm model.arpa"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e41f7951",
+   "metadata": {},
+   "source": [
+    "### Predykcje"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6865301b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import kenlm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e32de662",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_str = 'really good'\n",
+    "\n",
+    "model = kenlm.Model('model.binary')\n",
+    "print(model.score(test_str, bos = True, eos = True))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a18b6ebd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for i in model.full_scores(test_str):\n",
+    "    print(i)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}