246 lines
6.5 KiB
Plaintext
246 lines
6.5 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"id": "c16d72a6",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import lzma\n",
|
|
"import csv\n",
|
|
"import re\n",
|
|
"import math\n",
|
|
"from collections import Counter"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"id": "a1ff03c8",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def read_data(folder_name, test_data=False):\n",
|
|
" \n",
|
|
" all_data = lzma.open(f'{folder_name}/in.tsv.xz').read().decode('UTF-8').split('\\n')\n",
|
|
" data = [line.split('\\t') for line in all_data][:-1]\n",
|
|
" data = [[i[6].replace('\\\\n', ' '), i[7].replace('\\\\n', ' ')] for i in data]\n",
|
|
" \n",
|
|
" if not test_data:\n",
|
|
" words = []\n",
|
|
" with open(f'{folder_name}/expected.tsv') as file:\n",
|
|
" tsv_file = csv.reader(file, delimiter=\"\\t\")\n",
|
|
" for line in tsv_file:\n",
|
|
" words.append(line[0])\n",
|
|
" \n",
|
|
" return data, words\n",
|
|
" \n",
|
|
" return data"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"id": "ce522af5",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def generate_N_grams(text, ngram=1, no_punctuation=True):\n",
|
|
" text = re.sub(r'[\\-] ', '', text).lower()\n",
|
|
" if no_punctuation:\n",
|
|
" text = re.sub(r'[^\\w\\s]', ' ', text)\n",
|
|
" words=[word for word in text.split()]\n",
|
|
" temp=zip(*[words[i:] for i in range(0,ngram)])\n",
|
|
" ans=[' '.join(ngram) for ngram in temp]\n",
|
|
" return ans"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"id": "317ade72",
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"def check_prob(N_grams):\n",
|
|
" if ' ' not in N_grams[0]:\n",
|
|
" counter = Counter()\n",
|
|
" a = Counter(N_grams)\n",
|
|
" total = sum(a.values())\n",
|
|
" return {k: v / total for total in (sum(a.values()),) for k, v in a.items()}\n",
|
|
" count = {}\n",
|
|
" for i in N_grams:\n",
|
|
" i = i.rsplit(maxsplit=1)\n",
|
|
" if i[0] in count:\n",
|
|
" if i[1] in count[i[0]]:\n",
|
|
" count[i[0]][i[1]] += 1\n",
|
|
" else:\n",
|
|
" count[i[0]][i[1]] = 1\n",
|
|
" else:\n",
|
|
" count[i[0]] = {i[1]: 1}\n",
|
|
" \n",
|
|
" for word in count:\n",
|
|
" s = sum(count[word].values())\n",
|
|
" for i in count[word]:\n",
|
|
" count[word][i] = count[word][i] / s\n",
|
|
" count[word] = sorted(count[word].items(), key=lambda x: x[1], reverse=True)\n",
|
|
" \n",
|
|
" return count"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"id": "86aeda02",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def find_word(words, model):\n",
|
|
" n = len(words)\n",
|
|
" tmp = {}\n",
|
|
" while n > 1:\n",
|
|
" if ' '.join(words[-n:]) in model[n]:\n",
|
|
" tmp = model[n][' '.join(words[-n:])][:2]\n",
|
|
" break\n",
|
|
" else:\n",
|
|
" n -= 1\n",
|
|
" \n",
|
|
" res = ' '.join([i[0] + ':' + str(i[1]) for i in tmp])\n",
|
|
" s = 1 - sum(n for _, n in tmp)\n",
|
|
" if s == 0:\n",
|
|
" s = 1\n",
|
|
" res += ' :' + str(s)\n",
|
|
" if tmp == {}:\n",
|
|
" if words[-1] in model[0]:\n",
|
|
" return f'{words[-1]}:{model[0][words[-1]]} :{1 - model[0][words[-1]]}'\n",
|
|
" else:\n",
|
|
" return ':1'\n",
|
|
" return res"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"id": "3b713dc3",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def find_words(data, n, model):\n",
|
|
" found_words = []\n",
|
|
" for i in data:\n",
|
|
" t = i[0]\n",
|
|
" t = re.sub(r'[\\-] ', '', t).lower()\n",
|
|
" if True:\n",
|
|
" t = re.sub(r'[^\\w\\s]', ' ', t)\n",
|
|
" words=[word for word in t.split()]\n",
|
|
" found_words.append(find_word(words[-n:], model))\n",
|
|
" return found_words"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"id": "17be7468",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def save_data(folder, words):\n",
|
|
" f = open(f'{folder}/out.tsv', 'w')\n",
|
|
" f.write('\\n'.join(words) + '\\n')\n",
|
|
" f.close()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"id": "8c127bae",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def train(n, data_size = 5000):\n",
|
|
" train_data, train_words = read_data('train')\n",
|
|
" N_grams = [[] for i in range(n)]\n",
|
|
" probs = [[] for i in range(n)]\n",
|
|
" for i in range(len(train_data[:data_size])):\n",
|
|
" for j in range(n):\n",
|
|
" N_grams[j] += generate_N_grams(f'{train_data[i][0]} {train_words[i]} {train_data[i][1]}', j + 1)\n",
|
|
" for i in range(n):\n",
|
|
" probs[i] = check_prob(N_grams[i])\n",
|
|
" return probs\n",
|
|
" \n",
|
|
"model = train(4)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"id": "935c0f87",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def predict(model, n, data_name, test_data=False):\n",
|
|
" if not test_data:\n",
|
|
" data, _ = read_data(data_name, test_data)\n",
|
|
" else:\n",
|
|
" data = read_data(data_name, test_data)\n",
|
|
" found_words = find_words(data, n - 1, model)\n",
|
|
" save_data(data_name, found_words)\n",
|
|
" \n",
|
|
"predict(model, 4, 'dev-0')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"id": "e43fd5b3",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"794.13\r\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"!./geval -t dev-0"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"id": "b2e52242",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"predict(model, 4, 'test-A', True)"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.9.5"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|