{
"cells": [
{
"cell_type": "code",
"execution_count": 27,
"id": "8b023ab4",
"metadata": {},
"outputs": [],
"source": [
"train_file ='train/in.tsv.xz'\n",
"test_file = 'test-A/in.tsv.xz'\n",
"out_file = 'test-A/out.tsv'"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "39b223cf",
"metadata": {},
"outputs": [],
"source": [
"from itertools import islice\n",
"import regex as re\n",
"import sys\n",
"from torchtext.vocab import build_vocab_from_iterator\n",
"import lzma\n",
"import pickle\n",
"import re\n",
"import torch\n",
"from torch import nn\n",
"from torch.utils.data import IterableDataset\n",
"import itertools\n",
"from torch.utils.data import DataLoader\n",
"import yaml"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "a0b0b73e",
"metadata": {},
"outputs": [],
"source": [
"epochs = 3\n",
"embed_size = 200\n",
"device = 'cuda'\n",
"vocab_size = 30000\n",
"batch_s = 1600\n",
"learning_rate = 0.01\n",
"k = 20 #top k words\n",
"wildcard_minweight = 0.02"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "9668da9f",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/tmp/ipykernel_30815/1141171476.py:1: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n",
" params = yaml.load(open('config/params.yaml'))\n"
]
}
],
"source": [
"params = yaml.load(open('config/params.yaml'))\n",
"#then, entire code should go about those params with params[epochs] etc"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "deb31dbb",
"metadata": {},
"outputs": [],
"source": [
"params = {\n",
"'epochs': 3,\n",
"'embed_size': 300,\n",
"'device': 'cuda',\n",
"'vocab_size': 30000,\n",
"'batch_size': 3200,\n",
"'learning_rate': 0.001,\n",
"'k': 15, #top k words\n",
"'wildcard_minweight': 0.1\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "01a6cf33",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'epochs': 3,\n",
" 'embed_size': 300,\n",
" 'device': 'cuda',\n",
" 'vocab_size': 30000,\n",
" 'batch_size': 3200,\n",
" 'learning_rate': 0.001,\n",
" 'k': 15,\n",
" 'wildcard_minweight': 0.1}"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"params"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "7526e30c",
"metadata": {},
"outputs": [],
"source": [
"def get_words_from_line(line):\n",
" line = line.rstrip()\n",
" yield ''\n",
" line = preprocess(line)\n",
" for t in line.split(' '):\n",
" yield t\n",
" yield ''\n",
"\n",
"\n",
"def get_word_lines_from_file(file_name):\n",
" n = 0\n",
" with lzma.open(file_name, 'r') as fh:\n",
" for line in fh:\n",
" n+=1\n",
" if n%1000==0:\n",
" print(n)\n",
" yield get_words_from_line(line.decode('utf-8'))\n"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "db7a1737",
"metadata": {},
"outputs": [],
"source": [
"###preprocessing\n",
"def preprocess(line):\n",
" line = get_rid_of_header(line)\n",
" line = replace_endline(line)\n",
" return line\n",
"\n",
"def get_rid_of_header(line):\n",
" line = line.split('\\t')[6:]\n",
" return \" \".join(line)\n",
" \n",
"def replace_endline(line):\n",
" line = line.replace(\"\\\\n\", \" \")\n",
" return line"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "01cde371",
"metadata": {},
"outputs": [],
"source": [
"def look_ahead_iterator(gen):\n",
" prev2 = None\n",
" prev1 = None\n",
" for item in gen:\n",
" if prev2 is not None and prev1 is not None:\n",
" yield (prev2, prev1, item)\n",
" prev2 = prev1\n",
" prev1 = item\n",
"\n",
"class Trigrams(IterableDataset):\n",
" def __init__(self, text_file, vocabulary_size):\n",
" self.vocab = build_vocab_from_iterator(\n",
" get_word_lines_from_file(text_file),\n",
" max_tokens = vocabulary_size,\n",
" specials = [''])\n",
" self.vocab.set_default_index(self.vocab[''])\n",
" self.vocabulary_size = vocabulary_size\n",
" self.text_file = text_file\n",
"\n",
" def __iter__(self):\n",
" return look_ahead_iterator(\n",
" (self.vocab[t] for t in itertools.chain.from_iterable(get_word_lines_from_file(self.text_file))))\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "6be87c51",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"Vocab()"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": []
},
{
"cell_type": "code",
"execution_count": 23,
"id": "198b1dd3",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1000\n",
"2000\n",
"3000\n",
"4000\n",
"5000\n",
"6000\n",
"7000\n",
"8000\n",
"9000\n",
"10000\n",
"11000\n",
"12000\n",
"13000\n",
"14000\n",
"15000\n",
"16000\n",
"17000\n",
"18000\n",
"19000\n",
"20000\n",
"21000\n",
"22000\n",
"23000\n",
"24000\n",
"25000\n",
"26000\n",
"27000\n",
"28000\n",
"29000\n",
"30000\n",
"31000\n",
"32000\n",
"33000\n",
"34000\n",
"35000\n",
"36000\n",
"37000\n",
"38000\n",
"39000\n",
"40000\n",
"41000\n",
"42000\n",
"43000\n",
"44000\n",
"45000\n",
"46000\n",
"47000\n",
"48000\n",
"49000\n",
"50000\n",
"51000\n",
"52000\n",
"53000\n",
"54000\n",
"55000\n",
"56000\n",
"57000\n",
"58000\n",
"59000\n",
"60000\n",
"61000\n",
"62000\n",
"63000\n",
"64000\n",
"65000\n",
"66000\n",
"67000\n",
"68000\n",
"69000\n",
"70000\n",
"71000\n",
"72000\n",
"73000\n",
"74000\n",
"75000\n",
"76000\n",
"77000\n",
"78000\n",
"79000\n",
"80000\n",
"81000\n",
"82000\n",
"83000\n",
"84000\n",
"85000\n",
"86000\n",
"87000\n",
"88000\n",
"89000\n",
"90000\n",
"91000\n",
"92000\n",
"93000\n",
"94000\n",
"95000\n",
"96000\n",
"97000\n",
"98000\n",
"99000\n",
"100000\n",
"101000\n",
"102000\n",
"103000\n",
"104000\n",
"105000\n",
"106000\n",
"107000\n",
"108000\n",
"109000\n",
"110000\n",
"111000\n",
"112000\n",
"113000\n",
"114000\n",
"115000\n",
"116000\n",
"117000\n",
"118000\n",
"119000\n",
"120000\n",
"121000\n",
"122000\n",
"123000\n",
"124000\n",
"125000\n",
"126000\n",
"127000\n",
"128000\n",
"129000\n",
"130000\n",
"131000\n",
"132000\n",
"133000\n",
"134000\n",
"135000\n",
"136000\n",
"137000\n",
"138000\n",
"139000\n",
"140000\n",
"141000\n",
"142000\n",
"143000\n",
"144000\n",
"145000\n",
"146000\n",
"147000\n",
"148000\n",
"149000\n",
"150000\n",
"151000\n",
"152000\n",
"153000\n",
"154000\n",
"155000\n",
"156000\n",
"157000\n",
"158000\n",
"159000\n",
"160000\n",
"161000\n",
"162000\n",
"163000\n",
"164000\n",
"165000\n",
"166000\n",
"167000\n",
"168000\n",
"169000\n",
"170000\n",
"171000\n",
"172000\n",
"173000\n",
"174000\n",
"175000\n",
"176000\n",
"177000\n",
"178000\n",
"179000\n",
"180000\n",
"181000\n",
"182000\n",
"183000\n",
"184000\n",
"185000\n",
"186000\n",
"187000\n",
"188000\n",
"189000\n",
"190000\n",
"191000\n",
"192000\n",
"193000\n",
"194000\n",
"195000\n",
"196000\n",
"197000\n",
"198000\n",
"199000\n",
"200000\n",
"201000\n",
"202000\n",
"203000\n",
"204000\n",
"205000\n",
"206000\n",
"207000\n",
"208000\n",
"209000\n",
"210000\n",
"211000\n",
"212000\n",
"213000\n",
"214000\n",
"215000\n",
"216000\n",
"217000\n",
"218000\n",
"219000\n",
"220000\n",
"221000\n",
"222000\n",
"223000\n",
"224000\n",
"225000\n",
"226000\n",
"227000\n",
"228000\n",
"229000\n",
"230000\n",
"231000\n",
"232000\n",
"233000\n",
"234000\n",
"235000\n",
"236000\n",
"237000\n",
"238000\n",
"239000\n",
"240000\n",
"241000\n",
"242000\n",
"243000\n",
"244000\n",
"245000\n",
"246000\n",
"247000\n",
"248000\n",
"249000\n",
"250000\n",
"251000\n",
"252000\n",
"253000\n",
"254000\n",
"255000\n",
"256000\n",
"257000\n",
"258000\n",
"259000\n",
"260000\n",
"261000\n",
"262000\n",
"263000\n",
"264000\n",
"265000\n",
"266000\n",
"267000\n",
"268000\n",
"269000\n",
"270000\n",
"271000\n",
"272000\n",
"273000\n",
"274000\n",
"275000\n",
"276000\n",
"277000\n",
"278000\n",
"279000\n",
"280000\n",
"281000\n",
"282000\n",
"283000\n",
"284000\n",
"285000\n",
"286000\n",
"287000\n",
"288000\n",
"289000\n",
"290000\n",
"291000\n",
"292000\n",
"293000\n",
"294000\n",
"295000\n",
"296000\n",
"297000\n",
"298000\n",
"299000\n",
"300000\n",
"301000\n",
"302000\n",
"303000\n",
"304000\n",
"305000\n",
"306000\n",
"307000\n",
"308000\n",
"309000\n",
"310000\n",
"311000\n",
"312000\n",
"313000\n",
"314000\n",
"315000\n",
"316000\n",
"317000\n",
"318000\n",
"319000\n",
"320000\n",
"321000\n",
"322000\n",
"323000\n",
"324000\n",
"325000\n",
"326000\n",
"327000\n",
"328000\n",
"329000\n",
"330000\n",
"331000\n",
"332000\n",
"333000\n",
"334000\n",
"335000\n",
"336000\n",
"337000\n",
"338000\n",
"339000\n",
"340000\n",
"341000\n",
"342000\n",
"343000\n",
"344000\n",
"345000\n",
"346000\n",
"347000\n",
"348000\n",
"349000\n",
"350000\n",
"351000\n",
"352000\n",
"353000\n",
"354000\n",
"355000\n",
"356000\n",
"357000\n",
"358000\n",
"359000\n",
"360000\n",
"361000\n",
"362000\n",
"363000\n",
"364000\n",
"365000\n",
"366000\n",
"367000\n",
"368000\n",
"369000\n",
"370000\n",
"371000\n",
"372000\n",
"373000\n",
"374000\n",
"375000\n",
"376000\n",
"377000\n",
"378000\n",
"379000\n",
"380000\n",
"381000\n",
"382000\n",
"383000\n",
"384000\n",
"385000\n",
"386000\n",
"387000\n",
"388000\n",
"389000\n",
"390000\n",
"391000\n",
"392000\n",
"393000\n",
"394000\n",
"395000\n",
"396000\n",
"397000\n",
"398000\n",
"399000\n",
"400000\n",
"401000\n",
"402000\n",
"403000\n",
"404000\n",
"405000\n",
"406000\n",
"407000\n",
"408000\n",
"409000\n",
"410000\n",
"411000\n",
"412000\n",
"413000\n",
"414000\n",
"415000\n",
"416000\n",
"417000\n",
"418000\n",
"419000\n",
"420000\n",
"421000\n",
"422000\n",
"423000\n",
"424000\n",
"425000\n",
"426000\n",
"427000\n",
"428000\n",
"429000\n",
"430000\n",
"431000\n",
"432000\n"
]
}
],
"source": [
"vocab = build_vocab_from_iterator(\n",
" get_word_lines_from_file(train_file),\n",
" max_tokens = params['vocab_size'],\n",
" specials = [''])"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "6136fbb9",
"metadata": {},
"outputs": [],
"source": [
"with open('filename.pickle', 'wb') as handle:\n",
" pickle.dump(vocab, handle, protocol=pickle.HIGHEST_PROTOCOL)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "30a5b26b",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1000\n",
"2000\n",
"3000\n",
"4000\n",
"5000\n",
"6000\n",
"7000\n",
"8000\n",
"9000\n",
"10000\n",
"11000\n",
"12000\n",
"13000\n",
"14000\n",
"15000\n",
"16000\n",
"17000\n",
"18000\n",
"19000\n",
"20000\n",
"21000\n",
"22000\n",
"23000\n",
"24000\n",
"25000\n",
"26000\n",
"27000\n",
"28000\n",
"29000\n",
"30000\n",
"31000\n",
"32000\n",
"33000\n",
"34000\n",
"35000\n",
"36000\n",
"37000\n",
"38000\n",
"39000\n",
"40000\n",
"41000\n",
"42000\n",
"43000\n",
"44000\n",
"45000\n",
"46000\n",
"47000\n",
"48000\n",
"49000\n",
"50000\n",
"51000\n",
"52000\n",
"53000\n",
"54000\n",
"55000\n",
"56000\n",
"57000\n",
"58000\n",
"59000\n",
"60000\n",
"61000\n",
"62000\n",
"63000\n",
"64000\n",
"65000\n",
"66000\n",
"67000\n",
"68000\n",
"69000\n",
"70000\n",
"71000\n",
"72000\n",
"73000\n",
"74000\n",
"75000\n",
"76000\n",
"77000\n",
"78000\n",
"79000\n",
"80000\n",
"81000\n",
"82000\n",
"83000\n",
"84000\n",
"85000\n",
"86000\n",
"87000\n",
"88000\n",
"89000\n",
"90000\n",
"91000\n",
"92000\n",
"93000\n",
"94000\n",
"95000\n",
"96000\n",
"97000\n",
"98000\n",
"99000\n",
"100000\n",
"101000\n",
"102000\n",
"103000\n",
"104000\n",
"105000\n",
"106000\n",
"107000\n",
"108000\n",
"109000\n",
"110000\n",
"111000\n",
"112000\n",
"113000\n",
"114000\n",
"115000\n",
"116000\n",
"117000\n",
"118000\n",
"119000\n",
"120000\n",
"121000\n",
"122000\n",
"123000\n",
"124000\n",
"125000\n",
"126000\n",
"127000\n",
"128000\n",
"129000\n",
"130000\n",
"131000\n",
"132000\n",
"133000\n",
"134000\n",
"135000\n",
"136000\n",
"137000\n",
"138000\n",
"139000\n",
"140000\n",
"141000\n",
"142000\n",
"143000\n",
"144000\n",
"145000\n",
"146000\n",
"147000\n",
"148000\n",
"149000\n",
"150000\n",
"151000\n",
"152000\n",
"153000\n",
"154000\n",
"155000\n",
"156000\n",
"157000\n",
"158000\n",
"159000\n",
"160000\n",
"161000\n",
"162000\n",
"163000\n",
"164000\n",
"165000\n",
"166000\n",
"167000\n",
"168000\n",
"169000\n",
"170000\n",
"171000\n",
"172000\n",
"173000\n",
"174000\n",
"175000\n",
"176000\n",
"177000\n",
"178000\n",
"179000\n",
"180000\n",
"181000\n",
"182000\n",
"183000\n",
"184000\n",
"185000\n",
"186000\n",
"187000\n",
"188000\n",
"189000\n",
"190000\n",
"191000\n",
"192000\n",
"193000\n",
"194000\n",
"195000\n",
"196000\n",
"197000\n",
"198000\n",
"199000\n",
"200000\n",
"201000\n",
"202000\n",
"203000\n",
"204000\n",
"205000\n",
"206000\n",
"207000\n",
"208000\n",
"209000\n",
"210000\n",
"211000\n",
"212000\n",
"213000\n",
"214000\n",
"215000\n",
"216000\n",
"217000\n",
"218000\n",
"219000\n",
"220000\n",
"221000\n",
"222000\n",
"223000\n",
"224000\n",
"225000\n",
"226000\n",
"227000\n",
"228000\n",
"229000\n",
"230000\n",
"231000\n",
"232000\n",
"233000\n",
"234000\n",
"235000\n",
"236000\n",
"237000\n",
"238000\n",
"239000\n",
"240000\n",
"241000\n",
"242000\n",
"243000\n",
"244000\n",
"245000\n",
"246000\n",
"247000\n",
"248000\n",
"249000\n",
"250000\n",
"251000\n",
"252000\n",
"253000\n",
"254000\n",
"255000\n",
"256000\n",
"257000\n",
"258000\n",
"259000\n",
"260000\n",
"261000\n",
"262000\n",
"263000\n",
"264000\n",
"265000\n",
"266000\n",
"267000\n",
"268000\n",
"269000\n",
"270000\n",
"271000\n",
"272000\n",
"273000\n",
"274000\n",
"275000\n",
"276000\n",
"277000\n",
"278000\n",
"279000\n",
"280000\n",
"281000\n",
"282000\n",
"283000\n",
"284000\n",
"285000\n",
"286000\n",
"287000\n",
"288000\n",
"289000\n",
"290000\n",
"291000\n",
"292000\n",
"293000\n",
"294000\n",
"295000\n",
"296000\n",
"297000\n",
"298000\n",
"299000\n",
"300000\n",
"301000\n",
"302000\n",
"303000\n",
"304000\n",
"305000\n",
"306000\n",
"307000\n",
"308000\n",
"309000\n",
"310000\n",
"311000\n",
"312000\n",
"313000\n",
"314000\n",
"315000\n",
"316000\n",
"317000\n",
"318000\n",
"319000\n",
"320000\n",
"321000\n",
"322000\n",
"323000\n",
"324000\n",
"325000\n",
"326000\n",
"327000\n",
"328000\n",
"329000\n",
"330000\n",
"331000\n",
"332000\n",
"333000\n",
"334000\n",
"335000\n",
"336000\n",
"337000\n",
"338000\n",
"339000\n",
"340000\n",
"341000\n",
"342000\n",
"343000\n",
"344000\n",
"345000\n",
"346000\n",
"347000\n",
"348000\n",
"349000\n",
"350000\n",
"351000\n",
"352000\n",
"353000\n",
"354000\n",
"355000\n",
"356000\n",
"357000\n",
"358000\n",
"359000\n",
"360000\n",
"361000\n",
"362000\n",
"363000\n",
"364000\n",
"365000\n",
"366000\n",
"367000\n",
"368000\n",
"369000\n",
"370000\n",
"371000\n",
"372000\n",
"373000\n",
"374000\n",
"375000\n",
"376000\n",
"377000\n",
"378000\n",
"379000\n",
"380000\n",
"381000\n",
"382000\n",
"383000\n",
"384000\n",
"385000\n",
"386000\n",
"387000\n",
"388000\n",
"389000\n",
"390000\n",
"391000\n",
"392000\n",
"393000\n",
"394000\n",
"395000\n",
"396000\n",
"397000\n",
"398000\n",
"399000\n",
"400000\n",
"401000\n",
"402000\n",
"403000\n",
"404000\n",
"405000\n",
"406000\n",
"407000\n",
"408000\n",
"409000\n",
"410000\n",
"411000\n",
"412000\n",
"413000\n",
"414000\n",
"415000\n",
"416000\n",
"417000\n",
"418000\n",
"419000\n",
"420000\n",
"421000\n",
"422000\n",
"423000\n",
"424000\n",
"425000\n",
"426000\n",
"427000\n",
"428000\n",
"429000\n",
"430000\n",
"431000\n",
"432000\n"
]
}
],
"source": [
"with open('filename.pickle','rb') as handle:\n",
" vocab = pickle.load(handle)\n",
" \n",
"train_dataset = Trigrams(train_file, params['vocab_size'])"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "eaa681b4",
"metadata": {},
"outputs": [],
"source": [
"data = DataLoader(train_dataset, batch_size=params['batch_size']) #load data "
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "c80b004e",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"print(type(data))"
]
},
{
"cell_type": "code",
"execution_count": 33,
"id": "3aea0574",
"metadata": {},
"outputs": [],
"source": [
"class SimpleTrigramNeuralLanguageModel(nn.Module):\n",
" def __init__(self, vocabulary_size, embedding_size):\n",
" super(SimpleTrigramNeuralLanguageModel, self).__init__()\n",
" self.embeddings = nn.Embedding(vocabulary_size, embedding_size)\n",
" self.linear = nn.Linear(2*embedding_size, vocabulary_size)\n",
" self.linear_matrix_2 = nn.Linear(embedding_size*2, embedding_size*2)\n",
" self.relu = nn.ReLU()\n",
" self.softmax = nn.Softmax()\n",
" \n",
" #for each word in vocabulary theres a separate embedding vector, consisting of embedding_size entries\n",
" #self.linear is linear layer consisting of concatenated embeddings of left, and right context words\n",
" #self.linear_matrix_2 is linear layer \n",
" \n",
" def forward(self, x): #x is list of prev and following embeddings\n",
" emb_left = self.embeddings(x[0])\n",
" emb_right = self.embeddings(x[1])\n",
" #create two embeddings vectors, for word before and after, respectively\n",
" \n",
" first_layer_size_2 = self.linear_matrix_2(torch.cat((emb_left, emb_right), dim=1))\n",
" first_relu = self.relu(first_layer_size_2)\n",
" concated = self.linear(first_relu)\n",
" out = self.softmax(concated)\n",
" return out"
]
},
{
"cell_type": "code",
"execution_count": 65,
"id": "e4757295",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0"
]
},
"execution_count": 65,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import gc\n",
"torch.cuda.empty_cache()\n",
"gc.collect()"
]
},
{
"cell_type": "code",
"execution_count": 39,
"id": "0a41831e",
"metadata": {},
"outputs": [],
"source": [
"device = 'cuda'\n",
"model = SimpleTrigramNeuralLanguageModel(params['vocab_size'], params['embed_size']).to(device)\n",
"optimizer = torch.optim.Adam(model.parameters(), lr=params['learning_rate'])\n",
"criterion = torch.nn.NLLLoss()"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "281b9010",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"epoch: = 0\n",
"0 "
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/tmp/ipykernel_30815/606935597.py:22: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
" out = self.softmax(concated)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"tensor(10.3473, device='cuda:0', grad_fn=)\n",
"1000\n",
"100 tensor(6.4702, device='cuda:0', grad_fn=)\n",
"200 tensor(6.1728, device='cuda:0', grad_fn=)\n",
"2000\n",
"300 tensor(6.1315, device='cuda:0', grad_fn=)\n",
"3000\n",
"400 tensor(6.1413, device='cuda:0', grad_fn=)\n",
"4000\n",
"500 tensor(5.9379, device='cuda:0', grad_fn=)\n",
"5000\n",
"600 tensor(6.0123, device='cuda:0', grad_fn=)\n",
"6000\n",
"700 tensor(6.0021, device='cuda:0', grad_fn=)\n",
"7000\n",
"800 tensor(5.8060, device='cuda:0', grad_fn=)\n",
"8000\n",
"900 tensor(5.8374, device='cuda:0', grad_fn=)\n",
"9000\n",
"1000 tensor(5.7561, device='cuda:0', grad_fn=)\n",
"10000\n",
"1100 tensor(5.7806, device='cuda:0', grad_fn=)\n",
"11000\n",
"1200 tensor(5.9041, device='cuda:0', grad_fn=)\n",
"12000\n",
"1300 tensor(5.8457, device='cuda:0', grad_fn=)\n",
"13000\n",
"1400 tensor(5.9296, device='cuda:0', grad_fn=)\n",
"14000\n",
"1500 tensor(5.9857, device='cuda:0', grad_fn=)\n",
"15000\n",
"1600 tensor(5.6528, device='cuda:0', grad_fn=)\n",
"16000\n",
"1700 tensor(5.7602, device='cuda:0', grad_fn=)\n",
"17000\n",
"1800 tensor(5.5709, device='cuda:0', grad_fn=)\n",
"18000\n",
"1900 tensor(5.6301, device='cuda:0', grad_fn=)\n",
"19000\n",
"2000 tensor(5.7465, device='cuda:0', grad_fn=)\n",
"20000\n",
"2100 tensor(5.5858, device='cuda:0', grad_fn=)\n",
"21000\n",
"2200 tensor(5.7398, device='cuda:0', grad_fn=)\n",
"22000\n",
"2300 tensor(5.5739, device='cuda:0', grad_fn=)\n",
"23000\n",
"2400 tensor(5.7891, device='cuda:0', grad_fn=)\n",
"24000\n",
"2500 tensor(5.3979, device='cuda:0', grad_fn=)\n",
"25000\n",
"2600 tensor(5.6079, device='cuda:0', grad_fn=)\n",
"26000\n",
"2700 tensor(5.4694, device='cuda:0', grad_fn=)\n",
"27000\n",
"2800 tensor(5.4906, device='cuda:0', grad_fn=)\n",
"28000\n",
"2900 tensor(5.7360, device='cuda:0', grad_fn=)\n",
"29000\n",
"3000 tensor(5.6648, device='cuda:0', grad_fn=)\n",
"30000\n",
"3100 tensor(5.4550, device='cuda:0', grad_fn=)\n",
"31000\n",
"3200 tensor(5.5493, device='cuda:0', grad_fn=)\n",
"32000\n",
"3300 tensor(5.4989, device='cuda:0', grad_fn=)\n",
"33000\n",
"3400 tensor(5.4791, device='cuda:0', grad_fn=)\n",
"34000\n",
"3500 tensor(5.5922, device='cuda:0', grad_fn=)\n",
"35000\n",
"3600 tensor(5.6143, device='cuda:0', grad_fn=)\n",
"36000\n",
"3700 tensor(5.6245, device='cuda:0', grad_fn=)\n",
"37000\n",
"3800 tensor(5.6404, device='cuda:0', grad_fn=)\n",
"38000\n",
"3900 tensor(5.4369, device='cuda:0', grad_fn=)\n",
"39000\n",
"4000 tensor(5.6512, device='cuda:0', grad_fn=)\n",
"40000\n",
"4100 tensor(5.4947, device='cuda:0', grad_fn=)\n",
"41000\n",
"4200 tensor(5.4070, device='cuda:0', grad_fn=)\n",
"42000\n",
"4300 tensor(5.2637, device='cuda:0', grad_fn=)\n",
"43000\n",
"4400 tensor(5.5091, device='cuda:0', grad_fn=)\n",
"44000\n",
"4500 tensor(5.5042, device='cuda:0', grad_fn=)\n",
"45000\n",
"4600 tensor(5.4784, device='cuda:0', grad_fn=)\n",
"46000\n",
"4700 tensor(5.3096, device='cuda:0', grad_fn=)\n",
"47000\n",
"4800 tensor(5.7407, device='cuda:0', grad_fn=)\n",
"48000\n",
"4900 tensor(5.4874, device='cuda:0', grad_fn=)\n",
"49000\n",
"5000 tensor(5.4813, device='cuda:0', grad_fn=)\n",
"50000\n",
"5100 tensor(5.6151, device='cuda:0', grad_fn=)\n",
"51000\n",
"5200 tensor(5.4652, device='cuda:0', grad_fn=)\n",
"52000\n",
"5300 tensor(5.7041, device='cuda:0', grad_fn=)\n",
"53000\n",
"5400 tensor(5.5943, device='cuda:0', grad_fn=)\n",
"54000\n",
"5500 tensor(5.5253, device='cuda:0', grad_fn=)\n",
"55000\n",
"5600 tensor(5.2634, device='cuda:0', grad_fn=)\n",
"56000\n",
"5700 tensor(5.3914, device='cuda:0', grad_fn=)\n",
"57000\n",
"5800 tensor(5.4835, device='cuda:0', grad_fn=)\n",
"58000\n",
"5900 tensor(5.4290, device='cuda:0', grad_fn=)\n",
"59000\n",
"6000 tensor(5.4881, device='cuda:0', grad_fn=)\n",
"60000\n",
"6100 tensor(5.3520, device='cuda:0', grad_fn=)\n",
"61000\n",
"6200 tensor(5.3854, device='cuda:0', grad_fn=)\n",
"62000\n",
"6300 tensor(5.5601, device='cuda:0', grad_fn=)\n",
"63000\n",
"6400 tensor(5.3025, device='cuda:0', grad_fn=)\n",
"64000\n",
"6500 tensor(5.3010, device='cuda:0', grad_fn=)\n",
"65000\n",
"6600 tensor(5.3221, device='cuda:0', grad_fn=)\n",
"66000\n",
"6700 tensor(5.3945, device='cuda:0', grad_fn=)\n",
"67000\n",
"6800 tensor(5.2509, device='cuda:0', grad_fn=)\n",
"68000\n",
"6900 tensor(5.4767, device='cuda:0', grad_fn=)\n",
"69000\n",
"7000 tensor(5.5083, device='cuda:0', grad_fn=)\n",
"70000\n",
"7100 tensor(5.5266, device='cuda:0', grad_fn=)\n",
"71000\n",
"7200 tensor(5.2670, device='cuda:0', grad_fn=)\n",
"72000\n",
"7300 tensor(5.3103, device='cuda:0', grad_fn=)\n",
"73000\n",
"7400 tensor(5.4863, device='cuda:0', grad_fn=)\n",
"74000\n",
"7500 tensor(5.2700, device='cuda:0', grad_fn=)\n",
"7600 tensor(5.3116, device='cuda:0', grad_fn=)\n",
"75000\n",
"7700 tensor(5.2636, device='cuda:0', grad_fn=)\n",
"76000\n",
"7800 tensor(5.3873, device='cuda:0', grad_fn=)\n",
"77000\n",
"7900 tensor(5.3230, device='cuda:0', grad_fn=)\n",
"78000\n",
"8000 tensor(5.3560, device='cuda:0', grad_fn=)\n",
"79000\n",
"8100 tensor(4.9302, device='cuda:0', grad_fn=)\n",
"80000\n",
"8200 tensor(5.4023, device='cuda:0', grad_fn=)\n",
"81000\n",
"8300 tensor(5.5185, device='cuda:0', grad_fn=)\n",
"82000\n",
"8400 tensor(5.4327, device='cuda:0', grad_fn=)\n",
"83000\n",
"8500 tensor(5.2833, device='cuda:0', grad_fn=)\n",
"84000\n",
"8600 tensor(5.2445, device='cuda:0', grad_fn=)\n",
"85000\n",
"8700 tensor(5.5645, device='cuda:0', grad_fn=)\n",
"86000\n",
"8800 tensor(5.4786, device='cuda:0', grad_fn=)\n",
"87000\n",
"8900 tensor(5.5328, device='cuda:0', grad_fn=)\n",
"88000\n",
"9000 tensor(5.4828, device='cuda:0', grad_fn=)\n",
"89000\n",
"9100 tensor(5.4731, device='cuda:0', grad_fn=)\n",
"90000\n",
"9200 tensor(5.3773, device='cuda:0', grad_fn=)\n",
"91000\n",
"9300 tensor(5.5787, device='cuda:0', grad_fn=)\n",
"92000\n",
"9400 tensor(5.3616, device='cuda:0', grad_fn=)\n",
"93000\n",
"9500 tensor(5.1018, device='cuda:0', grad_fn=)\n",
"94000\n",
"9600 tensor(5.3579, device='cuda:0', grad_fn=)\n",
"95000\n",
"9700 tensor(5.1421, device='cuda:0', grad_fn=)\n",
"96000\n",
"9800 tensor(5.4230, device='cuda:0', grad_fn=)\n",
"97000\n",
"9900 tensor(5.1386, device='cuda:0', grad_fn=)\n",
"98000\n",
"10000 tensor(5.4389, device='cuda:0', grad_fn=)\n",
"99000\n",
"10100 tensor(5.4342, device='cuda:0', grad_fn=)\n",
"100000\n",
"10200 tensor(5.2866, device='cuda:0', grad_fn=)\n",
"101000\n",
"10300 tensor(5.4027, device='cuda:0', grad_fn=)\n",
"102000\n",
"10400 tensor(5.4081, device='cuda:0', grad_fn=)\n",
"103000\n",
"10500 tensor(5.3720, device='cuda:0', grad_fn=)\n",
"104000\n",
"10600 tensor(5.5047, device='cuda:0', grad_fn=)\n",
"105000\n",
"10700 tensor(5.3374, device='cuda:0', grad_fn=)\n",
"106000\n",
"10800 tensor(5.2147, device='cuda:0', grad_fn=)\n",
"107000\n",
"10900 tensor(5.3578, device='cuda:0', grad_fn=)\n",
"108000\n",
"11000 tensor(5.5004, device='cuda:0', grad_fn=)\n",
"109000\n",
"11100 tensor(5.4427, device='cuda:0', grad_fn=)\n",
"110000\n",
"11200 tensor(5.2546, device='cuda:0', grad_fn=)\n",
"111000\n",
"11300 tensor(5.1242, device='cuda:0', grad_fn=)\n",
"112000\n",
"11400 tensor(5.3266, device='cuda:0', grad_fn=)\n",
"113000\n",
"11500 tensor(5.4107, device='cuda:0', grad_fn=)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"114000\n",
"11600 tensor(5.2242, device='cuda:0', grad_fn=)\n",
"115000\n",
"11700 tensor(5.4189, device='cuda:0', grad_fn=)\n",
"116000\n",
"11800 tensor(5.4812, device='cuda:0', grad_fn=)\n",
"117000\n",
"11900 tensor(5.5100, device='cuda:0', grad_fn=)\n",
"118000\n",
"12000 tensor(5.4680, device='cuda:0', grad_fn=)\n",
"119000\n",
"12100 tensor(5.4753, device='cuda:0', grad_fn=)\n",
"120000\n",
"12200 tensor(5.3741, device='cuda:0', grad_fn=)\n",
"121000\n",
"12300 tensor(5.2679, device='cuda:0', grad_fn=)\n",
"122000\n",
"12400 tensor(5.4868, device='cuda:0', grad_fn=)\n",
"123000\n",
"12500 tensor(5.2248, device='cuda:0', grad_fn=)\n",
"124000\n",
"12600 tensor(4.9390, device='cuda:0', grad_fn=)\n",
"125000\n",
"12700 tensor(5.2555, device='cuda:0', grad_fn=)\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/gedin/.local/lib/python3.10/site-packages/torch/autograd/__init__.py:200: UserWarning: Error detected in LogBackward0. Traceback of forward call that caused the error:\n",
" File \"/usr/lib/python3.10/runpy.py\", line 196, in _run_module_as_main\n",
" return _run_code(code, main_globals, None,\n",
" File \"/usr/lib/python3.10/runpy.py\", line 86, in _run_code\n",
" exec(code, run_globals)\n",
" File \"/home/gedin/.local/lib/python3.10/site-packages/ipykernel_launcher.py\", line 17, in \n",
" app.launch_new_instance()\n",
" File \"/home/gedin/.local/lib/python3.10/site-packages/traitlets/config/application.py\", line 1043, in launch_instance\n",
" app.start()\n",
" File \"/home/gedin/.local/lib/python3.10/site-packages/ipykernel/kernelapp.py\", line 725, in start\n",
" self.io_loop.start()\n",
" File \"/home/gedin/.local/lib/python3.10/site-packages/tornado/platform/asyncio.py\", line 195, in start\n",
" self.asyncio_loop.run_forever()\n",
" File \"/usr/lib/python3.10/asyncio/base_events.py\", line 600, in run_forever\n",
" self._run_once()\n",
" File \"/usr/lib/python3.10/asyncio/base_events.py\", line 1896, in _run_once\n",
" handle._run()\n",
" File \"/usr/lib/python3.10/asyncio/events.py\", line 80, in _run\n",
" self._context.run(self._callback, *self._args)\n",
" File \"/home/gedin/.local/lib/python3.10/site-packages/ipykernel/kernelbase.py\", line 513, in dispatch_queue\n",
" await self.process_one()\n",
" File \"/home/gedin/.local/lib/python3.10/site-packages/ipykernel/kernelbase.py\", line 502, in process_one\n",
" await dispatch(*args)\n",
" File \"/home/gedin/.local/lib/python3.10/site-packages/ipykernel/kernelbase.py\", line 409, in dispatch_shell\n",
" await result\n",
" File \"/home/gedin/.local/lib/python3.10/site-packages/ipykernel/kernelbase.py\", line 729, in execute_request\n",
" reply_content = await reply_content\n",
" File \"/home/gedin/.local/lib/python3.10/site-packages/ipykernel/ipkernel.py\", line 422, in do_execute\n",
" res = shell.run_cell(\n",
" File \"/home/gedin/.local/lib/python3.10/site-packages/ipykernel/zmqshell.py\", line 540, in run_cell\n",
" return super().run_cell(*args, **kwargs)\n",
" File \"/home/gedin/.local/lib/python3.10/site-packages/IPython/core/interactiveshell.py\", line 3009, in run_cell\n",
" result = self._run_cell(\n",
" File \"/home/gedin/.local/lib/python3.10/site-packages/IPython/core/interactiveshell.py\", line 3064, in _run_cell\n",
" result = runner(coro)\n",
" File \"/home/gedin/.local/lib/python3.10/site-packages/IPython/core/async_helpers.py\", line 129, in _pseudo_sync_runner\n",
" coro.send(None)\n",
" File \"/home/gedin/.local/lib/python3.10/site-packages/IPython/core/interactiveshell.py\", line 3269, in run_cell_async\n",
" has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n",
" File \"/home/gedin/.local/lib/python3.10/site-packages/IPython/core/interactiveshell.py\", line 3448, in run_ast_nodes\n",
" if await self.run_code(code, result, async_=asy):\n",
" File \"/home/gedin/.local/lib/python3.10/site-packages/IPython/core/interactiveshell.py\", line 3508, in run_code\n",
" exec(code_obj, self.user_global_ns, self.user_ns)\n",
" File \"/tmp/ipykernel_30815/3845719666.py\", line 13, in \n",
" loss = criterion(torch.log(ypredicted), x) #x is to_predict\n",
" (Triggered internally at ../torch/csrc/autograd/python_anomaly_mode.cpp:114.)\n",
" Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass\n"
]
},
{
"ename": "RuntimeError",
"evalue": "Function 'LogBackward0' returned nan values in its 0th output.",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[14], line 19\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m step \u001b[38;5;241m%\u001b[39m \u001b[38;5;241m10000\u001b[39m \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 18\u001b[0m torch\u001b[38;5;241m.\u001b[39msave(model\u001b[38;5;241m.\u001b[39mstate_dict(), \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmodel-tri-2following-embed=200steps-\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mstep\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.bin\u001b[39m\u001b[38;5;124m'\u001b[39m) \n\u001b[0;32m---> 19\u001b[0m \u001b[43mloss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 20\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m 21\u001b[0m \u001b[38;5;66;03m# torch.save(model.state_dict(), f'model-tri-2following-{i}.bin') \u001b[39;00m\n",
"File \u001b[0;32m~/.local/lib/python3.10/site-packages/torch/_tensor.py:487\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m 477\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 478\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m 479\u001b[0m Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m 480\u001b[0m (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 485\u001b[0m inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m 486\u001b[0m )\n\u001b[0;32m--> 487\u001b[0m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mautograd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 488\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgradient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\n\u001b[1;32m 489\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/.local/lib/python3.10/site-packages/torch/autograd/__init__.py:200\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m 195\u001b[0m retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m 197\u001b[0m \u001b[38;5;66;03m# The reason we repeat same the comment below is that\u001b[39;00m\n\u001b[1;32m 198\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m 199\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 200\u001b[0m \u001b[43mVariable\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execution_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_backward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[1;32m 201\u001b[0m \u001b[43m \u001b[49m\u001b[43mtensors\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgrad_tensors_\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 202\u001b[0m \u001b[43m \u001b[49m\u001b[43mallow_unreachable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maccumulate_grad\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
"\u001b[0;31mRuntimeError\u001b[0m: Function 'LogBackward0' returned nan values in its 0th output."
]
}
],
"source": [
"torch.autograd.set_detect_anomaly(True)\n",
"# model.load_state_dict(torch.load(f'model-tri-2following-final.bin'))\n",
"for i in range(params['epochs']):\n",
" print('epoch: =', i)\n",
" model.train()\n",
" step = 0\n",
" for x, y, z in data: # word, following, 2nd_following words\n",
" x = x.to(device)\n",
" y = y.to(device)\n",
" z = z.to(device)\n",
" optimizer.zero_grad()\n",
" ypredicted = model([y, z]) #following, 2nd_following word\n",
" loss = criterion(torch.log(ypredicted), x) #x is to_predict\n",
" if step % 100 == 0:\n",
" print(step, loss)\n",
" step += 1\n",
" if step % 10000 == 0:\n",
" torch.save(model.state_dict(), f'model-tri-2following-embed=200steps-{step}.bin') \n",
" loss.backward()\n",
" optimizer.step()\n",
"# torch.save(model.state_dict(), f'model-tri-2following-{i}.bin') "
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "54b018d8",
"metadata": {},
"outputs": [],
"source": [
"torch.save(model.state_dict(),'model-tri-2following-embed=300.bin')"
]
},
{
"cell_type": "code",
"execution_count": 337,
"id": "8d0c2694",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 319,
"id": "ced7956c",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'is': 0.11248013377189636, 'be': 0.0887126550078392, 'are': 0.05770659074187279, 'was': 0.04821299761533737, 'and': 0.031015213578939438, '': 0.02695723995566368, 'been': 0.016298962756991386, 'Is': 0.01539035513997078, 'not': 0.015107596293091774, 'were': 0.014968073926866055}\n"
]
}
],
"source": [
"a = results[5]\n",
"print(a)"
]
},
{
"cell_type": "code",
"execution_count": 338,
"id": "6546a0a9",
"metadata": {},
"outputs": [],
"source": [
"b={'is': 0.2, 'the': 0.2, '': 0.4}"
]
},
{
"cell_type": "code",
"execution_count": 324,
"id": "f0aa2e2e",
"metadata": {},
"outputs": [],
"source": [
"b={'is': 0.2, 'the': 0.2}"
]
},
{
"cell_type": "code",
"execution_count": 328,
"id": "f4396efc",
"metadata": {},
"outputs": [],
"source": [
"results_summ = summarize_probs_unk(b, const_wildcard = False)"
]
},
{
"cell_type": "code",
"execution_count": 329,
"id": "d982245e",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[('is', 0.25), ('the', 0.25), ('', 0.5)]\n"
]
}
],
"source": [
"print(results_summ)"
]
},
{
"cell_type": "code",
"execution_count": 293,
"id": "8e091219",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'is': 0.495, 'the': 0.495}\n"
]
}
],
"source": [
"print(b)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "7dd5e6f8",
"metadata": {},
"outputs": [],
"source": [
"def get_first_word(text):\n",
" \"\"\"Return the first word of a string.\"\"\"\n",
" word = \"\"\n",
" for i in range(len(text)-1):\n",
"# if text[i] in [' ', ',', '.']\n",
" if text[i] == ' ':\n",
" return word.rstrip()\n",
" else:\n",
" word += text[i]\n",
" return word.rstrip()\n",
"\n",
"def get_values_from_model(context: list, model, vocab, k=10):\n",
" words = [vocab.forward([word]) for word in context]\n",
" ixs = torch.tensor(words).to(device)\n",
" out = model(ixs)\n",
" top = torch.topk(out[0], k)\n",
" top_indices = top.indices.tolist()\n",
" top_probs = top.values.tolist()\n",
" top_words = vocab.lookup_tokens(top_indices)\n",
" return list(zip(top_words, top_probs))\n",
"\n",
"def summarize_probs_unk(dic, const_wildcard=True):\n",
" ''' \n",
" dic: dictionary of probabilities returned by model \n",
" returns: tab of probabilities, with specificly as last element\n",
" '''\n",
" if const_wildcard: \n",
" if '' in dic.keys():\n",
" del dic['']\n",
" probsum = sum(float(val) for key, val in dic.items())\n",
" for key in dic:\n",
" dic[key] = dic[key]/(probsum*(1+params['wildcard_minweight'])) ###leave some space for wildcard\n",
" tab = [(key, val) for key, val in dic.items()]\n",
" tab.append(('', 1-sum([val for val in dic.values()])))\n",
" elif '' not in dic.keys():\n",
" tab = [(key, val) for key, val in dic.items()]\n",
" tab.append(('', 1-sum([val for val in dic.values()])))\n",
" else:\n",
" wildcard_value = dic['']\n",
" del dic['']\n",
" probsum = sum([float(val) for key, val in dic.items()])\n",
" for key in dic:\n",
" dic[key] = dic[key]/(probsum+wildcard_value)\n",
" tab = [(key, val) for key, val in dic.items()]\n",
" tab.append(('', 1-sum([val for val in dic.values()])))\n",
" return tab\n",
"\n",
"def gonito_format(dic, const_wildcard = True):\n",
" tab = summarize_probs_unk(dic=dic, const_wildcard=const_wildcard)\n",
" result = ''\n",
" for element in tab[:-1]:\n",
" result+=str(element[0])+':'+str(element[1])+'\\t'\n",
" result += ':'+ str(tab[-1][1]) + '\\n'\n",
" return result"
]
},
{
"cell_type": "code",
"execution_count": 342,
"id": "2b7513f3",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 19,
"id": "4b0e66e2",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/tmp/ipykernel_30815/606935597.py:22: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
" out = self.softmax(concated)\n"
]
},
{
"data": {
"text/plain": [
"[('the', 1, 0.28959694504737854),\n",
" ('', 0, 0.2600848376750946),\n",
" ('United', 157, 0.24865739047527313),\n",
" ('loyal', 3181, 0.011600562371313572),\n",
" ('ed', 193, 0.009287656284868717),\n",
" ('other', 59, 0.009135955013334751),\n",
" ('of', 2, 0.00894735287874937),\n",
" ('Southern', 853, 0.007434409577399492),\n",
" ('tho', 33, 0.00726652005687356),\n",
" ('The', 17, 0.006302454508841038)]"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"\n",
"ixs = torch.tensor([vocab.forward(['States']), vocab.forward(['of'])]).to(device)\n",
"\n",
"out = model(ixs)\n",
"top = torch.topk(out[0], 10)\n",
"top_indices = top.indices.tolist()\n",
"top_probs = top.values.tolist()\n",
"top_words = vocab.lookup_tokens(top_indices)\n",
"list(zip(top_words, top_indices, top_probs))"
]
},
{
"cell_type": "code",
"execution_count": 40,
"id": "a92abbf2",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
""
]
},
"execution_count": 40,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.load_state_dict(torch.load(f'model-tri-2following-embed=100.bin'))"
]
},
{
"cell_type": "code",
"execution_count": 41,
"id": "fc7cf293",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/tmp/ipykernel_30815/606935597.py:22: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
" out = self.softmax(concated)\n"
]
}
],
"source": [
"with lzma.open(test_file, 'rt') as file:\n",
" predict_words = []\n",
" results = []\n",
" for line in file:\n",
" line = replace_endline(line) #get only relevant\n",
" line = line.split('\\t')[6:]\n",
" context = line[1].rstrip().split(\" \")[:2] #two words from the left context\n",
" predict_words.append(context) #get_first_word(split[1cd \n",
" vocab = train_dataset.vocab\n",
" for context_words in predict_words:\n",
" predicted_distribution = get_values_from_model(context_words, model, vocab, k=10)\n",
" distribution_as_dict = dict(predicted_distribution)\n",
"# print(distribution_as_dict)\n",
" results.append(distribution_as_dict)\n"
]
},
{
"cell_type": "code",
"execution_count": 85,
"id": "4f6de0f6",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[{'': 0.22333872318267822, 'the': 0.04290543496608734, 'of': 0.038316406309604645, 'to': 0.03587133809924126, 'and': 0.03049224615097046, 'in': 0.015468255616724491, 'was': 0.012148280628025532, 'a': 0.011980442330241203, 'be': 0.011976692825555801, 'is': 0.010855000466108322}, {'': 0.21075238287448883, 'hundred': 0.0673641711473465, 'dred': 0.005059292074292898, 'eight': 0.004337533377110958, 'degrees': 0.0038635244127362967, 'due': 0.003796575590968132, 'dollars': 0.003567937295883894, 'long': 0.003466016612946987, 'north': 0.003463953034952283, 'men': 0.00341203180141747}, {'a': 0.16613157093524933, 'the': 0.15993879735469818, '': 0.0938526839017868, 'this': 0.019656965509057045, 'other': 0.01839813031256199, 'of': 0.015854211524128914, 'and': 0.014281678013503551, 'his': 0.012650839053094387, 'The': 0.01190563477575779, 'tho': 0.010437354445457458}, {'': 0.11687463521957397, 'it': 0.049038924276828766, 'they': 0.03917419910430908, 'and': 0.03811464458703995, 'which': 0.03098314255475998, 'he': 0.028336884453892708, 'you': 0.024328071624040604, 'It': 0.022830305621027946, 'I': 0.021094566211104393, 'who': 0.02095915377140045}, {'': 0.09226696193218231, 'able': 0.015251555480062962, 'and': 0.012043489143252373, 'order': 0.011638428084552288, 'enough': 0.010548103600740433, 'is': 0.009963924996554852, 'right': 0.009118305519223213, 'him': 0.008917313069105148, 'attempt': 0.007974582724273205, 'was': 0.007815857417881489}, {'is': 0.11511300504207611, 'be': 0.08824419230222702, 'are': 0.05649878457188606, 'was': 0.04927655681967735, 'and': 0.03056171163916588, '': 0.02702254056930542, 'been': 0.016901914030313492, 'Is': 0.016430748626589775, 'not': 0.015687286853790283, 'were': 0.01444857195019722}, {'': 0.1327797919511795, 'and': 0.024617979303002357, 'was': 0.00857674703001976, 'held': 0.007952810265123844, 'look': 0.006116086151450872, 'is': 0.005847071297466755, 'Beginning': 0.0056276749819517136, 'arrived': 0.005414668470621109, 'that': 0.005328293424099684, 'thereon': 0.005003671627491713}, {'': 0.10696937143802643, 'and': 0.05472623184323311, 'of': 0.04754647985100746, 'is': 0.018190041184425354, 'are': 0.01705046370625496, 'after': 0.016676705330610275, 'it': 0.016619550064206123, 'in': 0.013238836079835892, 'that': 0.009560071863234043, 'the': 0.009014944545924664}, {'of': 0.20836004614830017, 'in': 0.09059623628854752, '': 0.07429423183202744, 'and': 0.06032003462314606, 'to': 0.046535905450582504, 'with': 0.0423453189432621, 'for': 0.038104332983493805, 'that': 0.024313272908329964, 'In': 0.019720906391739845, 'on': 0.018904248252511024}, {'': 0.1407906711101532, 'and': 0.08648557215929031, 'that': 0.06921716779470444, 'as': 0.06733249872922897, 'but': 0.02099286951124668, 'even': 0.017272774130105972, 'But': 0.01066331472247839, 'or': 0.010613738559186459, 'and,': 0.009398640133440495, 'And': 0.009269009344279766}, {'': 0.2627038061618805, 'the': 0.060708094388246536, 'of': 0.036793846637010574, 'and': 0.029553795233368874, 'a': 0.021964464336633682, 'to': 0.015695985406637192, 'in': 0.010639042593538761, 'at': 0.0073963068425655365, '.': 0.006468658801168203, '': 0.006342368200421333}, {'the': 0.4618034064769745, 'a': 0.07617291063070297, 'be-': 0.03713545948266983, '': 0.03308023884892464, 'The': 0.029952576383948326, 'tho': 0.024199577048420906, 'be¬': 0.012612164951860905, 'no': 0.010983242653310299, 'and': 0.010637091472744942, 'be\\xad': 0.009932301007211208}, {'': 0.11150252819061279, 'of': 0.042597446590662, 'and': 0.02449633926153183, 'to': 0.018879158422350883, 'the': 0.01835102029144764, 'for': 0.012080149725079536, 'a': 0.010379815474152565, 'in': 0.01028379611670971, 'be': 0.009574895724654198, 'was': 0.007363500073552132}, {'': 0.20835958421230316, '': 0.014537888579070568, 'and': 0.011784338392317295, 'made': 0.005978980101644993, 'was': 0.005651533603668213, 'recorded': 0.004739651456475258, 'that': 0.004241344518959522, 'be': 0.003857344388961792, 'is': 0.0036198573652654886, 'placed': 0.0035222468432039022}, {'to': 0.2214805781841278, 'and': 0.05094548314809799, '': 0.042482953518629074, 'we': 0.042465101927518845, 'I': 0.03351859003305435, 'will': 0.031639572232961655, 'not': 0.03125987946987152, 'would': 0.02364269085228443, 'they': 0.018556812778115273, 'you': 0.01713520474731922}, {'': 0.23146259784698486, 'and': 0.021180296316742897, 'of': 0.014936191029846668, 'to': 0.013211317360401154, 'the': 0.011549517512321472, '': 0.0063598062843084335, 'a': 0.006266749929636717, 'his': 0.005807745270431042, 'in': 0.005254070274531841, 'at': 0.00491053843870759}, {'of': 0.15778660774230957, '': 0.09474023431539536, 'to': 0.060130488127470016, 'and': 0.0411783792078495, 'for': 0.027365759015083313, 'with': 0.026851307600736618, 'by': 0.0247786957770586, 'that': 0.01456021424382925, 'a': 0.014512889087200165, 'from': 0.013957730494439602}, {'': 0.153826504945755, 'and': 0.01779167912900448, 'them': 0.007542381528764963, 'put': 0.007432775106281042, 'out': 0.006674687843769789, 'work': 0.005296988412737846, 'was': 0.005275645758956671, 'carry': 0.005257096607238054, 'that': 0.004899365361779928, 'it': 0.004891034681349993}, {'': 0.21511124074459076, 'of': 0.04556671157479286, 'and': 0.024301134049892426, 'in': 0.014269850216805935, 'to': 0.013811415992677212, 'that': 0.010617067106068134, 'on': 0.009551281109452248, 'for': 0.008698469959199429, 'things': 0.00600464316084981, 'by': 0.004462456330657005}, {'': 0.2627038061618805, 'the': 0.060708094388246536, 'of': 0.036793846637010574, 'and': 0.029553795233368874, 'a': 0.021964464336633682, 'to': 0.015695985406637192, 'in': 0.010639042593538761, 'at': 0.0073963068425655365, '.': 0.006468658801168203, '': 0.006342368200421333}, {'': 0.2137608826160431, 'and': 0.03420325741171837, 'of': 0.026632975786924362, 'as': 0.015281828120350838, 'that': 0.014173726551234722, 'all': 0.01293349638581276, 'but': 0.011043060570955276, 'for': 0.010435784235596657, 'so': 0.007831132970750332, 'fact': 0.005383650306612253}, {'to': 0.21688184142112732, 'will': 0.11243939399719238, 'not': 0.05901087448000908, 'may': 0.038121771067380905, 'the': 0.03546403348445892, 'and': 0.03337235003709793, 'shall': 0.03245922178030014, '': 0.03185460343956947, 'a': 0.029422687366604805, 'would': 0.029093723744153976}, {'': 0.09543072432279587, 'with-': 0.01749255880713463, 'and': 0.016958490014076233, 'find': 0.01670389249920845, 'pointed': 0.014505373314023018, 'carry': 0.012778980657458305, 'go': 0.012505585327744484, 'taken': 0.010809807106852531, 'get': 0.010587628930807114, 'come': 0.010311534628272057}, {'': 0.19435767829418182, 'and': 0.011447369121015072, 'as': 0.009315592236816883, 'went': 0.006825866177678108, 'him': 0.00656928401440382, 'them': 0.00550707895308733, 'up': 0.005280894692987204, 'is': 0.005211804993450642, 'able': 0.0050199986435472965, 'go': 0.004996961914002895}, {'': 0.2349490374326706, 'and': 0.02566382847726345, 'to': 0.016682205721735954, 'will': 0.012654948979616165, 'not': 0.010987816378474236, 'I': 0.010471819899976254, 'the': 0.010222157463431358, 'would': 0.00841697957366705, 'he': 0.007742380723357201, 'is': 0.007085733115673065}, {'of': 0.18639788031578064, '': 0.08190521597862244, 'to': 0.05711210146546364, 'in': 0.056451402604579926, 'and': 0.03365849703550339, 'on': 0.030336424708366394, 'by': 0.02989782951772213, 'for': 0.02898428961634636, 'with': 0.022200822830200195, 'that': 0.02205679938197136}, {'the': 0.22700978815555573, 'of': 0.08291079103946686, '': 0.0696745291352272, 'a': 0.057839397341012955, 'and': 0.03251868113875389, 'by': 0.029356244951486588, 'for': 0.02723454125225544, 'to': 0.020866040140390396, 'The': 0.01324895117431879, 'their': 0.01050820853561163}, {'': 0.2093510776758194, 'and': 0.017766309902071953, 'it': 0.017628418281674385, 'to': 0.011019784025847912, 'that': 0.00878529530018568, 'is': 0.00811536330729723, 'It': 0.006203165743499994, 'of': 0.005397103726863861, 'which': 0.00527550233528018, 'was': 0.0052408091723918915}, {'the': 0.1448695808649063, 'of': 0.12381888180971146, '': 0.06354980170726776, 'his': 0.02314835786819458, 'to': 0.023146068677306175, 'in': 0.022248776629567146, 'their': 0.02190326154232025, 'good': 0.021138714626431465, 'public': 0.01731843315064907, 'for': 0.01596975512802601}, {'': 0.10086608678102493, 'was': 0.014179660938680172, 'and': 0.011706585064530373, 'looked': 0.011435654014348984, 'held': 0.011394278146326542, 'arrived': 0.010205651633441448, 'presided': 0.007968415506184101, 'laughed': 0.007675629574805498, 'called': 0.006654257886111736, 'be': 0.00631422083824873}, {'': 0.16595689952373505, 'to': 0.026783792302012444, 'the': 0.026740582659840584, 'of': 0.021110257133841515, 'and': 0.016586044803261757, 'in': 0.009475589729845524, 'be': 0.007946169003844261, 'was': 0.006941476371139288, 'that': 0.0063302223570644855, 'a': 0.005784955341368914}, {'': 0.10322562605142593, 'we': 0.047937992960214615, 'I': 0.03349553421139717, 'and': 0.028309199959039688, 'they': 0.02772800624370575, 'he': 0.026365092024207115, 'which': 0.02500963769853115, 'who': 0.02432396449148655, 'it': 0.02241417020559311, 'that': 0.011394820176064968}, {'the': 0.2530023157596588, 'a': 0.10176271200180054, '': 0.08845433592796326, 'at': 0.041515082120895386, 'of': 0.03245680034160614, 'The': 0.0314832367002964, 'for': 0.02129232883453369, 'an': 0.020840294659137726, 'and': 0.01785355433821678, 'tho': 0.013970787636935711}, {'': 0.1552576869726181, 'the': 0.048198092728853226, 'of': 0.028564373031258583, 'and': 0.023097002878785133, 'to': 0.022242017090320587, 'in': 0.007789774797856808, 'for': 0.0077826837077736855, 'be': 0.007046535145491362, 'is': 0.00605395482853055, 'a': 0.005664240103214979}, {'': 0.2811985909938812, 'and': 0.0038096015341579914, 'men': 0.0031136798206716776, 'in': 0.003024699166417122, ';': 0.002690844004973769, 'to': 0.002255685394629836, 'there': 0.0022502816282212734, 'man': 0.0021516899578273296, 'time': 0.0020753047429025173, '': 0.002062465064227581}, {'': 0.13334812223911285, 'of': 0.08560147881507874, 'and': 0.056608375161886215, 'or': 0.0464644655585289, 'the': 0.03345730900764465, 'about': 0.0241240207105875, 'to': 0.022067740559577942, 'for': 0.019864672794938087, 'by': 0.015989569947123528, 'in': 0.012961748987436295}, {'': 0.15090197324752808, 'and': 0.028511079028248787, 'to': 0.02640763856470585, 'of': 0.024458639323711395, 'the': 0.020212605595588684, 'was': 0.013200847432017326, 'be': 0.01264059729874134, 'for': 0.011889615096151829, 'in': 0.009505866095423698, 'is': 0.00942089594900608}, {'': 0.2958369553089142, 'feet;': 0.08044090121984482, ';': 0.02983943000435829, 'feet,': 0.029169786721467972, 'running': 0.02876182273030281, '2;': 0.026335008442401886, '3;': 0.02302425168454647, '5;': 0.020926974713802338, '4;': 0.02011979930102825, 'feet:': 0.018075475469231606}, {'': 0.1584177315235138, 'No.': 0.08424711972475052, 'and': 0.05144903436303139, 'the': 0.038102131336927414, 'section': 0.030560946092009544, 'Section': 0.02335219830274582, 'Book': 0.018578067421913147, 'lot': 0.01393901091068983, '.': 0.011827166192233562, 'of': 0.011415652930736542}, {'of': 0.11761322617530823, '': 0.07540010660886765, 'in': 0.07077375054359436, 'and': 0.057942427694797516, 'by': 0.05039125680923462, 'for': 0.03406035527586937, 'In': 0.027113260701298714, 'are': 0.026015564799308777, 'is': 0.018230849876999855, 'with': 0.014756410382688046}, {'': 0.19916553795337677, 'the': 0.07090885937213898, 'of': 0.03878570720553398, 'and': 0.026194265112280846, 'a': 0.02108762413263321, 'to': 0.018788030371069908, 'in': 0.017393428832292557, 'be': 0.008278656750917435, 'for': 0.006737950257956982, 'was': 0.0067204381339251995}, {'this': 0.2805272340774536, 'the': 0.26955896615982056, 'said': 0.07197678089141846, '': 0.05247585102915764, 'that': 0.02539687231183052, 'York': 0.021105779334902763, 'a': 0.015501136891543865, 'tho': 0.013333857990801334, 'The': 0.013328607194125652, 'of': 0.012415573932230473}, {'the': 0.15293720364570618, 'of': 0.08826734870672226, '': 0.06351207941770554, 'and': 0.04176100715994835, 'live': 0.03349549323320389, 'a': 0.02991774119436741, 'The': 0.023885784670710564, 'capital': 0.013669596053659916, 'tho': 0.013321874663233757, 'his': 0.012635689228773117}, {'to': 0.4478341341018677, '': 0.06887559592723846, 'and': 0.04334395006299019, 'not': 0.0355389229953289, 'will': 0.029719946905970573, 'would': 0.019348537549376488, 'they': 0.014547325670719147, 'may': 0.013506502844393253, 'shall': 0.01072483230382204, 'the': 0.010310418903827667}, {'': 0.19079641997814178, 'the': 0.06604263931512833, 'and': 0.0405050665140152, 'of': 0.028105024248361588, 'a': 0.018237678334116936, 'was': 0.0177164189517498, 'be': 0.01570487953722477, 'Mr.': 0.011061737313866615, 'The': 0.010664232075214386, 'is': 0.010502614080905914}, {'the': 0.12414415180683136, '': 0.07357737421989441, 'an': 0.06351612508296967, 'any': 0.036189284175634384, 'that': 0.0244316253811121, 'last': 0.021560126915574074, 'such': 0.01905794069170952, 'and': 0.01793791726231575, 'a': 0.017535587772727013, 'of': 0.016196707263588905}, {'': 0.16636979579925537, 'the': 0.07424440234899521, 'of': 0.031639352440834045, 'The': 0.031398020684719086, 'Mr.': 0.029782554134726524, 'and': 0.019942330196499825, 'that': 0.019061796367168427, 'a': 0.01213876623660326, 'Mrs.': 0.009736793115735054, 'his': 0.007111952640116215}, {'of': 0.2650444209575653, 'in': 0.07592297345399857, 'to': 0.06257907301187515, 'for': 0.04815496876835823, 'by': 0.0364488810300827, '': 0.03624077886343002, 'at': 0.03106565587222576, 'with': 0.02742449752986431, 'from': 0.0269857719540596, 'In': 0.020415468141436577}, {'': 0.09690776467323303, 'and': 0.015597463585436344, 'necessary': 0.01436231005936861, 'is': 0.014166467823088169, 'as': 0.01257582101970911, 'enough': 0.011679481714963913, 'able': 0.011601371690630913, 'order': 0.010675780475139618, 'have': 0.009222178719937801, 'was': 0.00919886864721775}, {'': 0.13388632237911224, 'the': 0.07657106220722198, 'of': 0.046360649168491364, 'and': 0.03315750136971474, 'to': 0.024125175550580025, 'a': 0.02000236138701439, 'in': 0.01346728764474392, 'be': 0.012692473828792572, 'was': 0.010378892533481121, 'is': 0.009564382024109364}, {'It': 0.2023593932390213, 'it': 0.1580650955438614, '': 0.08693230152130127, 'which': 0.06967539340257645, 'there': 0.04238494113087654, 'that': 0.02560305781662464, 'he': 0.02346610277891159, 'There': 0.02172955684363842, 'and': 0.017989816144108772, 'who': 0.015684504061937332}, {'': 0.12309729307889938, 'to': 0.06693068891763687, 'and': 0.037865594029426575, 'in': 0.019001852720975876, 'of': 0.014264767989516258, 'know': 0.01425018347799778, 'that': 0.012781950645148754, 'or': 0.012018205597996712, 'question': 0.011336366645991802, 'determine': 0.01034859474748373}, {'': 0.16459277272224426, 'one': 0.01960723102092743, 'part': 0.01937238499522209, 'some': 0.012009568512439728, 'out': 0.00936990324407816, 'members': 0.006722881458699703, 'and': 0.005733037833124399, 'tion': 0.005703493487089872, 'side': 0.0055899848230183125, 'portion': 0.005201635882258415}, {'the': 0.5587978959083557, '': 0.056747399270534515, 'a': 0.05188652500510216, 'his': 0.022821972146630287, 'tho': 0.019640790298581123, 'The': 0.017851518467068672, 'of': 0.01729748211801052, 'and': 0.015954583883285522, 'said': 0.012346330098807812, 'this': 0.012271452695131302}, {'of': 0.20458759367465973, 'to': 0.07442232966423035, 'in': 0.06207741051912308, '': 0.050878193229436874, 'on': 0.03592154011130333, 'by': 0.03302523493766785, 'and': 0.03274650126695633, 'for': 0.0324152372777462, 'that': 0.02703711949288845, 'from': 0.02594793401658535}, {'the': 0.6219492554664612, 'a': 0.04427891969680786, 'The': 0.041617970913648605, '': 0.037857986986637115, 'of': 0.03687625378370285, 'tho': 0.028705911710858345, 'and': 0.010256663896143436, 'in': 0.008192397654056549, 'tbe': 0.007718721870332956, 'by': 0.00531271006911993}, {'': 0.08993969857692719, 'the': 0.06810890138149261, 'a': 0.06206537038087845, 'of': 0.034445617347955704, 'and': 0.031003830954432487, 'to': 0.028768161311745644, 'from': 0.019300518557429314, 'is': 0.018910136073827744, 'are': 0.016878407448530197, 'electric': 0.014177226461470127}, {'': 0.18855883181095123, 'to': 0.04066105931997299, 'and': 0.035258661955595016, 'the': 0.024452824145555496, 'of': 0.020135903730988503, 'not': 0.015071392990648746, 'in': 0.015021132305264473, 'I': 0.009083894081413746, 'a': 0.007654462940990925, 'or': 0.007527998648583889}, {'': 0.16636979579925537, 'the': 0.07424440234899521, 'of': 0.031639352440834045, 'The': 0.031398020684719086, 'Mr.': 0.029782554134726524, 'and': 0.019942330196499825, 'that': 0.019061796367168427, 'a': 0.01213876623660326, 'Mrs.': 0.009736793115735054, 'his': 0.007111952640116215}, {'': 0.1273660808801651, 'one': 0.026916934177279472, 'part': 0.01550621259957552, 'out': 0.012488282285630703, 'some': 0.008507400751113892, 'members': 0.008122355677187443, 'side': 0.008007538504898548, 'portion': 0.0057683284394443035, 'office': 0.00534091517329216, 'end': 0.005275852978229523}, {'': 0.09255300462245941, 'and': 0.018078381195664406, 'as': 0.013416668400168419, 'able': 0.011580395512282848, 'order': 0.01084372028708458, 'began': 0.009791013784706593, 'enough': 0.009761173278093338, 'time': 0.008410961367189884, 'right': 0.006645579356700182, 'desire': 0.006200398318469524}, {'': 0.23499979078769684, 'I': 0.08796831220388412, 'he': 0.0356081947684288, 'they': 0.034290798008441925, 'and': 0.023758266121149063, 'we': 0.022871745750308037, 'that': 0.014736046083271503, 'it': 0.014676817692816257, 'We': 0.012473245151340961, 'who': 0.011135317385196686}, {'the': 0.2615965008735657, '': 0.08843378722667694, 'court': 0.08022492378950119, 'a': 0.05075463652610779, 'The': 0.021291088312864304, 'his': 0.01969745382666588, 'school': 0.018414979800581932, 'tho': 0.014739091508090496, 'clearing': 0.013796720653772354, 'dwelling': 0.00967891700565815}, {'the': 0.05515838414430618, '': 0.049362342804670334, 'a': 0.045221809297800064, 'was': 0.03969758749008179, 'is': 0.036361709237098694, 'are': 0.028332088142633438, 'be': 0.021571312099695206, 'and': 0.021194377914071083, 'were': 0.020839329808950424, 'been': 0.014942698180675507}, {'': 0.41779717803001404, 'his': 0.055066924542188644, 'the': 0.04845258593559265, 'her': 0.03529108688235283, 'John-': 0.028980758041143417, 'Jack-': 0.021150827407836914, 'sea-': 0.020473448559641838, 'of': 0.016892939805984497, 'their': 0.015144322998821735, 'per-': 0.014349921606481075}, {'': 0.14123116433620453, 'and': 0.08508452028036118, 'that': 0.03139442950487137, 'but': 0.027639800682663918, 'But': 0.01148972287774086, 'time': 0.009631537832319736, 'And': 0.008882002905011177, 'or': 0.006277700420469046, 'even': 0.006170748732984066, 'ago,': 0.004611112643033266}, {'': 0.15313118696212769, 'and': 0.03815886005759239, 'the': 0.02815636619925499, 'a': 0.016214614734053612, 'of': 0.013509503565728664, 'to': 0.011384996585547924, 'in': 0.011384604498744011, 'I': 0.00919684860855341, 'will': 0.008166464976966381, 'for': 0.006106128450483084}, {'the': 0.17217190563678741, 'this': 0.07584196329116821, '': 0.05829581618309021, 'a': 0.04550384730100632, 'any': 0.02100536786019802, 'his': 0.018010130152106285, 'good': 0.016632571816444397, 'no': 0.016003739088773727, 'same': 0.015403805300593376, 'in': 0.014960847795009613}, {'a': 0.15352678298950195, 'the': 0.15181739628314972, '': 0.060553211718797684, 'his': 0.04563713073730469, 'and': 0.03399541229009628, 'The': 0.0231952928006649, 'her': 0.017882507294416428, 'my': 0.013166196644306183, 'their': 0.012191245332360268, 'no': 0.012006066739559174}, {'': 0.08599492162466049, 'and': 0.02511093206703663, 'according': 0.016413647681474686, 'as': 0.012819021008908749, 'went': 0.011031052097678185, 'go': 0.010524680837988853, 'subject': 0.01033820677548647, 'them': 0.007004676852375269, 'or': 0.006527750287204981, 'addition': 0.005618991795927286}, {'': 0.179456427693367, 'up': 0.006148810964077711, 'in': 0.0031821741722524166, 'him': 0.002734625246375799, 'them,': 0.0025135648902505636, ';': 0.0024639256298542023, 'time': 0.0024632164277136326, 'it,': 0.002309486735612154, 'out': 0.0020205711480230093, 'down': 0.0019974440801888704}, {'': 0.08554667234420776, 'and': 0.0736304298043251, 'was': 0.06480255722999573, 'is': 0.059190716594457626, 'are': 0.042718589305877686, 'be': 0.02023109421133995, 'were': 0.019197452813386917, 'been': 0.018038803711533546, 'not': 0.017732709646224976, 'or': 0.017340250313282013}, {'': 0.1688309609889984, 'and': 0.02006111852824688, 'that': 0.007770112715661526, 'was': 0.00709958840161562, 'made': 0.005767476744949818, 'is': 0.005590095184743404, 'as': 0.004862690344452858, 'it': 0.0046456195414066315, 'up': 0.004455664660781622, \"o'clock\": 0.004236712586134672}, {'not': 0.2605684995651245, 'has': 0.11688710749149323, '': 0.08176139742136002, 'have': 0.07797925919294357, 'had': 0.046040892601013184, 'as': 0.0446464940905571, 'and': 0.03977198898792267, 'is': 0.020715106278657913, 'which': 0.013691405765712261, 'it': 0.011899206787347794}, {'the': 0.17105141282081604, '': 0.10605166852474213, 'said': 0.0995902493596077, 'of': 0.021969184279441833, 'a': 0.021848417818546295, 'this': 0.015864653512835503, 'his': 0.013750908896327019, 'tho': 0.009714529849588871, 'in': 0.006580637767910957, 'an': 0.006177004426717758}, {'': 0.1642853021621704, 'and': 0.06527991592884064, 'the': 0.019977740943431854, 'I': 0.012722053565084934, 'was': 0.01182668935507536, 'had': 0.008691038005053997, 'is': 0.00866643711924553, 'he': 0.008282666094601154, 'have': 0.007593042217195034, 'of': 0.007095601875334978}, {'': 0.12305183708667755, 'virtue': 0.013736439868807793, 'out': 0.011897969990968704, 'part': 0.007474182173609734, 'one': 0.00636790506541729, 'quarter': 0.005797136574983597, 'favor': 0.004433568567037582, 'result': 0.004207964986562729, 'that': 0.004180330317467451, 'guilty': 0.004170230124145746}, {'the': 0.11358048021793365, 'our': 0.07933776825666428, 'their': 0.06890609115362167, '': 0.054550476372241974, 'of': 0.0532047338783741, 'and': 0.043392911553382874, 'his': 0.03271344304084778, 'in': 0.02565833181142807, 'fellow': 0.013195527717471123, 'its': 0.013151085935533047}, {'': 0.23009847104549408, 'one': 0.015031878836452961, 'on': 0.005045732017606497, 'receipt,': 0.0050370884127914906, 'two': 0.004681880120187998, 'person': 0.003189430106431246, 'law': 0.003014675108715892, 'and': 0.0028828298673033714, 'day': 0.002821944886818528, 'man': 0.0026652233209460974}, {'': 0.17699190974235535, 'the': 0.06058167293667793, 'of': 0.054017964750528336, 'on': 0.03213147073984146, 'to': 0.022938324138522148, 'and': 0.019131002947688103, 'in': 0.016315989196300507, 'by': 0.012727695517241955, 'a': 0.011241450905799866, 'for': 0.011063478887081146}, {'': 0.09069057554006577, 'amount': 0.026211997494101524, 'number': 0.02383045293390751, 'power': 0.011207845993340015, 'out': 0.009357242844998837, 'piece': 0.008311481215059757, 'board': 0.008021222427487373, 'state': 0.007838902994990349, 'place': 0.006854182109236717, 'plenty': 0.006226851139217615}, {'the': 0.7980270385742188, 'tho': 0.031302716583013535, 'The': 0.027618413791060448, '': 0.012410642579197884, 'tbe': 0.011987672187387943, 'and': 0.00816554855555296, 'this': 0.004949354566633701, 'its': 0.004592251498252153, 'their': 0.003953348845243454, 'of': 0.0036193954292684793}, {'the': 0.21156883239746094, '': 0.0711536854505539, 'of': 0.05599840730428696, 'his': 0.02090224251151085, 'their': 0.01816452108323574, 'in': 0.016432473435997963, 'a': 0.014130156487226486, 'and': 0.01388604287058115, 'our': 0.013473914004862309, 'all': 0.01202459167689085}, {'': 0.21722713112831116, 'the': 0.09697026014328003, 'of': 0.08321390300989151, 'sai': 0.06674136966466904, 'a': 0.03770812973380089, 'in': 0.03742728382349014, 'and': 0.02563336491584778, 'New': 0.025257492437958717, 'an': 0.01509688887745142, 'to': 0.011446448042988777}, {'': 0.3338654041290283, '': 0.04618749022483826, 'it.': 0.009774107486009598, 'them.': 0.006868171039968729, 'country.': 0.0043514203280210495, 'time.': 0.004198272712528706, 'him.': 0.0037608551792800426, 'of': 0.0037137873005121946, 'years.': 0.003684385446831584, '.': 0.003497631289064884}, {'to': 0.17029626667499542, 'has': 0.09273231029510498, 'had': 0.0632791519165039, 'will': 0.05724276974797249, 'have': 0.053740035742521286, 'and': 0.05175934359431267, '': 0.03575572744011879, 'would': 0.035254284739494324, 'shall': 0.02098708413541317, 'may': 0.019686179235577583}, {'': 0.18103262782096863, 'the': 0.07785932719707489, 'of': 0.0481172651052475, 'and': 0.02372605912387371, 'a': 0.01900666207075119, 'to': 0.01806434616446495, 'in': 0.01182300504297018, 'was': 0.009058146737515926, 'his': 0.008844747208058834, 'be': 0.00792929157614708}, {'of': 0.1365719735622406, 'to': 0.1128191202878952, 'in': 0.07669951766729355, '': 0.07579964399337769, 'for': 0.05387192219495773, 'with': 0.0408676415681839, 'on': 0.028228603303432465, 'from': 0.02079148404300213, 'by': 0.015250957570970058, 'In': 0.014320558868348598}, {'': 0.13174818456172943, 'the': 0.05219246447086334, 'of': 0.03916771709918976, 'to': 0.016302049160003662, 'boy.': 0.013558179140090942, 'girl.': 0.01308105792850256, 'and': 0.012123104184865952, 'a': 0.011349006555974483, 'in': 0.009833695366978645, 'for': 0.008450793102383614}, {'of': 0.1401580423116684, 'in': 0.07013728469610214, '': 0.06959009170532227, 'to': 0.06802380830049515, 'with': 0.04675764590501785, 'and': 0.04130720719695091, 'all': 0.02651023678481579, 'for': 0.025844819843769073, 'on': 0.02419038489460945, 'by': 0.01584576815366745}, {'not': 0.20823605358600616, 'to': 0.13188952207565308, 'I': 0.10428959876298904, \"don't\": 0.06486573815345764, 'we': 0.053476762026548386, 'you': 0.04355204850435257, 'We': 0.030626624822616577, '': 0.02547943964600563, 'they': 0.023313086479902267, \"didn't\": 0.01955903321504593}, {'': 0.18794043362140656, 'the': 0.05969518423080444, 'of': 0.045794907957315445, 'to': 0.03612609580159187, 'in': 0.02196568436920643, 'and': 0.0213912446051836, 'at': 0.020096417516469955, 'a': 0.013334343209862709, '.': 0.009806562215089798, 'for': 0.008462193422019482}, {'': 0.1292857676744461, 'and': 0.047095365822315216, 'is': 0.009685859084129333, 'or': 0.008473139256238937, 'but': 0.008095779456198215, 'be': 0.006762219592928886, 'was': 0.006589047610759735, 'not': 0.006192698143422604, 'done': 0.005344223231077194, 'that': 0.0053388033993542194}, {'': 0.1456758826971054, 'they': 0.08661529421806335, 'who': 0.050809480249881744, 'we': 0.04678933322429657, 'which': 0.04495099559426308, 'there': 0.037610653787851334, 'that': 0.027404461055994034, 'They': 0.02301052398979664, 'and': 0.022478176280856133, 'We': 0.022469667717814445}, {'': 0.17266257107257843, 'the': 0.061241041868925095, 'of': 0.03665431588888168, 'and': 0.025804154574871063, 'a': 0.016591370105743408, 'to': 0.015231333673000336, 'his': 0.009704713709652424, 'be': 0.009219302795827389, 'my': 0.009122492745518684, 'I': 0.00804972369223833}, {'the': 0.49018922448158264, 'a': 0.07971637696027756, 'and': 0.0372132807970047, '': 0.02979383058845997, 'The': 0.023972969502210617, 'tho': 0.02117161825299263, 'in': 0.020621564239263535, 'al-': 0.0157414972782135, 'of': 0.011821978725492954, 'our': 0.010191377252340317}, {'of': 0.1930237114429474, 'in': 0.078556589782238, 'to': 0.05460745468735695, 'by': 0.045269932597875595, '': 0.04075690358877182, 'for': 0.03135232999920845, 'that': 0.029826167970895767, 'and': 0.02643091231584549, 'with': 0.021314244717359543, 'on': 0.02111966162919998}, {'the': 0.32039138674736023, 'of': 0.158921018242836, 'a': 0.05315949395298958, 'for': 0.0471026673913002, 'in': 0.04194990172982216, 'our': 0.031883276998996735, '': 0.02380768395960331, 'and': 0.020505186170339584, 'with': 0.01895971968770027, 'The': 0.016887381672859192}, {'to': 0.556331217288971, 'not': 0.07191608101129532, 'will': 0.04452606663107872, 'would': 0.03863736242055893, 'can': 0.02686115726828575, 'and': 0.026668453589081764, 'could': 0.015597746707499027, '': 0.015025721862912178, 'should': 0.014898914843797684, 'may': 0.014028624631464481}, {'': 0.15861724317073822, 'and': 0.02303946204483509, 'together': 0.02202565036714077, 'connected': 0.017643732950091362, 'connection': 0.01763218268752098, 'accordance': 0.0130849564447999, 'comply': 0.007123312447220087, 'acquainted': 0.0062169479206204414, 'compared': 0.006194624118506908, 'contact': 0.005691333673894405}]\n"
]
}
],
"source": []
},
{
"cell_type": "code",
"execution_count": 419,
"id": "361213dc",
"metadata": {},
"outputs": [],
"source": [
"with open(out_file, 'w') as outfile:\n",
" for elem in results: \n",
" outfile.write(gonito_format(elem, const_wildcard=False))"
]
},
{
"cell_type": "code",
"execution_count": 42,
"id": "b1e7ffd9",
"metadata": {},
"outputs": [],
"source": [
"# torch.save(model.state_dict(), f'model-tri-2following-embed=100.bin') \n",
"with open(\"test-A/out-embed100.tsv\", 'w') as outfile:\n",
" for elem in results: \n",
" outfile.write(gonito_format(elem, const_wildcard=False))\n"
]
},
{
"cell_type": "code",
"execution_count": 37,
"id": "50bd2af5",
"metadata": {},
"outputs": [],
"source": [
"# torch.save(model.state_dict(), f'model-tri-2following-embed=200.bin')\n",
"with open('test-A/out-embed=200.tsv', 'w') as outfile:\n",
" for elem in results: \n",
" outfile.write(gonito_format(elem, const_wildcard=False))"
]
},
{
"cell_type": "code",
"execution_count": 31,
"id": "8525bb9f",
"metadata": {},
"outputs": [],
"source": [
"# torch.save(model.state_dict(), f'model-tri-2following-embed=300.bin')\n",
"with open('test-A/out-embed=300.tsv', 'w') as outfile:\n",
" for elem in results: \n",
" outfile.write(gonito_format(elem, const_wildcard=False))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3b9be41c",
"metadata": {},
"outputs": [],
"source": [
"#############Below is only testing, trying ################"
]
},
{
"cell_type": "code",
"execution_count": 357,
"id": "61cbbf04",
"metadata": {},
"outputs": [],
"source": [
"results_summ = summarize_probs_unk(a, const_wildcard = False)"
]
},
{
"cell_type": "code",
"execution_count": 414,
"id": "6563ccfc",
"metadata": {},
"outputs": [],
"source": [
"results_summ = [summarize_probs_unk(result, const_wildcard = False) for result in results]"
]
},
{
"cell_type": "code",
"execution_count": 412,
"id": "d7aa952a",
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/plain": [
"[[('and', 0.09401928299350774),\n",
" ('was', 0.042643945320216904),\n",
" ('out', 0.028740502993675726),\n",
" ('that', 0.02787611493798393),\n",
" ('placed', 0.02753900259090668),\n",
" ('work', 0.026609481081919257),\n",
" ('made', 0.02404004335064763),\n",
" ('is', 0.023600215460896846),\n",
" ('up', 0.022804205421178007),\n",
" ('', 0.6821272058490673)],\n",
" [('and', 0.14603855372856134),\n",
" ('that', 0.03460113439693535),\n",
" ('or', 0.03433247108224818),\n",
" ('as', 0.03266713033798552),\n",
" ('was', 0.0304657482478034),\n",
" ('is', 0.026710330960551327),\n",
" ('be', 0.026167258600020275),\n",
" ('made', 0.02382438016840368),\n",
" ('contained', 0.02104032870501253),\n",
" ('