646 lines
30 KiB
Plaintext
646 lines
30 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 2,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/"
|
||
},
|
||
"id": "KYySXV60UbL4",
|
||
"outputId": "bb7d4752-ccc2-48cf-f5d4-540ffbcc1243"
|
||
},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"<torch._C.Generator at 0x7f3f2c178990>"
|
||
]
|
||
},
|
||
"execution_count": 2,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"import torch\n",
|
||
"import torch.nn as nn\n",
|
||
"import torch.nn.functional as F\n",
|
||
"import torch.optim as optim\n",
|
||
"\n",
|
||
"torch.manual_seed(1)\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"id": "Wle6wRL0UbL9"
|
||
},
|
||
"source": [
|
||
"### Data loading"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 3,
|
||
"metadata": {
|
||
"id": "63t_aV_IUbL-"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"import pickle\n",
|
||
"import lzma\n",
|
||
"import regex as re\n",
|
||
"\n",
|
||
"\n",
|
||
"def load_pickle(filename):\n",
|
||
" with open(filename, \"rb\") as f:\n",
|
||
" return pickle.load(f)\n",
|
||
"\n",
|
||
"\n",
|
||
"def save_pickle(d):\n",
|
||
" with open(\"vocabulary.pkl\", \"wb\") as f:\n",
|
||
" pickle.dump(d, f, protocol=pickle.HIGHEST_PROTOCOL)\n",
|
||
"\n",
|
||
"\n",
|
||
"def clean_document(document: str) -> str:\n",
|
||
" document = document.lower().replace(\"’\", \"'\")\n",
|
||
" document = re.sub(r\"'s|[\\-]\\\\n\", \"\", document)\n",
|
||
" document = re.sub(\n",
|
||
" r\"(\\\\+n|[{}\\[\\]”&:•¦()*0-9;\\\"«»$\\-><^,®¬¿?¡!#+. \\t\\n])+\", \" \", document\n",
|
||
" )\n",
|
||
" for to_find, substitute in zip(\n",
|
||
" [\"i'm\", \"won't\", \"n't\", \"'ll\"], [\"i am\", \"will not\", \" not\", \" will\"]\n",
|
||
" ):\n",
|
||
" document = document.replace(to_find, substitute)\n",
|
||
" return document\n",
|
||
"\n",
|
||
"\n",
|
||
"def get_words_from_line(line, clean_text=True):\n",
|
||
" if clean_text:\n",
|
||
" line = clean_document(line) # .rstrip()\n",
|
||
" else:\n",
|
||
" line = line.strip()\n",
|
||
" yield \"<s>\"\n",
|
||
" for m in re.finditer(r\"[\\p{L}0-9\\*]+|\\p{P}+\", line):\n",
|
||
" yield m.group(0).lower()\n",
|
||
" yield \"</s>\"\n",
|
||
"\n",
|
||
"\n",
|
||
"def get_word_lines_from_file(file_name, clean_text=True, only_text=False):\n",
|
||
" with lzma.open(file_name, \"r\") as fh:\n",
|
||
" for i, line in enumerate(fh):\n",
|
||
" if only_text:\n",
|
||
" line = \"\\t\".join(line.decode(\"utf-8\").split(\"\\t\")[:-2])\n",
|
||
" else:\n",
|
||
" line = line.decode(\"utf-8\")\n",
|
||
" if i % 10000 == 0:\n",
|
||
" print(i)\n",
|
||
" yield get_words_from_line(line, clean_text)\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"id": "N7HGIM40UbL-"
|
||
},
|
||
"source": [
|
||
"### Dataclasses"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 5,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/"
|
||
},
|
||
"id": "50ezMyyNUbL_",
|
||
"outputId": "88af632f-fa88-43ea-ea7e-a709a2428c11"
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"0\n",
|
||
"10000\n",
|
||
"20000\n",
|
||
"30000\n",
|
||
"40000\n",
|
||
"50000\n",
|
||
"60000\n",
|
||
"70000\n",
|
||
"80000\n",
|
||
"90000\n",
|
||
"100000\n",
|
||
"110000\n",
|
||
"120000\n",
|
||
"130000\n",
|
||
"140000\n",
|
||
"150000\n",
|
||
"160000\n",
|
||
"170000\n",
|
||
"180000\n",
|
||
"190000\n",
|
||
"200000\n",
|
||
"210000\n",
|
||
"220000\n",
|
||
"230000\n",
|
||
"240000\n",
|
||
"250000\n",
|
||
"260000\n",
|
||
"270000\n",
|
||
"280000\n",
|
||
"290000\n",
|
||
"300000\n",
|
||
"310000\n",
|
||
"320000\n",
|
||
"330000\n",
|
||
"340000\n",
|
||
"350000\n",
|
||
"360000\n",
|
||
"370000\n",
|
||
"380000\n",
|
||
"390000\n",
|
||
"400000\n",
|
||
"410000\n",
|
||
"420000\n",
|
||
"430000\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"from torch.utils.data import IterableDataset\n",
|
||
"from torchtext.vocab import build_vocab_from_iterator\n",
|
||
"import itertools\n",
|
||
"\n",
|
||
"\n",
|
||
"VOCAB_SIZE = 20000\n",
|
||
"\n",
|
||
"\n",
|
||
"def look_ahead_iterator(gen):\n",
|
||
" prev = None\n",
|
||
" for item in gen:\n",
|
||
" if prev is not None:\n",
|
||
" yield (prev, item)\n",
|
||
" prev = item\n",
|
||
"\n",
|
||
"\n",
|
||
"class Bigrams(IterableDataset):\n",
|
||
" def __init__(\n",
|
||
" self, text_file, vocabulary_size, vocab=None, only_text=False, clean_text=True\n",
|
||
" ):\n",
|
||
" self.vocab = (\n",
|
||
" build_vocab_from_iterator(\n",
|
||
" get_word_lines_from_file(text_file, clean_text, only_text),\n",
|
||
" max_tokens=vocabulary_size,\n",
|
||
" specials=[\"<unk>\"],\n",
|
||
" )\n",
|
||
" if vocab is None\n",
|
||
" else vocab\n",
|
||
" )\n",
|
||
" self.vocab.set_default_index(self.vocab[\"<unk>\"])\n",
|
||
" self.vocabulary_size = vocabulary_size\n",
|
||
" self.text_file = text_file\n",
|
||
" self.clean_text = clean_text\n",
|
||
" self.only_text = only_text\n",
|
||
"\n",
|
||
" def __iter__(self):\n",
|
||
" return look_ahead_iterator(\n",
|
||
" (\n",
|
||
" self.vocab[t]\n",
|
||
" for t in itertools.chain.from_iterable(\n",
|
||
" get_word_lines_from_file(\n",
|
||
" self.text_file, self.clean_text, self.only_text\n",
|
||
" )\n",
|
||
" )\n",
|
||
" )\n",
|
||
" )\n",
|
||
"\n",
|
||
"\n",
|
||
"vocab = None # torch.load('./vocab.pth')\n",
|
||
"\n",
|
||
"train_dataset = Bigrams(\"/content/train/in.tsv.xz\", VOCAB_SIZE, vocab, clean_text=False)\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 6,
|
||
"metadata": {
|
||
"id": "SCZ87kVxUbL_"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# torch.save(train_dataset.vocab, \"vocab.pth\")\n",
|
||
"# torch.save(train_dataset.vocab, \"vocab_only_text.pth\")\n",
|
||
"# torch.save(train_dataset.vocab, \"vocab_only_text_clean.pth\")\n",
|
||
"torch.save(train_dataset.vocab, \"vocab_2.pth\")\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"id": "bv_Adw_lUbMA"
|
||
},
|
||
"source": [
|
||
"### Model definition"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 7,
|
||
"metadata": {
|
||
"id": "oaoLz3jPUbMB"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"class SimpleBigramNeuralLanguageModel(nn.Module):\n",
|
||
" def __init__(self, vocabulary_size, embedding_size):\n",
|
||
" super(SimpleBigramNeuralLanguageModel, self).__init__()\n",
|
||
" self.model = nn.Sequential(\n",
|
||
" nn.Embedding(vocabulary_size, embedding_size),\n",
|
||
" nn.Linear(embedding_size, vocabulary_size),\n",
|
||
" nn.Softmax(),\n",
|
||
" )\n",
|
||
"\n",
|
||
" def forward(self, x):\n",
|
||
" return self.model(x)\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 8,
|
||
"metadata": {
|
||
"id": "YBDf3nEvUbMC"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"EMBED_SIZE = 100\n",
|
||
"\n",
|
||
"model = SimpleBigramNeuralLanguageModel(VOCAB_SIZE, EMBED_SIZE)\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 9,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 1000
|
||
},
|
||
"id": "CRe3STJUUbMC",
|
||
"outputId": "ab4b234a-49c7-4878-f8e4-e21072efa9ee"
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"0\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/container.py:217: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
|
||
" input = module(input)\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"0 tensor(10.0674, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"100 tensor(8.4352, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"200 tensor(7.6662, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"300 tensor(7.0716, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"400 tensor(6.6710, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"500 tensor(6.4540, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"600 tensor(5.9974, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"700 tensor(5.7973, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"800 tensor(5.8026, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"10000\n",
|
||
"900 tensor(5.7118, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"1000 tensor(5.7471, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"1100 tensor(5.6865, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"1200 tensor(5.4205, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"1300 tensor(5.4954, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"1400 tensor(5.5415, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"1500 tensor(5.3322, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"1600 tensor(5.4665, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"1700 tensor(5.4710, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"20000\n",
|
||
"1800 tensor(5.3953, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"1900 tensor(5.4881, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"2000 tensor(5.4915, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"2100 tensor(5.3621, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"2200 tensor(5.2872, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"2300 tensor(5.2590, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"2400 tensor(5.3661, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"2500 tensor(5.3305, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"30000\n",
|
||
"2600 tensor(5.3789, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"2700 tensor(5.3548, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"2800 tensor(5.4579, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"2900 tensor(5.2660, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"3000 tensor(5.3253, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"3100 tensor(5.4020, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"3200 tensor(5.2962, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"3300 tensor(5.2570, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"3400 tensor(5.2317, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"40000\n",
|
||
"3500 tensor(5.2410, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"3600 tensor(5.2404, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"3700 tensor(5.1738, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"3800 tensor(5.2654, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"3900 tensor(5.2595, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"4000 tensor(5.2850, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"4100 tensor(5.2995, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"4200 tensor(5.2581, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"4300 tensor(5.3323, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"50000\n",
|
||
"4400 tensor(5.2498, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"4500 tensor(5.2674, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"4600 tensor(5.3033, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"4700 tensor(5.2066, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"4800 tensor(5.2302, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"4900 tensor(5.2617, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"5000 tensor(5.2306, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"5100 tensor(5.2781, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"60000\n",
|
||
"5200 tensor(5.1833, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"5300 tensor(5.2166, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"5400 tensor(5.0845, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"5500 tensor(5.2272, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"5600 tensor(5.3175, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"5700 tensor(5.2425, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"5800 tensor(5.2449, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"5900 tensor(5.3225, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"6000 tensor(5.2786, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"70000\n",
|
||
"6100 tensor(5.1489, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"6200 tensor(5.1793, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"6300 tensor(5.2194, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"6400 tensor(5.1708, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"6500 tensor(5.1394, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"6600 tensor(5.1280, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"6700 tensor(5.0869, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"6800 tensor(5.3255, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"6900 tensor(5.3426, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"80000\n",
|
||
"7000 tensor(5.1176, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"7100 tensor(5.1991, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"7200 tensor(5.1227, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"7300 tensor(5.1744, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"7400 tensor(5.2222, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"7500 tensor(5.2110, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"7600 tensor(5.1553, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"7700 tensor(5.3283, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"90000\n",
|
||
"7800 tensor(5.2544, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"7900 tensor(5.1871, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"8000 tensor(5.2215, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"8100 tensor(5.1744, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"8200 tensor(5.1087, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"8300 tensor(5.1639, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"8400 tensor(5.1604, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"8500 tensor(5.1612, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"8600 tensor(5.2307, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"100000\n",
|
||
"8700 tensor(5.1648, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"8800 tensor(5.1066, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"8900 tensor(5.2405, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"9000 tensor(5.2184, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"9100 tensor(5.2677, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
|
||
"9200 tensor(5.0773, device='cuda:0', grad_fn=<NllLossBackward0>)\n"
|
||
]
|
||
},
|
||
{
|
||
"ename": "KeyboardInterrupt",
|
||
"evalue": "ignored",
|
||
"output_type": "error",
|
||
"traceback": [
|
||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
|
||
"\u001b[0;32m<ipython-input-9-c690ed9ba7ad>\u001b[0m in \u001b[0;36m<cell line: 11>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 21\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
|
||
"\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/_tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m 485\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 486\u001b[0m )\n\u001b[0;32m--> 487\u001b[0;31m torch.autograd.backward(\n\u001b[0m\u001b[1;32m 488\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 489\u001b[0m )\n",
|
||
"\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m 198\u001b[0m \u001b[0;31m# some Python versions print out the first line of a multi-line function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 199\u001b[0m \u001b[0;31m# calls in the traceback and some print out the last line\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 200\u001b[0;31m Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass\n\u001b[0m\u001b[1;32m 201\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 202\u001b[0m allow_unreachable=True, accumulate_grad=True) # Calls into the C++ engine to run the backward pass\n",
|
||
"\u001b[0;31mKeyboardInterrupt\u001b[0m: "
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"from torch.utils.data import DataLoader\n",
|
||
"\n",
|
||
"device = \"cuda\"\n",
|
||
"model = SimpleBigramNeuralLanguageModel(VOCAB_SIZE, EMBED_SIZE).to(device)\n",
|
||
"data = DataLoader(train_dataset, batch_size=5000)\n",
|
||
"optimizer = torch.optim.Adam(model.parameters())\n",
|
||
"criterion = torch.nn.NLLLoss()\n",
|
||
"\n",
|
||
"model.train()\n",
|
||
"step = 0\n",
|
||
"for x, y in data:\n",
|
||
" x = x.to(device)\n",
|
||
" y = y.to(device)\n",
|
||
" optimizer.zero_grad()\n",
|
||
" ypredicted = model(x)\n",
|
||
" loss = criterion(torch.log(ypredicted), y)\n",
|
||
" if step % 100 == 0:\n",
|
||
" print(step, loss)\n",
|
||
" step += 1\n",
|
||
" loss.backward()\n",
|
||
" optimizer.step()\n",
|
||
"\n",
|
||
"torch.save(model.state_dict(), \"model_2.bin\")\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 10,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/"
|
||
},
|
||
"id": "kS9NHTGeom3y",
|
||
"outputId": "ce83640e-d2fe-41e6-cd5d-38a5b0410323"
|
||
},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"[('the', 2, 0.15899169445037842),\n",
|
||
" ('\\\\', 1, 0.10546761751174927),\n",
|
||
" ('he', 28, 0.06849857419729233),\n",
|
||
" ('it', 15, 0.05329886078834534),\n",
|
||
" ('i', 26, 0.0421920120716095),\n",
|
||
" ('they', 50, 0.03895237296819687),\n",
|
||
" ('a', 8, 0.03352600708603859),\n",
|
||
" ('<unk>', 0, 0.031062396243214607),\n",
|
||
" ('we', 61, 0.02323235757648945),\n",
|
||
" ('she', 104, 0.02003088779747486)]"
|
||
]
|
||
},
|
||
"execution_count": 10,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"ixs = torch.tensor(train_dataset.vocab.forward([\"when\"])).to(device)\n",
|
||
"out = model(ixs)\n",
|
||
"top = torch.topk(out[0], 10)\n",
|
||
"top_indices = top.indices.tolist()\n",
|
||
"top_probs = top.values.tolist()\n",
|
||
"top_words = train_dataset.vocab.lookup_tokens(top_indices)\n",
|
||
"list(zip(top_words, top_indices, top_probs))\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 13,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/"
|
||
},
|
||
"id": "UTYHTCJvC_Nm",
|
||
"outputId": "576288fa-fdad-4c21-d924-f613eaf33063"
|
||
},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"<All keys matched successfully>"
|
||
]
|
||
},
|
||
"execution_count": 13,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"device = \"cuda\"\n",
|
||
"model = SimpleBigramNeuralLanguageModel(VOCAB_SIZE, EMBED_SIZE).to(device)\n",
|
||
"model.load_state_dict(torch.load(\"model1.bin\"))\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 14,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/"
|
||
},
|
||
"id": "8WexjGIAxaE4",
|
||
"outputId": "52252b81-3b98-42d3-b137-472af00dbb26"
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Training on /content/dev-0/in.tsv.xz\n",
|
||
"\rProgress: 0.01%\rProgress: 0.02%\rProgress: 0.03%\rProgress: 0.04%\rProgress: 0.05%\rProgress: 0.06%\rProgress: 0.07%\rProgress: 0.08%\rProgress: 0.09%\rProgress: 0.10%\rProgress: 0.10%\rProgress: 0.11%\rProgress: 0.12%\rProgress: 0.13%\rProgress: 0.14%\rProgress: 0.15%\rProgress: 0.16%\rProgress: 0.17%\rProgress: 0.18%\rProgress: 0.19%\rProgress: 0.20%\rProgress: 0.21%\rProgress: 0.22%\rProgress: 0.23%\rProgress: 0.24%\rProgress: 0.25%\rProgress: 0.26%\rProgress: 0.27%\rProgress: 0.28%\rProgress: 0.29%\rProgress: 0.29%\rProgress: 0.30%\rProgress: 0.31%\rProgress: 0.32%\rProgress: 0.33%\rProgress: 0.34%\rProgress: 0.35%\rProgress: 0.36%\rProgress: 0.37%\rProgress: 0.38%\rProgress: 0.39%\rProgress: 0.40%\rProgress: 0.41%\rProgress: 0.42%\rProgress: 0.43%\rProgress: 0.44%\rProgress: 0.45%\rProgress: 0.46%\rProgress: 0.47%\rProgress: 0.48%\rProgress: 0.48%\rProgress: 0.49%\rProgress: 0.50%\rProgress: 0.51%\rProgress: 0.52%\rProgress: 0.53%\rProgress: 0.54%\rProgress: 0.55%\rProgress: 0.56%\rProgress: 0.57%\rProgress: 0.58%\rProgress: 0.59%\rProgress: 0.60%\rProgress: 0.61%\rProgress: 0.62%\rProgress: 0.63%\rProgress: 0.64%\rProgress: 0.65%\rProgress: 0.66%\rProgress: 0.67%\rProgress: 0.67%\rProgress: 0.68%\rProgress: 0.69%\rProgress: 0.70%\rProgress: 0.71%\rProgress: 0.72%\rProgress: 0.73%\rProgress: 0.74%\rProgress: 0.75%\rProgress: 0.76%\rProgress: 0.77%\rProgress: 0.78%\rProgress: 0.79%\rProgress: 0.80%\rProgress: 0.81%\rProgress: 0.82%\rProgress: 0.83%\rProgress: 0.84%\rProgress: 0.85%\rProgress: 0.86%\rProgress: 0.87%\rProgress: 0.87%\rProgress: 0.88%\rProgress: 0.89%\rProgress: 0.90%\rProgress: 0.91%\rProgress: 0.92%\rProgress: 0.93%\rProgress: 0.94%\rProgress: 0.95%\rProgress: 0.96%\rProgress: 0.97%\rProgress: 0.98%\rProgress: 0.99%\rProgress: 1.00%\rProgress: 1.01%\rProgress: 1.02%\rProgress: 1.03%\rProgress: 1.04%\rProgress: 1.05%\rProgress: 1.06%\rProgress: 1.06%\rProgress: 1.07%\rProgress: 1.08%\rProgress: 1.09%\rProgress: 1.10%\rProgress: 1.11%\rProgress: 1.12%\rProgress: 1.13%\rProgress: 1.14%\rProgress: 1.15%\rProgress: 1.16%\rProgress: 1.17%\rProgress: 1.18%\rProgress: 1.19%\rProgress: 1.20%\rProgress: 1.21%\rProgress: 1.22%\rProgress: 1.23%\rProgress: 1.24%\rProgress: 1.25%\rProgress: 1.25%\rProgress: 1.26%\rProgress: 1.27%\rProgress: 1.28%\rProgress: 1.29%\rProgress: 1.30%\rProgress: 1.31%\rProgress: 1.32%\rProgress: 1.33%\rProgress: 1.34%\rProgress: 1.35%\rProgress: 1.36%\rProgress: 1.37%\rProgress: 1.38%\rProgress: 1.39%\rProgress: 1.40%\rProgress: 1.41%\rProgress: 1.42%\rProgress: 1.43%\rProgress: 1.44%\rProgress: 1.45%\rProgress: 1.45%\rProgress: 1.46%\rProgress: 1.47%\rProgress: 1.48%\rProgress: 1.49%\rProgress: 1.50%\rProgress: 1.51%\rProgress: 1.52%\rProgress: 1.53%\rProgress: 1.54%\rProgress: 1.55%\rProgress: 1.56%\rProgress: 1.57%\rProgress: 1.58%\rProgress: 1.59%\rProgress: 1.60%\rProgress: 1.61%\rProgress: 1.62%"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/container.py:217: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
|
||
" input = module(input)\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Progress: 100.00%\n",
|
||
"Training on /content/test-A/in.tsv.xz\n",
|
||
"Progress: 100.00%\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"def predict_word(ixs, model, top_k=5):\n",
|
||
" out = model(ixs)\n",
|
||
" top = torch.topk(out[0], 10)\n",
|
||
" top_indices = top.indices.tolist()\n",
|
||
" top_probs = top.values.tolist()\n",
|
||
" top_words = train_dataset.vocab.lookup_tokens(top_indices)\n",
|
||
" return list(zip(top_words, top_indices, top_probs))\n",
|
||
"\n",
|
||
"\n",
|
||
"def get_one_word(text, context=\"left\"):\n",
|
||
" # print(\"Getting word from:\", text)\n",
|
||
" if context == \"left\":\n",
|
||
" context = -1\n",
|
||
" else:\n",
|
||
" context = 0\n",
|
||
" return text.rstrip().split(\" \")[context]\n",
|
||
"\n",
|
||
"\n",
|
||
"def inference_on_file(filename, model, lines_no=1):\n",
|
||
" results_path = \"/\".join(filename.split(\"/\")[:-1]) + \"/out.tsv\"\n",
|
||
" with lzma.open(filename, \"r\") as fp, open(results_path, \"w\") as out_file:\n",
|
||
" print(\"Training on\", filename)\n",
|
||
" for i, line in enumerate(fp):\n",
|
||
" # left, right = [ get_one_word(text_part, context)\n",
|
||
" # for context, text_part in zip(line.split('\\t')[:-2], ('left', 'right'))]\n",
|
||
" line = line.decode(\"utf-8\")\n",
|
||
" # print(line)\n",
|
||
" left = get_one_word(line.split(\"\\t\")[-2])\n",
|
||
" # print(\"Current word:\", left)\n",
|
||
" tensor = torch.tensor(train_dataset.vocab.forward([left])).to(device)\n",
|
||
" results = predict_word(tensor, model, 9)\n",
|
||
" prob_sum = sum([word[2] for word in results])\n",
|
||
" result_line = (\n",
|
||
" \" \".join([f\"{word[0]}:{word[2]}\" for word in results])\n",
|
||
" + f\" :{prob_sum}\\n\"\n",
|
||
" )\n",
|
||
" # print(result_line)\n",
|
||
" out_file.write(result_line)\n",
|
||
" print(f\"\\rProgress: {(((i+1) / lines_no) * 100):.2f}%\", end=\"\")\n",
|
||
" print()\n",
|
||
"\n",
|
||
"\n",
|
||
"model.eval()\n",
|
||
"\n",
|
||
"for filepath, lines_no in zip(\n",
|
||
" (\"/content/dev-0/in.tsv.xz\", \"/content/test-A/in.tsv.xz\"), (10519.0, 7414.0)\n",
|
||
"):\n",
|
||
" inference_on_file(filepath, model, lines_no)\n"
|
||
]
|
||
}
|
||
],
|
||
"metadata": {
|
||
"accelerator": "GPU",
|
||
"colab": {
|
||
"provenance": []
|
||
},
|
||
"gpuClass": "standard",
|
||
"kernelspec": {
|
||
"display_name": "mj_venv",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.10.6"
|
||
},
|
||
"orig_nbformat": 4
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 0
|
||
}
|