{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [], "gpuType": "T4", "machine_shape": "hm" }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" }, "accelerator": "GPU" }, "cells": [ { "cell_type": "code", "execution_count": 4, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "3dV_4SJ2xY_C", "outputId": "c1039907-474a-427a-ee13-62a3b7b4a693" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n" ] } ], "source": [ "from google.colab import drive\n", "drive.mount(\"/content/gdrive\")" ] }, { "cell_type": "code", "source": [ "# %env DATA_DIR=/content/gdrive/MyDrive/data_gralinski\n", "DATA_DIR=\"/content/gdrive/MyDrive/data_gralinski/\"" ], "metadata": { "id": "VwdW1Qm3x9-N" }, "execution_count": 5, "outputs": [] }, { "cell_type": "code", "source": [ "import itertools\n", "import lzma\n", "import regex as re\n", "import torch\n", "from torch import nn\n", "from torch.utils.data import IterableDataset, DataLoader\n", "from torchtext.vocab import build_vocab_from_iterator" ], "metadata": { "id": "irsty5KcyYkR" }, "execution_count": 6, "outputs": [] }, { "cell_type": "code", "source": [ "def clean_line(line: str):\n", " separated = line.split('\\t')\n", " prefix = separated[6].replace(r'\\n', ' ')\n", " suffix = separated[7].replace(r'\\n', ' ')\n", " return prefix + ' ' + suffix" ], "metadata": { "id": "LXXtiKW3yY5J" }, "execution_count": 7, "outputs": [] }, { "cell_type": "code", "source": [ "def get_words_from_line(line):\n", " line = clean_line(line)\n", " for word in line.split():\n", " yield word" ], "metadata": { "id": "y9r0wmD3ycIi" }, "execution_count": 8, "outputs": [] }, { "cell_type": "code", "source": [ "def get_word_lines_from_file(file_name):\n", " with lzma.open(file_name, mode='rt', encoding='utf-8') as fid:\n", " for line in fid:\n", " yield get_words_from_line(line)" ], "metadata": { "id": "HE3YfiHkycKt" }, "execution_count": 9, "outputs": [] }, { "cell_type": "code", "source": [ "def look_ahead_iterator(gen):\n", " prev = None\n", " for item in gen:\n", " if prev is not None:\n", " yield (prev, item)\n", " prev = item" ], "metadata": { "id": "lvHvJV6XycNZ" }, "execution_count": 10, "outputs": [] }, { "cell_type": "code", "source": [ "def prediction(word: str) -> str:\n", " ixs = torch.tensor(vocab.forward([word])).to(device)\n", " out = model(ixs)\n", " top = torch.topk(out[0], 5)\n", " top_indices = top.indices.tolist()\n", " top_probs = top.values.tolist()\n", " top_words = vocab.lookup_tokens(top_indices)\n", " zipped = list(zip(top_words, top_probs))\n", " for index, element in enumerate(zipped):\n", " unk = None\n", " if '' in element:\n", " unk = zipped.pop(index)\n", " zipped.append(('', unk[1]))\n", " break\n", " if unk is None:\n", " zipped[-1] = ('', zipped[-1][1])\n", " return ' '.join([f'{x[0]}:{x[1]}' for x in zipped])" ], "metadata": { "id": "sOKeZN9cycP-" }, "execution_count": 11, "outputs": [] }, { "cell_type": "code", "source": [ "def create_outputs(folder_name):\n", " print(f'Creating outputs in {folder_name}')\n", " with lzma.open(f'{folder_name}/in.tsv.xz', mode='rt', encoding='utf-8') as fid:\n", " with open(f'{folder_name}/out2.tsv', 'w', encoding='utf-8', newline='\\n') as f:\n", " for line in fid:\n", " separated = line.split('\\t')\n", " prefix = separated[6].replace(r'\\n', ' ').split()[-1]\n", " output_line = prediction(prefix)\n", " f.write(output_line + '\\n')" ], "metadata": { "id": "MN_RftZNycSB" }, "execution_count": 30, "outputs": [] }, { "cell_type": "code", "source": [ "class Bigrams(IterableDataset):\n", " def __init__(self, text_file, vocabulary_size):\n", " self.vocab = build_vocab_from_iterator(\n", " get_word_lines_from_file(text_file),\n", " max_tokens=vocabulary_size,\n", " specials=[''])\n", " self.vocab.set_default_index(self.vocab[''])\n", " self.vocabulary_size = vocabulary_size\n", " self.text_file = text_file\n", "\n", " def __iter__(self):\n", " return look_ahead_iterator(\n", " (self.vocab[t] for t in itertools.chain.from_iterable(get_word_lines_from_file(self.text_file))))" ], "metadata": { "id": "n9wIsbLEycUd" }, "execution_count": 13, "outputs": [] }, { "cell_type": "code", "source": [ "class SimpleBigramNeuralLanguageModel(nn.Module):\n", " def __init__(self, vocabulary_size, embedding_size):\n", " super(SimpleBigramNeuralLanguageModel, self).__init__()\n", " self.model = nn.Sequential(\n", " nn.Embedding(vocabulary_size, embedding_size),\n", " nn.Linear(embedding_size, vocabulary_size),\n", " nn.Softmax()\n", " )\n", "\n", " def forward(self, x):\n", " return self.model(x)" ], "metadata": { "id": "l490B5KFycXj" }, "execution_count": 14, "outputs": [] }, { "cell_type": "code", "source": [ "vocab_size = 15000\n", "embed_size = 300\n", "batch_size = 3000\n", "device = 'cuda'\n", "path_to_train = DATA_DIR+'train/in.tsv.xz'\n", "path_to_model = 'model.bin'" ], "metadata": { "id": "mMC84-OzycZ5" }, "execution_count": 21, "outputs": [] }, { "cell_type": "code", "source": [ "vocab = build_vocab_from_iterator(\n", " get_word_lines_from_file(path_to_train),\n", " max_tokens=vocab_size,\n", " specials=['']\n", ")\n", "\n", "vocab.set_default_index(vocab[''])" ], "metadata": { "id": "Fsvv3QJl7kWN" }, "execution_count": 22, "outputs": [] }, { "cell_type": "code", "source": [ "train_dataset = Bigrams(path_to_train, vocab_size)" ], "metadata": { "id": "UK73WsKnB8ZP" }, "execution_count": 23, "outputs": [] }, { "cell_type": "code", "source": [ "model = SimpleBigramNeuralLanguageModel(vocab_size, embed_size).to(device)\n", "data = DataLoader(train_dataset, batch_size=batch_size)\n", "optimizer = torch.optim.Adam(model.parameters())\n", "criterion = torch.nn.NLLLoss()\n", "\n", "model.train()\n", "step = 0\n", "for x, y in data:\n", " x = x.to(device)\n", " y = y.to(device)\n", " optimizer.zero_grad()\n", " ypredicted = model(x)\n", " loss = criterion(torch.log(ypredicted), y)\n", " if step % 5000 == 0:\n", " print(step, loss)\n", " step += 1\n", " loss.backward()\n", " optimizer.step()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Q_Gz-stnqHPg", "outputId": "4b6a3751-21da-48a9-afcb-802b21f7274b" }, "execution_count": 24, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/container.py:217: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n", " input = module(input)\n" ] }, { "output_type": "stream", "name": "stdout", "text": [ "0 tensor(9.8217, device='cuda:0', grad_fn=)\n", "5000 tensor(5.3191, device='cuda:0', grad_fn=)\n", "10000 tensor(5.0986, device='cuda:0', grad_fn=)\n", "15000 tensor(5.2346, device='cuda:0', grad_fn=)\n", "20000 tensor(5.4174, device='cuda:0', grad_fn=)\n", "25000 tensor(5.1875, device='cuda:0', grad_fn=)\n", "30000 tensor(5.1892, device='cuda:0', grad_fn=)\n", "35000 tensor(5.0867, device='cuda:0', grad_fn=)\n", "40000 tensor(5.1812, device='cuda:0', grad_fn=)\n", "45000 tensor(5.1327, device='cuda:0', grad_fn=)\n" ] } ] }, { "cell_type": "code", "source": [ "torch.save(model.state_dict(), path_to_model)" ], "metadata": { "id": "WTl82y44qHR_" }, "execution_count": 25, "outputs": [] }, { "cell_type": "code", "source": [ "model = SimpleBigramNeuralLanguageModel(vocab_size, embed_size).to(device)\n", "model.load_state_dict(torch.load(path_to_model))\n", "model.eval()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "jViNAbxIqHUr", "outputId": "3c30bed7-8eaf-4e7b-eb89-89bdcf95abe7" }, "execution_count": 26, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "SimpleBigramNeuralLanguageModel(\n", " (model): Sequential(\n", " (0): Embedding(15000, 300)\n", " (1): Linear(in_features=300, out_features=15000, bias=True)\n", " (2): Softmax(dim=None)\n", " )\n", ")" ] }, "metadata": {}, "execution_count": 26 } ] }, { "cell_type": "code", "source": [ "create_outputs(DATA_DIR+'dev-0')" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "LkHFiPvoqHW8", "outputId": "d4d12906-5f4c-4df6-a22e-95a4fbbd9850" }, "execution_count": 29, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Creating outputs in /content/gdrive/MyDrive/data_gralinski/dev-0\n" ] }, { "output_type": "stream", "name": "stderr", "text": [ "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/container.py:217: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n", " input = module(input)\n" ] } ] }, { "cell_type": "code", "source": [ "create_outputs(DATA_DIR+'test-A')" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "QoiHDZ_ZqO8F", "outputId": "0fc59359-ce83-4a43-d11b-8d43280df3a1" }, "execution_count": 31, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Creating outputs in /content/gdrive/MyDrive/data_gralinski/test-A\n" ] } ] } ] }