diff --git a/gonito.yaml b/gonito.yaml new file mode 100644 index 0000000..98d706c --- /dev/null +++ b/gonito.yaml @@ -0,0 +1,10 @@ +description: neural network, bigram +tags: + - neural-network + - bigram +params: + epochs: 1 + vocab_size: 10000 + embed_size: 250 + batch_size: 5000 + num_of_top: 10 diff --git a/neural_network_simple_colab.ipynb b/neural_network_simple_colab.ipynb new file mode 100644 index 0000000..1b12b49 --- /dev/null +++ b/neural_network_simple_colab.ipynb @@ -0,0 +1,334 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PAM8swqfl3YC" + }, + "outputs": [], + "source": [ + "import itertools\n", + "import lzma\n", + "import regex as re\n", + "import torch\n", + "from torch import nn\n", + "from torch.utils.data import IterableDataset, DataLoader\n", + "from torchtext.vocab import build_vocab_from_iterator\n", + "import pickle\n", + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "52BQle50l92y", + "outputId": "1f98398d-f385-4711-c2b7-3abe7418fbdb" + }, + "outputs": [], + "source": [ + "from google.colab import drive\n", + "drive.mount('/content/drive')" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "PNb3_zqUl3YD" + }, + "source": [ + "### Definitions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "b_6d7n2al3YE" + }, + "outputs": [], + "source": [ + "def clean_line(line: str):\n", + " separated = line.split('\\t')\n", + " prefix = separated[6].replace(r'\\n', ' ').strip()\n", + " suffix = separated[7].replace(r'\\n', ' ').strip()\n", + " return prefix + ' ' + suffix\n", + "\n", + "def get_words_from_line(line):\n", + " line = clean_line(line)\n", + " for word in line.split():\n", + " yield word\n", + "\n", + "def get_word_lines_from_file(file_name):\n", + " with lzma.open(file_name, mode='rt', encoding='utf-8') as fid:\n", + " for line in fid:\n", + " yield get_words_from_line(line)\n", + "\n", + "\n", + "def look_ahead_iterator(gen):\n", + " prev = None\n", + " for item in gen:\n", + " if prev is not None:\n", + " yield (prev, item)\n", + " prev = item\n", + "\n", + "def predict(word: str, num_of_top: str) -> str:\n", + " ixs = torch.tensor(vocab.forward([word])).to(device)\n", + " out = model(ixs)\n", + " top = torch.topk(out[0], num_of_top)\n", + " top_indices = top.indices.tolist()\n", + " top_probs = top.values.tolist()\n", + " top_words = vocab.lookup_tokens(top_indices)\n", + " zipped = list(zip(top_words, top_probs))\n", + " if '' in [element[0] for element in zipped]:\n", + " zipped = [(element[0] if element[0] != '' else '', element[1]) for element in zipped]\n", + " zipped[-1] = ('', zipped[-1][1])\n", + " else:\n", + " zipped[-1] = ('', zipped[-1][1])\n", + " return ' '.join([f'{element[0]}:{element[1]}' for element in zipped])\n", + "\n", + "def execute(path):\n", + " with lzma.open(f'{path}/in.tsv.xz', 'rt', encoding='utf-8') as f, \\\n", + " open(f'{path}/out.tsv', 'w', encoding='utf-8') as out:\n", + " for line in f:\n", + " prefix = line.split('\\t')[6]\n", + " left = prefix.replace(r'\\n', ' ').split()[-1]\n", + " result = predict(left, num_of_top)\n", + " out.write(f\"{result}\\n\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ZfV8fDhyl3YF" + }, + "outputs": [], + "source": [ + "class Bigrams(IterableDataset):\n", + " def __init__(self, text_file, vocabulary_size):\n", + " self.vocab = vocab\n", + " self.vocab.set_default_index(self.vocab[''])\n", + " self.vocabulary_size = vocabulary_size\n", + " self.text_file = text_file\n", + "\n", + " def __iter__(self):\n", + " return look_ahead_iterator(\n", + " (self.vocab[t] for t in itertools.chain.from_iterable(get_word_lines_from_file(self.text_file))))\n", + "\n", + " \n", + "class SimpleBigramNeuralLanguageModel(nn.Module):\n", + " def __init__(self, vocabulary_size, embedding_size):\n", + " super(SimpleBigramNeuralLanguageModel, self).__init__()\n", + " self.model = nn.Sequential(\n", + " nn.Embedding(vocabulary_size, embedding_size),\n", + " nn.Linear(embedding_size, vocabulary_size),\n", + " nn.Softmax()\n", + " )\n", + "\n", + " def forward(self, x):\n", + " return self.model(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "W0O6U62El3YG" + }, + "source": [ + "### Parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "eUS-U3_6l3YG" + }, + "outputs": [], + "source": [ + "vocab_size = 10000\n", + "embed_size = 250\n", + "batch_size = 5000\n", + "num_of_top = 10" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CPeVRcYZl3YG" + }, + "source": [ + "### Vocabulary building" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "4wBx0OTal3YH" + }, + "outputs": [], + "source": [ + "if os.path.exists('./vocabulary.pickle'):\n", + " with open('vocabulary.pickle', 'rb') as handle:\n", + " vocab = pickle.load(handle)\n", + "else:\n", + " vocab = build_vocab_from_iterator(\n", + " get_word_lines_from_file('./drive/MyDrive/ColabNotebooks/america/train/in.tsv.xz'),\n", + " max_tokens = vocab_size,\n", + " specials = [''])\n", + "\n", + " with open('vocabulary.pickle', 'wb') as handle:\n", + " pickle.dump(vocab, handle, protocol=pickle.HIGHEST_PROTOCOL)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "oVMVipnhl3YH", + "outputId": "e588d083-be33-4dce-e61c-b26e664b2c5f" + }, + "outputs": [], + "source": [ + "vocab.lookup_tokens([0, 1, 2, 3, 4, 4500])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1gJscHJUl3YI" + }, + "source": [ + "### Training" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "W_OjBUInl3YI" + }, + "outputs": [], + "source": [ + "model = SimpleBigramNeuralLanguageModel(vocab_size, embed_size)\n", + "vocab.set_default_index(vocab[''])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "f373Kduzl3YI", + "outputId": "0e6100df-d00e-4be3-83c4-cd2f671041e6" + }, + "outputs": [], + "source": [ + "#uczenie\n", + "from torch.utils.data import DataLoader\n", + "\n", + "device = 'cuda'\n", + "train_dataset = Bigrams('./drive/MyDrive/ColabNotebooks/america/train/in.tsv.xz', vocab_size)\n", + "model = SimpleBigramNeuralLanguageModel(vocab_size, embed_size).to(device)\n", + "data = DataLoader(train_dataset, batch_size=batch_size)\n", + "optimizer = torch.optim.Adam(model.parameters())\n", + "\n", + "#funkcja kosztu\n", + "criterion = torch.nn.NLLLoss()\n", + "\n", + "model.train()\n", + "step = 0\n", + "for x, y in data:\n", + " x = x.to(device)\n", + " y = y.to(device)\n", + " optimizer.zero_grad()\n", + " ypredicted = model(x)\n", + " loss = criterion(torch.log(ypredicted), y)\n", + " if step % 100 == 0:\n", + " print(step, loss)\n", + " step += 1\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + "torch.save(model.state_dict(), 'model1.bin')\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IHrqskyXl3YI" + }, + "source": [ + "### Evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "yK3oK65fl3YI", + "outputId": "ee3e64a6-c361-4e96-cad0-82f2950827ca" + }, + "outputs": [], + "source": [ + "model = SimpleBigramNeuralLanguageModel(vocab_size, embed_size).to(device)\n", + "model.load_state_dict(torch.load('model1.bin'))\n", + "model.eval()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ihz4Px0bl3YJ" + }, + "outputs": [], + "source": [ + "execute('./drive/MyDrive/ColabNotebooks/america/dev-0')\n", + "execute('./drive/MyDrive/ColabNotebooks/america/test-A')" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "provenance": [] + }, + "gpuClass": "standard", + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.8" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 0 +}