diff --git a/cw/08_regresja_logistyczna.ipynb b/cw/08_regresja_logistyczna.ipynb new file mode 100644 index 0000000..efcec1b --- /dev/null +++ b/cw/08_regresja_logistyczna.ipynb @@ -0,0 +1,1050 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Regresja logistyczna" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## import bibliotek" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/media/kuba/ssdsam/anaconda3/lib/python3.8/site-packages/gensim/similarities/__init__.py:15: UserWarning: The gensim.similarities.levenshtein submodule is disabled, because the optional Levenshtein package is unavailable. Install Levenhstein (e.g. `pip install python-Levenshtein`) to suppress this warning.\n", + " warnings.warn(msg)\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "import gensim\n", + "import torch\n", + "import pandas as pd\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "from sklearn.datasets import fetch_20newsgroups\n", + "# https://scikit-learn.org/0.19/datasets/twenty_newsgroups.html\n", + "\n", + "from sklearn.feature_extraction.text import TfidfVectorizer\n", + "from sklearn.metrics import accuracy_score" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "CATEGORIES = ['soc.religion.christian', 'alt.atheism']" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "newsgroups_train_dev = fetch_20newsgroups(subset = 'train', categories=CATEGORIES)\n", + "newsgroups_test = fetch_20newsgroups(subset = 'test', categories=CATEGORIES)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "newsgroups_train_dev_text = newsgroups_train_dev['data']\n", + "newsgroups_test_text = newsgroups_test['data']" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "Y_train_dev = newsgroups_train_dev['target']\n", + "Y_test = newsgroups_test['target']" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "newsgroups_train_text, newsgroups_dev_text, Y_train, Y_dev = train_test_split(newsgroups_train_dev_text, Y_train_dev, random_state=42)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "Y_names = newsgroups_train_dev['target_names']" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['alt.atheism', 'soc.religion.christian']" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Y_names" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## baseline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## zadanie (5 minut)\n", + "\n", + "- stworzyć baseline " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### PYTANIE: co jest nie tak z regresją liniową?" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Regresja logistyczna" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### wektoryzacja" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## zadanie (5 minut)\n", + "\n", + "- na podstawie newsgroups_train_text stworzyć tfidf wektoryzer ze słownikiem max 10_000\n", + "- wygenerować wektory: X_train, X_dev, X_test" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### model - inicjalizacja " + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "class LogisticRegressionModel(torch.nn.Module):\n", + "\n", + " def __init__(self):\n", + " super(LogisticRegressionModel, self).__init__()\n", + " self.fc = torch.nn.Linear(FEAUTERES,1)\n", + "\n", + " def forward(self, x):\n", + " x = self.fc(x)\n", + " x = torch.sigmoid(x)\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "lr_model = LogisticRegressionModel()" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.4978],\n", + " [0.5009],\n", + " [0.4998],\n", + " [0.4990],\n", + " [0.5018]], grad_fn=)" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "lr_model(torch.Tensor(X_train[0:5].astype(np.float32).todense()))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "LogisticRegressionModel(\n", + " (fc): Linear(in_features=10000, out_features=1, bias=True)\n", + ")" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "lr_model" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Parameter containing:\n", + " tensor([[-0.0059, 0.0035, 0.0021, ..., -0.0042, -0.0057, -0.0049]],\n", + " requires_grad=True),\n", + " Parameter containing:\n", + " tensor([-0.0023], requires_grad=True)]" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(lr_model.parameters())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## model - trenowanie" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "BATCH_SIZE = 5" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "criterion = torch.nn.BCELoss()" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = torch.optim.SGD(lr_model.parameters(), lr = 0.1)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "809" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Y_train.shape[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "loss_score = 0\n", + "acc_score = 0\n", + "items_total = 0\n", + "lr_model.train()\n", + "for i in range(0, Y_train.shape[0], BATCH_SIZE):\n", + " X = X_train[i:i+BATCH_SIZE]\n", + " X = torch.tensor(X.astype(np.float32).todense())\n", + " Y = Y_train[i:i+BATCH_SIZE]\n", + " Y = torch.tensor(Y.astype(np.float32)).reshape(-1,1)\n", + " Y_predictions = lr_model(X)\n", + " acc_score += torch.sum((Y_predictions > 0.5) == Y).item()\n", + " items_total += Y.shape[0] \n", + " \n", + " optimizer.zero_grad()\n", + " loss = criterion(Y_predictions, Y)\n", + " loss.backward()\n", + " optimizer.step()\n", + " \n", + "\n", + " loss_score += loss.item() * Y.shape[0] " + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.5667],\n", + " [0.5802],\n", + " [0.5757],\n", + " [0.5670]], grad_fn=)" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Y_predictions" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.],\n", + " [1.],\n", + " [1.],\n", + " [0.]])" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Y" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "452" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "acc_score" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "809" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "items_total" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "accuracy: 0.5587144622991347\n" + ] + } + ], + "source": [ + "print(f'accuracy: {acc_score / items_total}')" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "BCE loss: 0.6745463597170355\n" + ] + } + ], + "source": [ + "print(f'BCE loss: {loss_score / items_total}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### model - ewaluacja" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [], + "source": [ + "def get_loss_acc(model, X_dataset, Y_dataset):\n", + " loss_score = 0\n", + " acc_score = 0\n", + " items_total = 0\n", + " model.eval()\n", + " for i in range(0, Y_dataset.shape[0], BATCH_SIZE):\n", + " X = X_dataset[i:i+BATCH_SIZE]\n", + " X = torch.tensor(X.astype(np.float32).todense())\n", + " Y = Y_dataset[i:i+BATCH_SIZE]\n", + " Y = torch.tensor(Y.astype(np.float32)).reshape(-1,1)\n", + " Y_predictions = model(X)\n", + " acc_score += torch.sum((Y_predictions > 0.5) == Y).item()\n", + " items_total += Y.shape[0] \n", + "\n", + " optimizer.zero_grad()\n", + " loss = criterion(Y_predictions, Y)\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + "\n", + " loss_score += loss.item() * Y.shape[0] \n", + " return (loss_score / items_total), (acc_score / items_total)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(0.6443227143826974, 0.622991347342398)" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "get_loss_acc(lr_model, X_train, Y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(0.6369243131743537, 0.6037037037037037)" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "get_loss_acc(lr_model, X_dev, Y_dev)" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(0.6323775731785694, 0.6499302649930265)" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "get_loss_acc(lr_model, X_test, Y_test)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### wagi modelu" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Parameter containing:\n", + " tensor([[ 0.0314, -0.0375, 0.0131, ..., -0.0057, -0.0008, -0.0089]],\n", + " requires_grad=True),\n", + " Parameter containing:\n", + " tensor([0.0563], requires_grad=True)]" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(lr_model.parameters())" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([ 0.0314, -0.0375, 0.0131, ..., -0.0057, -0.0008, -0.0089],\n", + " grad_fn=)" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(lr_model.parameters())[0][0]" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.return_types.topk(\n", + "values=tensor([0.3753, 0.2305, 0.2007, 0.2006, 0.1993, 0.1952, 0.1930, 0.1898, 0.1831,\n", + " 0.1731, 0.1649, 0.1647, 0.1543, 0.1320, 0.1314, 0.1303, 0.1296, 0.1261,\n", + " 0.1245, 0.1243], grad_fn=),\n", + "indices=tensor([8942, 6336, 1852, 9056, 1865, 4039, 7820, 5002, 8208, 1857, 9709, 803,\n", + " 1046, 130, 4306, 6481, 4370, 4259, 4285, 1855]))" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.topk(list(lr_model.parameters())[0][0], 20)" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "the\n", + "of\n", + "christ\n", + "to\n", + "church\n", + "god\n", + "rutgers\n", + "jesus\n", + "sin\n", + "christians\n", + "we\n", + "and\n", + "athos\n", + "1993\n", + "hell\n", + "our\n", + "his\n", + "he\n", + "heaven\n", + "christian\n" + ] + } + ], + "source": [ + "for i in torch.topk(list(lr_model.parameters())[0][0], 20)[1]:\n", + " print(vectorizer.get_feature_names()[i])" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.return_types.topk(\n", + "values=tensor([-0.3478, -0.2578, -0.2455, -0.2347, -0.2330, -0.2265, -0.2205, -0.2050,\n", + " -0.2044, -0.1979, -0.1876, -0.1790, -0.1747, -0.1745, -0.1734, -0.1647,\n", + " -0.1639, -0.1617, -0.1601, -0.1592], grad_fn=),\n", + "indices=tensor([5119, 8096, 5420, 4436, 6194, 1627, 6901, 5946, 9970, 3116, 1036, 9906,\n", + " 5654, 8329, 7869, 1039, 1991, 4926, 5035, 4925]))" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.topk(list(lr_model.parameters())[0][0], 20, largest = False)" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "keith\n", + "sgi\n", + "livesey\n", + "host\n", + "nntp\n", + "caltech\n", + "posting\n", + "morality\n", + "you\n", + "edu\n", + "atheism\n", + "wpd\n", + "mathew\n", + "solntze\n", + "sandvik\n", + "atheists\n", + "com\n", + "islamic\n", + "jon\n", + "islam\n" + ] + } + ], + "source": [ + "for i in torch.topk(list(lr_model.parameters())[0][0], 20, largest = False)[1]:\n", + " print(vectorizer.get_feature_names()[i])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### sieć neuronowa" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [], + "source": [ + "class NeuralNetworkModel(torch.nn.Module):\n", + "\n", + " def __init__(self):\n", + " super(NeuralNetworkModel, self).__init__()\n", + " self.fc1 = torch.nn.Linear(FEAUTERES,500)\n", + " self.fc2 = torch.nn.Linear(500,1)\n", + "\n", + " def forward(self, x):\n", + " x = self.fc1(x)\n", + " x = self.fc2(x)\n", + " x = torch.sigmoid(x)\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [], + "source": [ + "nn_model = NeuralNetworkModel()" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [], + "source": [ + "BATCH_SIZE = 5" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": {}, + "outputs": [], + "source": [ + "criterion = torch.nn.BCELoss()" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = torch.optim.SGD(nn_model.parameters(), lr = 0.1)" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "0" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(0.6605833534551934, 0.5908529048207664)" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(0.6379233609747004, 0.6481481481481481)" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "1" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(0.4341224195120214, 0.896168108776267)" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(0.3649017943276299, 0.9074074074074074)" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "2" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(0.18619558424660096, 0.9765142150803461)" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(0.16293201995668588, 0.9888888888888889)" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "3" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(0.09108264647580784, 0.9962917181705809)" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(0.08985773311858927, 0.9962962962962963)" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "4" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(0.053487053708540566, 0.9987639060568603)" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(0.05794332528279887, 1.0)" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "for epoch in range(5):\n", + " loss_score = 0\n", + " acc_score = 0\n", + " items_total = 0\n", + " nn_model.train()\n", + " for i in range(0, Y_train.shape[0], BATCH_SIZE):\n", + " X = X_train[i:i+BATCH_SIZE]\n", + " X = torch.tensor(X.astype(np.float32).todense())\n", + " Y = Y_train[i:i+BATCH_SIZE]\n", + " Y = torch.tensor(Y.astype(np.float32)).reshape(-1,1)\n", + " Y_predictions = nn_model(X)\n", + " acc_score += torch.sum((Y_predictions > 0.5) == Y).item()\n", + " items_total += Y.shape[0] \n", + "\n", + " optimizer.zero_grad()\n", + " loss = criterion(Y_predictions, Y)\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + "\n", + " loss_score += loss.item() * Y.shape[0] \n", + "\n", + " display(epoch)\n", + " display(get_loss_acc(nn_model, X_train, Y_train))\n", + " display(get_loss_acc(nn_model, X_dev, Y_dev))" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(0.16834938257537793, 0.9428172942817294)" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "get_loss_acc(nn_model, X_test, Y_test)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Zadanie domowe\n", + "\n", + "- wybrać jedno z poniższych repozytoriów i je sforkować:\n", + " - https://git.wmi.amu.edu.pl/kubapok/paranormal-or-skeptic-ISI-public\n", + " - https://git.wmi.amu.edu.pl/kubapok/sport-text-classification-ball-ISI-public\n", + "- stworzyć klasyfikator bazujący na prostej sieci neuronowej feed forward w pytorchu (można bazować na tym jupyterze). Zamiast tfidf proszę skorzystać z jakieś reprezentacji gęstej (np. word2vec).\n", + "- stworzyć predykcje w plikach dev-0/out.tsv oraz test-A/out.tsv\n", + "- wynik accuracy sprawdzony za pomocą narzędzia geval (patrz poprzednie zadanie) powinien wynosić conajmniej 0.67\n", + "- proszę umieścić predykcję oraz skrypty generujące (w postaci tekstowej a nie jupyter) w repo, a w MS TEAMS umieścić link do swojego repo\n", + "termin 25.05, 70 punktów\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/cw/08_regresja_logistyczna_ODPOWIEDZI.ipynb b/cw/08_regresja_logistyczna_ODPOWIEDZI.ipynb new file mode 100644 index 0000000..dba395f --- /dev/null +++ b/cw/08_regresja_logistyczna_ODPOWIEDZI.ipynb @@ -0,0 +1,1242 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Regresja logistyczna" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## import bibliotek" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/media/kuba/ssdsam/anaconda3/lib/python3.8/site-packages/gensim/similarities/__init__.py:15: UserWarning: The gensim.similarities.levenshtein submodule is disabled, because the optional Levenshtein package is unavailable. Install Levenhstein (e.g. `pip install python-Levenshtein`) to suppress this warning.\n", + " warnings.warn(msg)\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "import gensim\n", + "import torch\n", + "import pandas as pd\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "from sklearn.datasets import fetch_20newsgroups\n", + "# https://scikit-learn.org/0.19/datasets/twenty_newsgroups.html\n", + "\n", + "from sklearn.feature_extraction.text import TfidfVectorizer\n", + "from sklearn.metrics import accuracy_score" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "CATEGORIES = ['soc.religion.christian', 'alt.atheism']" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "newsgroups_train_dev = fetch_20newsgroups(subset = 'train', categories=CATEGORIES)\n", + "newsgroups_test = fetch_20newsgroups(subset = 'test', categories=CATEGORIES)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "newsgroups_train_dev_text = newsgroups_train_dev['data']\n", + "newsgroups_test_text = newsgroups_test['data']" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "Y_train_dev = newsgroups_train_dev['target']\n", + "Y_test = newsgroups_test['target']" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "newsgroups_train_text, newsgroups_dev_text, Y_train, Y_dev = train_test_split(newsgroups_train_dev_text, Y_train_dev, random_state=42)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "Y_names = newsgroups_train_dev['target_names']" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['alt.atheism', 'soc.religion.christian']" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Y_names" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## baseline" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1,\n", + " 1, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0,\n", + " 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0,\n", + " 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1,\n", + " 0, 0, 1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1,\n", + " 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0,\n", + " 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0,\n", + " 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1,\n", + " 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1,\n", + " 1, 0, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0,\n", + " 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0,\n", + " 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0,\n", + " 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0,\n", + " 1, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0,\n", + " 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1,\n", + " 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1,\n", + " 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 1,\n", + " 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 0,\n", + " 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1,\n", + " 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1,\n", + " 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1,\n", + " 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1,\n", + " 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0,\n", + " 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0,\n", + " 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0,\n", + " 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1,\n", + " 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1,\n", + " 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 0,\n", + " 1, 0, 1, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0,\n", + " 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0,\n", + " 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1,\n", + " 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1,\n", + " 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1,\n", + " 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1,\n", + " 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1,\n", + " 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0])" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Y_train" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1 450\n", + "0 359\n", + "dtype: int64" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pd.value_counts(Y_train)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### train" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.5562422744128553" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "accuracy_score(np.ones_like(Y_train) * 1, Y_train)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### dev" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.5518518518518518" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "accuracy_score(np.ones_like(Y_dev) * 1, Y_dev)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### test" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.5550906555090656" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "accuracy_score(np.ones_like(Y_test) * 1, Y_test)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### PYTANIE: co jest nie tak z regresją liniową?" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Regresja logistyczna" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### wektoryzacja" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "FEAUTERES = 10_000" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "vectorizer = TfidfVectorizer(max_features=10_000)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "X_train = vectorizer.fit_transform(newsgroups_train_text)\n", + "X_dev = vectorizer.transform(newsgroups_dev_text)\n", + "X_test = vectorizer.transform(newsgroups_test_text)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "<717x10000 sparse matrix of type ''\n", + "\twith 120739 stored elements in Compressed Sparse Row format>" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X_test" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### model - inicjalizacja " + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "class LogisticRegressionModel(torch.nn.Module):\n", + "\n", + " def __init__(self):\n", + " super(LogisticRegressionModel, self).__init__()\n", + " self.fc = torch.nn.Linear(FEAUTERES,1)\n", + "\n", + " def forward(self, x):\n", + " x = self.fc(x)\n", + " x = torch.sigmoid(x)\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "lr_model = LogisticRegressionModel()" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.4978],\n", + " [0.5009],\n", + " [0.4998],\n", + " [0.4990],\n", + " [0.5018]], grad_fn=)" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "lr_model(torch.Tensor(X_train[0:5].astype(np.float32).todense()))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "LogisticRegressionModel(\n", + " (fc): Linear(in_features=10000, out_features=1, bias=True)\n", + ")" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "lr_model" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Parameter containing:\n", + " tensor([[-0.0059, 0.0035, 0.0021, ..., -0.0042, -0.0057, -0.0049]],\n", + " requires_grad=True),\n", + " Parameter containing:\n", + " tensor([-0.0023], requires_grad=True)]" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(lr_model.parameters())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## model - trenowanie" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "BATCH_SIZE = 5" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "criterion = torch.nn.BCELoss()" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = torch.optim.SGD(lr_model.parameters(), lr = 0.1)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "809" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Y_train.shape[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "loss_score = 0\n", + "acc_score = 0\n", + "items_total = 0\n", + "lr_model.train()\n", + "for i in range(0, Y_train.shape[0], BATCH_SIZE):\n", + " X = X_train[i:i+BATCH_SIZE]\n", + " X = torch.tensor(X.astype(np.float32).todense())\n", + " Y = Y_train[i:i+BATCH_SIZE]\n", + " Y = torch.tensor(Y.astype(np.float32)).reshape(-1,1)\n", + " Y_predictions = lr_model(X)\n", + " acc_score += torch.sum((Y_predictions > 0.5) == Y).item()\n", + " items_total += Y.shape[0] \n", + " \n", + " optimizer.zero_grad()\n", + " loss = criterion(Y_predictions, Y)\n", + " loss.backward()\n", + " optimizer.step()\n", + " \n", + "\n", + " loss_score += loss.item() * Y.shape[0] " + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.5667],\n", + " [0.5802],\n", + " [0.5757],\n", + " [0.5670]], grad_fn=)" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Y_predictions" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.],\n", + " [1.],\n", + " [1.],\n", + " [0.]])" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Y" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "452" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "acc_score" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "809" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "items_total" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "accuracy: 0.5587144622991347\n" + ] + } + ], + "source": [ + "print(f'accuracy: {acc_score / items_total}')" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "BCE loss: 0.6745463597170355\n" + ] + } + ], + "source": [ + "print(f'BCE loss: {loss_score / items_total}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### model - ewaluacja" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [], + "source": [ + "def get_loss_acc(model, X_dataset, Y_dataset):\n", + " loss_score = 0\n", + " acc_score = 0\n", + " items_total = 0\n", + " model.eval()\n", + " for i in range(0, Y_dataset.shape[0], BATCH_SIZE):\n", + " X = X_dataset[i:i+BATCH_SIZE]\n", + " X = torch.tensor(X.astype(np.float32).todense())\n", + " Y = Y_dataset[i:i+BATCH_SIZE]\n", + " Y = torch.tensor(Y.astype(np.float32)).reshape(-1,1)\n", + " Y_predictions = model(X)\n", + " acc_score += torch.sum((Y_predictions > 0.5) == Y).item()\n", + " items_total += Y.shape[0] \n", + "\n", + " optimizer.zero_grad()\n", + " loss = criterion(Y_predictions, Y)\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + "\n", + " loss_score += loss.item() * Y.shape[0] \n", + " return (loss_score / items_total), (acc_score / items_total)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(0.6443227143826974, 0.622991347342398)" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "get_loss_acc(lr_model, X_train, Y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(0.6369243131743537, 0.6037037037037037)" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "get_loss_acc(lr_model, X_dev, Y_dev)" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(0.6323775731785694, 0.6499302649930265)" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "get_loss_acc(lr_model, X_test, Y_test)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### wagi modelu" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Parameter containing:\n", + " tensor([[ 0.0314, -0.0375, 0.0131, ..., -0.0057, -0.0008, -0.0089]],\n", + " requires_grad=True),\n", + " Parameter containing:\n", + " tensor([0.0563], requires_grad=True)]" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(lr_model.parameters())" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([ 0.0314, -0.0375, 0.0131, ..., -0.0057, -0.0008, -0.0089],\n", + " grad_fn=)" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(lr_model.parameters())[0][0]" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.return_types.topk(\n", + "values=tensor([0.3753, 0.2305, 0.2007, 0.2006, 0.1993, 0.1952, 0.1930, 0.1898, 0.1831,\n", + " 0.1731, 0.1649, 0.1647, 0.1543, 0.1320, 0.1314, 0.1303, 0.1296, 0.1261,\n", + " 0.1245, 0.1243], grad_fn=),\n", + "indices=tensor([8942, 6336, 1852, 9056, 1865, 4039, 7820, 5002, 8208, 1857, 9709, 803,\n", + " 1046, 130, 4306, 6481, 4370, 4259, 4285, 1855]))" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.topk(list(lr_model.parameters())[0][0], 20)" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "the\n", + "of\n", + "christ\n", + "to\n", + "church\n", + "god\n", + "rutgers\n", + "jesus\n", + "sin\n", + "christians\n", + "we\n", + "and\n", + "athos\n", + "1993\n", + "hell\n", + "our\n", + "his\n", + "he\n", + "heaven\n", + "christian\n" + ] + } + ], + "source": [ + "for i in torch.topk(list(lr_model.parameters())[0][0], 20)[1]:\n", + " print(vectorizer.get_feature_names()[i])" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.return_types.topk(\n", + "values=tensor([-0.3478, -0.2578, -0.2455, -0.2347, -0.2330, -0.2265, -0.2205, -0.2050,\n", + " -0.2044, -0.1979, -0.1876, -0.1790, -0.1747, -0.1745, -0.1734, -0.1647,\n", + " -0.1639, -0.1617, -0.1601, -0.1592], grad_fn=),\n", + "indices=tensor([5119, 8096, 5420, 4436, 6194, 1627, 6901, 5946, 9970, 3116, 1036, 9906,\n", + " 5654, 8329, 7869, 1039, 1991, 4926, 5035, 4925]))" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.topk(list(lr_model.parameters())[0][0], 20, largest = False)" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "keith\n", + "sgi\n", + "livesey\n", + "host\n", + "nntp\n", + "caltech\n", + "posting\n", + "morality\n", + "you\n", + "edu\n", + "atheism\n", + "wpd\n", + "mathew\n", + "solntze\n", + "sandvik\n", + "atheists\n", + "com\n", + "islamic\n", + "jon\n", + "islam\n" + ] + } + ], + "source": [ + "for i in torch.topk(list(lr_model.parameters())[0][0], 20, largest = False)[1]:\n", + " print(vectorizer.get_feature_names()[i])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### sieć neuronowa" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [], + "source": [ + "class NeuralNetworkModel(torch.nn.Module):\n", + "\n", + " def __init__(self):\n", + " super(NeuralNetworkModel, self).__init__()\n", + " self.fc1 = torch.nn.Linear(FEAUTERES,500)\n", + " self.fc2 = torch.nn.Linear(500,1)\n", + "\n", + " def forward(self, x):\n", + " x = self.fc1(x)\n", + " x = self.fc2(x)\n", + " x = torch.sigmoid(x)\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [], + "source": [ + "nn_model = NeuralNetworkModel()" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [], + "source": [ + "BATCH_SIZE = 5" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": {}, + "outputs": [], + "source": [ + "criterion = torch.nn.BCELoss()" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = torch.optim.SGD(nn_model.parameters(), lr = 0.1)" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "0" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(0.6605833534551934, 0.5908529048207664)" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(0.6379233609747004, 0.6481481481481481)" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "1" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(0.4341224195120214, 0.896168108776267)" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(0.3649017943276299, 0.9074074074074074)" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "2" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(0.18619558424660096, 0.9765142150803461)" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(0.16293201995668588, 0.9888888888888889)" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "3" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(0.09108264647580784, 0.9962917181705809)" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(0.08985773311858927, 0.9962962962962963)" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "4" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(0.053487053708540566, 0.9987639060568603)" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(0.05794332528279887, 1.0)" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "for epoch in range(5):\n", + " loss_score = 0\n", + " acc_score = 0\n", + " items_total = 0\n", + " nn_model.train()\n", + " for i in range(0, Y_train.shape[0], BATCH_SIZE):\n", + " X = X_train[i:i+BATCH_SIZE]\n", + " X = torch.tensor(X.astype(np.float32).todense())\n", + " Y = Y_train[i:i+BATCH_SIZE]\n", + " Y = torch.tensor(Y.astype(np.float32)).reshape(-1,1)\n", + " Y_predictions = nn_model(X)\n", + " acc_score += torch.sum((Y_predictions > 0.5) == Y).item()\n", + " items_total += Y.shape[0] \n", + "\n", + " optimizer.zero_grad()\n", + " loss = criterion(Y_predictions, Y)\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + "\n", + " loss_score += loss.item() * Y.shape[0] \n", + "\n", + " display(epoch)\n", + " display(get_loss_acc(nn_model, X_train, Y_train))\n", + " display(get_loss_acc(nn_model, X_dev, Y_dev))" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(0.16834938257537793, 0.9428172942817294)" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "get_loss_acc(nn_model, X_test, Y_test)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Zadanie domowe\n", + "\n", + "- wybrać jedno z poniższych repozytoriów i je sforkować:\n", + " - https://git.wmi.amu.edu.pl/kubapok/paranormal-or-skeptic-ISI-public\n", + " - https://git.wmi.amu.edu.pl/kubapok/sport-text-classification-ball-ISI-public\n", + "- stworzyć klasyfikator bazujący na prostej sieci neuronowej feed forward w pytorchu (można bazować na tym jupyterze). Zamiast tfidf proszę skorzystać z jakieś reprezentacji gęstej (np. word2vec).\n", + "- stworzyć predykcje w plikach dev-0/out.tsv oraz test-A/out.tsv\n", + "- wynik accuracy sprawdzony za pomocą narzędzia geval (patrz poprzednie zadanie) powinien wynosić conajmniej 0.67\n", + "- proszę umieścić predykcję oraz skrypty generujące (w postaci tekstowej a nie jupyter) w repo, a w MS TEAMS umieścić link do swojego repo\n", + "termin 25.05, 70 punktów\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}