{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "32d973e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from load_data import get_dataset\n",
    "import numpy as np\n",
    "from tabulate import tabulate\n",
    "\n",
    "X_train, y_train, X_test, y_test = get_dataset(new_size=64)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e63a5e75-7a26-4a97-b53d-67b858345126",
   "metadata": {},
   "source": [
    "# Zadanie 1 (2 pkt)\n",
    "\n",
    "Rozwiń algorytm regresji logistycznej z lab. 1, wprowadzając do niego człon regularyzacyjny."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ea300c45",
   "metadata": {},
   "outputs": [],
   "source": [
    "class LogisticRegression():\n",
    "    def __init__(self, l2=1):\n",
    "        self.l2 = l2\n",
    "\n",
    "    def mapY(self, y, cls):\n",
    "        m = len(y)\n",
    "        yBi = np.matrix(np.zeros(m)).reshape(m, 1)\n",
    "        yBi[y == cls] = 1.\n",
    "        return yBi\n",
    "\n",
    "    def indicatorMatrix(self, y):\n",
    "        classes = np.unique(y.tolist())\n",
    "        m, k = len(y), len(classes)\n",
    "        # k = len(classes)\n",
    "        Y = np.matrix(np.zeros((m, k)))\n",
    "        for i, cls in enumerate(classes):\n",
    "            Y[:, i] = self.mapY(y, cls)\n",
    "        return Y\n",
    "    \n",
    "    # Zapis macierzowy funkcji softmax\n",
    "    def softmax(self, X):\n",
    "        return np.exp(X) / np.sum(np.exp(X))\n",
    "    \n",
    "    # Funkcja regresji logistcznej\n",
    "    def h(self, theta, X):\n",
    "        return 1.0 /(1.0 + np.exp(-X * theta))\n",
    "    \n",
    "    # Funkcja kosztu dla regresji logistycznej\n",
    "    def J(self, h, theta, X, y):\n",
    "        m = len(y)\n",
    "        h_val = h(theta, X)\n",
    "        s1 = np.multiply(y, np.log(h_val))\n",
    "        s2 = np.multiply((1 - y), np.log(1 - h_val))\n",
    "        s3 = np.sum(s1+s2, axis=0)/m\n",
    "        s4 = (self.l2 * np.sum(np.square(theta))) / 2*m\n",
    "        return -s3 + s4\n",
    "\n",
    "    # Gradient dla regresji logistycznej\n",
    "    def dJ(self, h, theta, X, y):\n",
    "        return 1.0 / (self.l2/len(y)) * (X.T * (h(theta, X) - y))\n",
    "\n",
    "    # Metoda gradientu prostego dla regresji logistycznej\n",
    "    def GD(self, h, fJ, fdJ, theta, X, y, alpha=0.01, eps=10**-3, maxSteps=1000):\n",
    "        errorCurr = fJ(h, theta, X, y)\n",
    "        errors = [[errorCurr, theta]]\n",
    "        while True:\n",
    "            # oblicz nowe theta\n",
    "            theta = theta - alpha * fdJ(h, theta, X, y)\n",
    "            # raportuj poziom błędu\n",
    "            errorCurr, errorPrev = fJ(h, theta, X, y), errorCurr\n",
    "            # kryteria stopu\n",
    "            if abs(errorPrev - errorCurr) <= eps:\n",
    "                break\n",
    "            if len(errors) > maxSteps:\n",
    "                break\n",
    "            errors.append([errorCurr, theta]) \n",
    "        # return theta, errors\n",
    "        return theta\n",
    "\n",
    "    def trainMaxEnt(self, X, Y):\n",
    "        n = X.shape[1]\n",
    "        thetas = []\n",
    "        for c in range(Y.shape[1]):\n",
    "            YBi = Y[:,c]\n",
    "            theta = np.matrix(np.random.random(n)).reshape(n,1)\n",
    "            # Macierz parametrów theta obliczona dla każdej klasy osobno.\n",
    "            # thetaBest, errors = self.GD(self.h, self.J, self.dJ, theta, \n",
    "            #                     X, YBi, alpha=0.1, eps=10**-4)\n",
    "            # thetas.append(thetaBest)\n",
    "            thetas.append(self.GD(self.h, self.J, self.dJ, theta, X, YBi, alpha=0.1, eps=10**-4))\n",
    "        return thetas\n",
    "\n",
    "    def classify(self, thetas, X):\n",
    "        regs = np.array([(X*theta).item() for theta in thetas])\n",
    "        return np.argmax(self.softmax(regs))\n",
    "        # probs = self.softmax(regs)\n",
    "        # result = np.argmax(probs)\n",
    "        # return result\n",
    "\n",
    "    def class_score(self, expected, predicted):\n",
    "        # accuracy = TP + TN / FP + FN + TP + TN\n",
    "        accuracy = sum(1 for exp, pred in zip(expected, predicted) if exp == pred) / len(expected)\n",
    "        # precision = TP / FP + TP\n",
    "        precision = sum(\n",
    "            1 for exp, pred in zip(expected, predicted) if exp == 1.0 and pred == 1.0) / sum(\n",
    "                1 for exp, pred in zip(expected, predicted) if exp == 1.0)\n",
    "        # recall = TP / FN + TP\n",
    "        recall = sum(\n",
    "            1 for exp, pred in zip(expected, predicted) if exp == 1.0 and pred == 1.0) / sum(\n",
    "                1 for exp, pred in zip(expected, predicted) if pred == 1.0)\n",
    "        f1 = (2 * precision * recall) / (precision + recall)\n",
    "        return accuracy, precision, recall, f1\n",
    "\n",
    "    def fit(self, X_train, y_train):\n",
    "        # Y = self.indicatorMatrix(y_train)\n",
    "        # self.thetas = self.trainMaxEnt(X_train, Y)\n",
    "        self.thetas = self.trainMaxEnt(X_train, self.indicatorMatrix(y_train))\n",
    "\n",
    "    def predict(self, X_test):\n",
    "        return np.array([self.classify(self.thetas, x) for x in X_test])\n",
    "\n",
    "    def accuracy(self, expected, predicted):\n",
    "        return sum(1 for x, y in zip(expected, predicted) if x == y) / len(expected)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "84fc2187",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/3r/c8tg1h051m18qhsdccdysrt40000gn/T/ipykernel_43367/551403934.py:33: RuntimeWarning: divide by zero encountered in log\n",
      "  s2 = np.multiply((1 - y), np.log(1 - h_val))\n",
      "/var/folders/3r/c8tg1h051m18qhsdccdysrt40000gn/T/ipykernel_43367/551403934.py:33: RuntimeWarning: invalid value encountered in multiply\n",
      "  s2 = np.multiply((1 - y), np.log(1 - h_val))\n",
      "/var/folders/3r/c8tg1h051m18qhsdccdysrt40000gn/T/ipykernel_43367/551403934.py:26: RuntimeWarning: overflow encountered in exp\n",
      "  return 1.0 /(1.0 + np.exp(-X * theta))\n",
      "/var/folders/3r/c8tg1h051m18qhsdccdysrt40000gn/T/ipykernel_43367/551403934.py:32: RuntimeWarning: divide by zero encountered in log\n",
      "  s1 = np.multiply(y, np.log(h_val))\n",
      "/var/folders/3r/c8tg1h051m18qhsdccdysrt40000gn/T/ipykernel_43367/551403934.py:32: RuntimeWarning: invalid value encountered in multiply\n",
      "  s1 = np.multiply(y, np.log(h_val))\n"
     ]
    }
   ],
   "source": [
    "# 16x16, l2=0.01 -> 5m 11.4s\n",
    "# 32x32, l2=0.1 -> 20m 31.3s\n",
    "# 64x64, l2=0.1 -> 219m 10.8s\n",
    "logreg = LogisticRegression(l2=0.1)\n",
    "logreg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f2a4b169",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/3r/c8tg1h051m18qhsdccdysrt40000gn/T/ipykernel_43367/551403934.py:22: RuntimeWarning: invalid value encountered in true_divide\n",
      "  return np.exp(X) / np.sum(np.exp(X))\n",
      "/var/folders/3r/c8tg1h051m18qhsdccdysrt40000gn/T/ipykernel_43367/551403934.py:22: RuntimeWarning: overflow encountered in exp\n",
      "  return np.exp(X) / np.sum(np.exp(X))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.32432432432432434"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logreg.accuracy(y_test, logreg.predict(X_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "945d1169-a13c-44a5-ad51-73ec62438487",
   "metadata": {},
   "source": [
    "# Zadanie 2 (4 pkt)\n",
    "\n",
    "Zaimplementuj algorytm SVM z miękkim marginesem (regularyzacją)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7a58dff1-f85d-4e07-b5e4-b639fc2af3ac",
   "metadata": {
    "deletable": false,
    "nbgrader": {
     "cell_type": "code",
     "checksum": "d0e86f0fc352ea3a95940ad21ba9cd6c",
     "grade": true,
     "grade_id": "cell-2e22217733fd5f34",
     "locked": false,
     "points": 4,
     "schema_version": 3,
     "solution": true,
     "task": false
    }
   },
   "outputs": [],
   "source": [
    "class SVM():\n",
    "    def __init__(self, lr=0.001, lambda_param=10**-6, n_iters=1000):\n",
    "        self.lr = lr\n",
    "        self.lambda_param = lambda_param\n",
    "        self.n_iters = n_iters\n",
    "\n",
    "    def _mapY(self, y, cls):\n",
    "        m = len(y)\n",
    "        yBi = np.matrix(np.zeros(m)).reshape(m, 1)\n",
    "        yBi[y == cls] = 1.\n",
    "        return yBi\n",
    "\n",
    "    def _indicatorMatrix(self, y):\n",
    "        classes = np.unique(y.tolist())\n",
    "        m, k = len(y), len(classes)\n",
    "        Y = np.matrix(np.zeros((m, k)))\n",
    "        for i, cls in enumerate(classes):\n",
    "            Y[:, i] = self._mapY(y, cls)\n",
    "        return Y\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        n_classes = len(np.unique(y))\n",
    "        y = self._indicatorMatrix(y)\n",
    "        y = np.where(y == 0, -1, 1)\n",
    "\n",
    "        n_features = X.shape[1]\n",
    "        self.weights, self.biases = [], []\n",
    "\n",
    "        for cls in range(n_classes):\n",
    "            y_ = y[:,cls]\n",
    "            y_ = np.where(y_ <= 0, -1, 1)\n",
    "\n",
    "            w, b = np.zeros(n_features), 0\n",
    "\n",
    "            for _ in range(self.n_iters):\n",
    "                for idx, x_i in enumerate(X):\n",
    "                    condition = y_[idx] * (np.dot(x_i, w) - b) >= 1\n",
    "                    if condition:\n",
    "                        w -= self.lr * (2 * self.lambda_param * w)\n",
    "                    else:\n",
    "                        w -= self.lr * (2 * self.lambda_param * w - np.dot(x_i, y_[idx]))\n",
    "                        b -= self.lr * y_[idx]\n",
    "            self.weights.append(w)\n",
    "            self.biases.append(b)\n",
    "\n",
    "    def _classify(self, x):\n",
    "        cls = [np.sign(np.dot(x, self.weights[i]) - self.biases[i]) for i in range(len(self.biases))]\n",
    "        return cls.index(1.0) if 1.0 in cls else 0\n",
    "\n",
    "    def predict(self, X):\n",
    "        return list(map(lambda x: self._classify(x), X))\n",
    "\n",
    "    def accuracy(self, expected, predicted):\n",
    "        return sum(1 for x, y in zip(expected, predicted) if x == y) / len(expected)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "62857aa5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 16x16 -> 1m 28.8s\n",
    "# 32x32 -> 2m 2.2s\n",
    "# 64x64 -> 4m 55.3s\n",
    "\n",
    "svm = SVM()\n",
    "svm.fit(X_train, y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "27bdbf4f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.32432432432432434"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm.accuracy(y_test, svm.predict(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0aa0c9d6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "vscode": {
   "interpreter": {
    "hash": "83181d593ff87630d4a0d7997796f399a1d96b6a7eabd5b16ad531f517f6300b"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}