From 5bea1088e3640cd54585f5a3aff8be023c35d6dc Mon Sep 17 00:00:00 2001 From: s444391 Date: Sat, 10 Dec 2022 00:55:36 +0100 Subject: [PATCH] add lab8 draft --- sw_lab8.ipynb | 1132 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1132 insertions(+) create mode 100644 sw_lab8.ipynb diff --git a/sw_lab8.ipynb b/sw_lab8.ipynb new file mode 100644 index 0000000..076663a --- /dev/null +++ b/sw_lab8.ipynb @@ -0,0 +1,1132 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Zadanie 8 - Alexnet + Dropout & BatchRegularization\n", + "### Aleksandra Jonas, Aleksandra Gronowska, Iwona Christop" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Przygotowanie danych" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import Image, display" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "2fe63b50", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "import subprocess\n", + "import pkg_resources\n", + "import numpy as np\n", + "\n", + "required = { 'scikit-image'}\n", + "installed = {pkg.key for pkg in pkg_resources.working_set}\n", + "missing = required - installed\n", + "# Alexnet requires images to be of dim = (227, 227, 3)\n", + "newSize = (227,227)\n", + "\n", + "if missing: \n", + " python = sys.executable\n", + " subprocess.check_call([python, '-m', 'pip', 'install', *missing], stdout=subprocess.DEVNULL)\n", + "\n", + "def load_train_data(input_dir):\n", + " import numpy as np\n", + " import pandas as pd\n", + " import os\n", + " from skimage.io import imread\n", + " import cv2 as cv\n", + " from pathlib import Path\n", + " import random\n", + " from shutil import copyfile, rmtree\n", + " import json\n", + "\n", + " import seaborn as sns\n", + " import matplotlib.pyplot as plt\n", + "\n", + " import matplotlib\n", + " \n", + " image_dir = Path(input_dir)\n", + " categories_name = []\n", + " for file in os.listdir(image_dir):\n", + " d = os.path.join(image_dir, file)\n", + " if os.path.isdir(d):\n", + " categories_name.append(file)\n", + "\n", + " folders = [directory for directory in image_dir.iterdir() if directory.is_dir()]\n", + "\n", + " train_img = []\n", + " categories_count=[]\n", + " labels=[]\n", + " for i, direc in enumerate(folders):\n", + " count = 0\n", + " for obj in direc.iterdir():\n", + " if os.path.isfile(obj) and os.path.basename(os.path.normpath(obj)) != 'desktop.ini':\n", + " labels.append(os.path.basename(os.path.normpath(direc)))\n", + " count += 1\n", + " img = imread(obj)#zwraca ndarry postaci xSize x ySize x colorDepth\n", + " img = img[:, :, :3]\n", + " img = cv.resize(img, newSize, interpolation=cv.INTER_AREA)# zwraca ndarray\n", + " img = img / 255 #normalizacja\n", + " train_img.append(img)\n", + " categories_count.append(count)\n", + " X={}\n", + " X[\"values\"] = np.array(train_img)\n", + " X[\"categories_name\"] = categories_name\n", + " X[\"categories_count\"] = categories_count\n", + " X[\"labels\"]=labels\n", + " return X\n", + "\n", + "def load_test_data(input_dir):\n", + " import numpy as np\n", + " import pandas as pd\n", + " import os\n", + " from skimage.io import imread\n", + " import cv2 as cv\n", + " from pathlib import Path\n", + " import random\n", + " from shutil import copyfile, rmtree\n", + " import json\n", + "\n", + " import seaborn as sns\n", + " import matplotlib.pyplot as plt\n", + "\n", + " import matplotlib\n", + "\n", + " image_path = Path(input_dir)\n", + "\n", + " labels_path = image_path.parents[0] / 'test_labels.json'\n", + "\n", + " jsonString = labels_path.read_text()\n", + " objects = json.loads(jsonString)\n", + "\n", + " categories_name = []\n", + " categories_count=[]\n", + " count = 0\n", + " c = objects[0]['value']\n", + " for e in objects:\n", + " if e['value'] != c:\n", + " categories_count.append(count)\n", + " c = e['value']\n", + " count = 1\n", + " else:\n", + " count += 1\n", + " if not e['value'] in categories_name:\n", + " categories_name.append(e['value'])\n", + "\n", + " categories_count.append(count)\n", + " \n", + " test_img = []\n", + "\n", + " labels=[]\n", + " for e in objects:\n", + " p = image_path / e['filename']\n", + " img = imread(p)#zwraca ndarry postaci xSize x ySize x colorDepth\n", + " img = img[:, :, :3]\n", + " img = cv.resize(img, newSize, interpolation=cv.INTER_AREA)# zwraca ndarray\n", + " img = img / 255#normalizacja\n", + " test_img.append(img)\n", + " labels.append(e['value'])\n", + "\n", + " X={}\n", + " X[\"values\"] = np.array(test_img)\n", + " X[\"categories_name\"] = categories_name\n", + " X[\"categories_count\"] = categories_count\n", + " X[\"labels\"]=labels\n", + " return X" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "cc941c5a", + "metadata": {}, + "outputs": [], + "source": [ + "# Data load\n", + "data_train = load_train_data(\"./train_test_sw/train_sw\")\n", + "values_train = data_train['values']\n", + "labels_train = data_train['labels']\n", + "\n", + "data_test = load_test_data(\"./train_test_sw/test_sw\")\n", + "X_test = data_test['values']\n", + "y_test = data_test['labels']" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "25040ac9", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "18d44949", + "metadata": {}, + "outputs": [], + "source": [ + "X_train, X_validate, y_train, y_validate = train_test_split(values_train, labels_train, test_size=0.2, random_state=42)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "a1fe47e6", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.preprocessing import LabelEncoder" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "d90af799", + "metadata": {}, + "outputs": [], + "source": [ + "class_le = LabelEncoder()\n", + "y_train_enc = class_le.fit_transform(y_train)\n", + "y_validate_enc = class_le.fit_transform(y_validate)\n", + "y_test_enc = class_le.fit_transform(y_test)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "c2323985", + "metadata": {}, + "outputs": [], + "source": [ + "import tensorflow as tf" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "dfe674dc", + "metadata": {}, + "outputs": [], + "source": [ + "train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train_enc))\n", + "validation_ds = tf.data.Dataset.from_tensor_slices((X_validate, y_validate_enc))\n", + "test_ds = tf.data.Dataset.from_tensor_slices((X_test, y_test_enc))" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "076c8ac9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training data size: 820\n", + "Test data size: 259\n", + "Validation data size: 206\n" + ] + } + ], + "source": [ + "train_ds_size = tf.data.experimental.cardinality(train_ds).numpy()\n", + "test_ds_size = tf.data.experimental.cardinality(test_ds).numpy()\n", + "validation_ds_size = tf.data.experimental.cardinality(validation_ds).numpy()\n", + "print(\"Training data size:\", train_ds_size)\n", + "print(\"Test data size:\", test_ds_size)\n", + "print(\"Validation data size:\", validation_ds_size)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "07ebcd4a", + "metadata": {}, + "outputs": [], + "source": [ + "train_ds = (train_ds\n", + " .shuffle(buffer_size=train_ds_size)\n", + " .batch(batch_size=32, drop_remainder=True))\n", + "test_ds = (test_ds\n", + " .shuffle(buffer_size=train_ds_size)\n", + " .batch(batch_size=32, drop_remainder=True))\n", + "validation_ds = (validation_ds\n", + " .shuffle(buffer_size=train_ds_size)\n", + " .batch(batch_size=32, drop_remainder=True))" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "from tensorflow import keras\n", + "import os\n", + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "root_logdir = os.path.join(os.curdir, \"logs\\\\fit\\\\\")\n", + "def get_run_logdir():\n", + " run_id = time.strftime(\"run_%Y_%m_%d-%H_%M_%S\")\n", + " return os.path.join(root_logdir, run_id)\n", + "run_logdir = get_run_logdir()\n", + "tensorboard_cb = keras.callbacks.TensorBoard(run_logdir)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Dropout" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Do warstw spłaszczonych" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "model_flat_drop = keras.models.Sequential([\n", + " keras.layers.Conv2D(filters=96, kernel_size=(11,11), strides=(4,4), activation='relu', input_shape=(227,227,3)),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Conv2D(filters=256, kernel_size=(5,5), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Conv2D(filters=256, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Flatten(),\n", + " keras.layers.Dense(4096, activation='relu'),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Dense(4096, activation='relu'),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Dense(10, activation='softmax')\n", + "])" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:`lr` is deprecated, please use `learning_rate` instead, or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.SGD.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model: \"sequential\"\n", + "_________________________________________________________________\n", + " Layer (type) Output Shape Param # \n", + "=================================================================\n", + " conv2d (Conv2D) (None, 55, 55, 96) 34944 \n", + " \n", + " max_pooling2d (MaxPooling2D (None, 27, 27, 96) 0 \n", + " ) \n", + " \n", + " conv2d_1 (Conv2D) (None, 27, 27, 256) 614656 \n", + " \n", + " max_pooling2d_1 (MaxPooling (None, 13, 13, 256) 0 \n", + " 2D) \n", + " \n", + " conv2d_2 (Conv2D) (None, 13, 13, 384) 885120 \n", + " \n", + " conv2d_3 (Conv2D) (None, 13, 13, 384) 1327488 \n", + " \n", + " conv2d_4 (Conv2D) (None, 13, 13, 256) 884992 \n", + " \n", + " max_pooling2d_2 (MaxPooling (None, 6, 6, 256) 0 \n", + " 2D) \n", + " \n", + " flatten (Flatten) (None, 9216) 0 \n", + " \n", + " dense (Dense) (None, 4096) 37752832 \n", + " \n", + " dropout (Dropout) (None, 4096) 0 \n", + " \n", + " dense_1 (Dense) (None, 4096) 16781312 \n", + " \n", + " dropout_1 (Dropout) (None, 4096) 0 \n", + " \n", + " dense_2 (Dense) (None, 10) 40970 \n", + " \n", + "=================================================================\n", + "Total params: 58,322,314\n", + "Trainable params: 58,322,314\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n" + ] + } + ], + "source": [ + "model_flat_drop.compile(loss='sparse_categorical_crossentropy', optimizer=tf.optimizers.SGD(lr=.001), metrics=['accuracy'])\n", + "model_flat_drop.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/10\n", + "25/25 [==============================] - 41s 2s/step - loss: 2.2561 - accuracy: 0.2300 - val_loss: 2.1845 - val_accuracy: 0.2500\n", + "Epoch 2/10\n", + "25/25 [==============================] - 37s 1s/step - loss: 1.9685 - accuracy: 0.2025 - val_loss: 1.6442 - val_accuracy: 0.3698\n", + "Epoch 3/10\n", + "25/25 [==============================] - 37s 1s/step - loss: 1.6744 - accuracy: 0.2450 - val_loss: 1.5763 - val_accuracy: 0.2396\n", + "Epoch 4/10\n", + "25/25 [==============================] - 37s 1s/step - loss: 1.6398 - accuracy: 0.2562 - val_loss: 1.5639 - val_accuracy: 0.3073\n", + "Epoch 5/10\n", + "25/25 [==============================] - 37s 1s/step - loss: 1.6422 - accuracy: 0.2625 - val_loss: 1.5708 - val_accuracy: 0.2240\n", + "Epoch 6/10\n", + "25/25 [==============================] - 37s 1s/step - loss: 1.6021 - accuracy: 0.2887 - val_loss: 1.5474 - val_accuracy: 0.3698\n", + "Epoch 7/10\n", + "25/25 [==============================] - 37s 1s/step - loss: 1.6052 - accuracy: 0.2675 - val_loss: 1.5356 - val_accuracy: 0.3021\n", + "Epoch 8/10\n", + "25/25 [==============================] - 38s 2s/step - loss: 1.5586 - accuracy: 0.3300 - val_loss: 1.5064 - val_accuracy: 0.4167\n", + "Epoch 9/10\n", + "25/25 [==============================] - 37s 1s/step - loss: 1.5581 - accuracy: 0.3363 - val_loss: 1.4447 - val_accuracy: 0.3646\n", + "Epoch 10/10\n", + "25/25 [==============================] - 38s 2s/step - loss: 1.5023 - accuracy: 0.3625 - val_loss: 1.4218 - val_accuracy: 0.5052\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model_flat_drop.fit(train_ds,\n", + " epochs=100,\n", + " validation_data=validation_ds,\n", + " validation_freq=1,\n", + " callbacks=[tensorboard_cb])" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "8/8 [==============================] - 4s 293ms/step - loss: 1.4377 - accuracy: 0.4258\n" + ] + }, + { + "data": { + "text/plain": [ + "[1.437693476676941, 0.42578125]" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model_flat_drop.evaluate(test_ds)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Do warstw maxpooling" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "model_pool_drop = keras.models.Sequential([\n", + " keras.layers.Conv2D(filters=96, kernel_size=(11,11), strides=(4,4), activation='relu', input_shape=(227,227,3)),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Conv2D(filters=256, kernel_size=(5,5), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Conv2D(filters=256, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Flatten(),\n", + " keras.layers.Dense(4096, activation='relu'),\n", + " keras.layers.Dense(4096, activation='relu'),\n", + " keras.layers.Dense(10, activation='softmax')\n", + "])" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:`lr` is deprecated, please use `learning_rate` instead, or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.SGD.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model: \"sequential_1\"\n", + "_________________________________________________________________\n", + " Layer (type) Output Shape Param # \n", + "=================================================================\n", + " conv2d_5 (Conv2D) (None, 55, 55, 96) 34944 \n", + " \n", + " max_pooling2d_3 (MaxPooling (None, 27, 27, 96) 0 \n", + " 2D) \n", + " \n", + " dropout_2 (Dropout) (None, 27, 27, 96) 0 \n", + " \n", + " conv2d_6 (Conv2D) (None, 27, 27, 256) 614656 \n", + " \n", + " max_pooling2d_4 (MaxPooling (None, 13, 13, 256) 0 \n", + " 2D) \n", + " \n", + " dropout_3 (Dropout) (None, 13, 13, 256) 0 \n", + " \n", + " conv2d_7 (Conv2D) (None, 13, 13, 384) 885120 \n", + " \n", + " conv2d_8 (Conv2D) (None, 13, 13, 384) 1327488 \n", + " \n", + " conv2d_9 (Conv2D) (None, 13, 13, 256) 884992 \n", + " \n", + " max_pooling2d_5 (MaxPooling (None, 6, 6, 256) 0 \n", + " 2D) \n", + " \n", + " dropout_4 (Dropout) (None, 6, 6, 256) 0 \n", + " \n", + " flatten_1 (Flatten) (None, 9216) 0 \n", + " \n", + " dense_3 (Dense) (None, 4096) 37752832 \n", + " \n", + " dense_4 (Dense) (None, 4096) 16781312 \n", + " \n", + " dense_5 (Dense) (None, 10) 40970 \n", + " \n", + "=================================================================\n", + "Total params: 58,322,314\n", + "Trainable params: 58,322,314\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n" + ] + } + ], + "source": [ + "model_pool_drop.compile(loss='sparse_categorical_crossentropy', optimizer=tf.optimizers.SGD(lr=.001), metrics=['accuracy'])\n", + "model_pool_drop.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/10\n", + "25/25 [==============================] - 41s 2s/step - loss: 2.1329 - accuracy: 0.1912 - val_loss: 1.9968 - val_accuracy: 0.1979\n", + "Epoch 2/10\n", + " 4/25 [===>..........................] - ETA: 29s - loss: 1.7623 - accuracy: 0.1953" + ] + } + ], + "source": [ + "model_pool_drop.fit(train_ds,\n", + " epochs=100,\n", + " validation_data=validation_ds,\n", + " validation_freq=1,\n", + " callbacks=[tensorboard_cb])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_pool_drop.evaluate(test_ds)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Do warstw splotowych" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_conv_drop = keras.models.Sequential([\n", + " keras.layers.Conv2D(filters=96, kernel_size=(11,11), strides=(4,4), activation='relu', input_shape=(227,227,3)),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Conv2D(filters=256, kernel_size=(5,5), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Conv2D(filters=256, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Flatten(),\n", + " keras.layers.Dense(4096, activation='relu'),\n", + " keras.layers.Dense(4096, activation='relu'),\n", + " keras.layers.Dense(10, activation='softmax')\n", + "])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_conv_drop.compile(loss='sparse_categorical_crossentropy', optimizer=tf.optimizers.SGD(lr=.001), metrics=['accuracy'])\n", + "model_conv_drop.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_conv_drop.fit(train_ds,\n", + " epochs=100,\n", + " validation_data=validation_ds,\n", + " validation_freq=1,\n", + " callbacks=[tensorboard_cb])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_conv_drop.evaluate(test_ds)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Do warstw spłaszczonych i maxpooling" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_flat_pool_drop = keras.models.Sequential([\n", + " keras.layers.Conv2D(filters=96, kernel_size=(11,11), strides=(4,4), activation='relu', input_shape=(227,227,3)),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Conv2D(filters=256, kernel_size=(5,5), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Conv2D(filters=256, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Flatten(),\n", + " keras.layers.Dense(4096, activation='relu'),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Dense(4096, activation='relu'),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Dense(10, activation='softmax')\n", + "])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_flat_pool_drop.compile(loss='sparse_categorical_crossentropy', optimizer=tf.optimizers.SGD(lr=.001), metrics=['accuracy'])\n", + "model_flat_pool_drop.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_flat_pool_drop.fit(train_ds,\n", + " epochs=100,\n", + " validation_data=validation_ds,\n", + " validation_freq=1,\n", + " callbacks=[tensorboard_cb])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_flat_pool_drop.evaluate(test_ds)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Do warstw spłaszczonych i splotowych" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_flat_conv_drop = keras.models.Sequential([\n", + " keras.layers.Conv2D(filters=96, kernel_size=(11,11), strides=(4,4), activation='relu', input_shape=(227,227,3)),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Conv2D(filters=256, kernel_size=(5,5), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Conv2D(filters=256, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Flatten(),\n", + " keras.layers.Dense(4096, activation='relu'),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Dense(4096, activation='relu'),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Dense(10, activation='softmax')\n", + "])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_flat_conv_drop.compile(loss='sparse_categorical_crossentropy', optimizer=tf.optimizers.SGD(lr=.001), metrics=['accuracy'])\n", + "model_flat_conv_drop.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_flat_conv_drop.fit(train_ds,\n", + " epochs=100,\n", + " validation_data=validation_ds,\n", + " validation_freq=1,\n", + " callbacks=[tensorboard_cb])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_flat_conv_drop.evaluate(test_ds)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Do warstw maxpooling i splotowych" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_pool_conv_drop = keras.models.Sequential([\n", + " keras.layers.Conv2D(filters=96, kernel_size=(11,11), strides=(4,4), activation='relu', input_shape=(227,227,3)),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Conv2D(filters=256, kernel_size=(5,5), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Conv2D(filters=256, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Flatten(),\n", + " keras.layers.Dense(4096, activation='relu'),\n", + " keras.layers.Dense(4096, activation='relu'),\n", + " keras.layers.Dense(10, activation='softmax')\n", + "])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_pool_conv_drop.compile(loss='sparse_categorical_crossentropy', optimizer=tf.optimizers.SGD(lr=.001), metrics=['accuracy'])\n", + "model_pool_conv_drop.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_pool_conv_drop.fit(train_ds,\n", + " epochs=100,\n", + " validation_data=validation_ds,\n", + " validation_freq=1,\n", + " callbacks=[tensorboard_cb])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_pool_conv_drop.evaluate(test_ds)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Do warstw spłaszczonych, maxpooling i splotowych" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_drop = keras.models.Sequential([\n", + " keras.layers.Conv2D(filters=96, kernel_size=(11,11), strides=(4,4), activation='relu', input_shape=(227,227,3)),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Conv2D(filters=256, kernel_size=(5,5), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Conv2D(filters=256, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Flatten(),\n", + " keras.layers.Dense(4096, activation='relu'),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Dense(4096, activation='relu'),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Dense(10, activation='softmax')\n", + "])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_drop.compile(loss='sparse_categorical_crossentropy', optimizer=tf.optimizers.SGD(lr=.001), metrics=['accuracy'])\n", + "model_drop.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_drop.fit(train_ds,\n", + " epochs=100,\n", + " validation_data=validation_ds,\n", + " validation_freq=1,\n", + " callbacks=[tensorboard_cb])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_drop.evaluate(test_ds)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Batch Regularization" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Bez dropoutu" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_batch = keras.models.Sequential([\n", + " keras.layers.Conv2D(filters=96, kernel_size=(11,11), strides=(4,4), activation='relu', input_shape=(227,227,3)),\n", + " keras.layers.BatchNormalization(),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Conv2D(filters=256, kernel_size=(5,5), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.BatchNormalization(),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.BatchNormalization(),\n", + " keras.layers.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.BatchNormalization(),\n", + " keras.layers.Conv2D(filters=256, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.BatchNormalization(),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Flatten(),\n", + " keras.layers.Dense(4096, activation='relu'),\n", + " keras.layers.Dense(4096, activation='relu'),\n", + " keras.layers.Dense(10, activation='softmax')\n", + "])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_batch.compile(loss='sparse_categorical_crossentropy', optimizer=tf.optimizers.SGD(lr=.001), metrics=['accuracy'])\n", + "model_batch.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_batch.fit(train_ds,\n", + " epochs=100,\n", + " validation_data=validation_ds,\n", + " validation_freq=1,\n", + " callbacks=[tensorboard_cb])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_batch.evaluate(test_ds)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Z dropoutem" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_batch_drop = keras.models.Sequential([\n", + " keras.layers.Conv2D(filters=96, kernel_size=(11,11), strides=(4,4), activation='relu', input_shape=(227,227,3)),\n", + " keras.layers.BatchNormalization(),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Conv2D(filters=256, kernel_size=(5,5), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.BatchNormalization(),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.BatchNormalization(),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.BatchNormalization(),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Conv2D(filters=256, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n", + " keras.layers.BatchNormalization(),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.MaxPool2D(pool_size=(3,3), strides=(2,2)),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Flatten(),\n", + " keras.layers.Dense(4096, activation='relu'),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Dense(4096, activation='relu'),\n", + " keras.layers.Dropout(.5),\n", + " keras.layers.Dense(10, activation='softmax')\n", + "])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_batch_drop.compile(loss='sparse_categorical_crossentropy', optimizer=tf.optimizers.SGD(lr=.001), metrics=['accuracy'])\n", + "model_batch_drop.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_batch_drop.fit(train_ds,\n", + " epochs=100,\n", + " validation_data=validation_ds,\n", + " validation_freq=1,\n", + " callbacks=[tensorboard_cb])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_batch_drop.evaluate(test_ds)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.9.4 64-bit", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.4" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "085c51388782ab7dcc7b32a500f9634129d1cddb82cd7a37058a5984251a0bc1" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}