Inception_From_Scratch/improvements/Inception_v1_from_Scratch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/faizankshaikh/AV_Article_Codes/blob/master/Inception_v1_from_Scratch.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Improvements\n",
    "\n",
    "1. Tested on Google Colab\n",
    "2. Updated the code to use data generators to process image data on the fly\n",
    "3. Code style improved"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "JIuCtW728WD2",
    "outputId": "c806e217-e60b-4e50-ef37-198106096fcd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorFlow 1.x selected.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "%tensorflow_version 1.x\n",
    "\n",
    "import cv2\n",
    "import math\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import keras.backend as K\n",
    "\n",
    "from keras.models import Model\n",
    "from keras.optimizers import SGD\n",
    "from keras.utils import np_utils\n",
    "from keras.datasets import cifar10\n",
    "from keras.layers.core import Layer\n",
    "from keras.callbacks import LearningRateScheduler\n",
    "from keras.initializers import glorot_uniform, Constant\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.layers import (\n",
    "    Conv2D,\n",
    "    MaxPool2D,\n",
    "    Dropout,\n",
    "    Dense,\n",
    "    Input,\n",
    "    concatenate,\n",
    "    GlobalAveragePooling2D,\n",
    "    AveragePooling2D,\n",
    "    Flatten,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "m5p4szZUEQeZ"
   },
   "outputs": [],
   "source": [
    "(X_train, y_train), (X_valid, y_valid) = cifar10.load_data()\n",
    "y_train = np_utils.to_categorical(y_train, 10)\n",
    "y_valid = np_utils.to_categorical(y_valid, 10)\n",
    "\n",
    "\n",
    "def CustomImageDataGenerator(X, y, batch_size):\n",
    "    generator = ImageDataGenerator(rescale=1 / 255.0, dtype=\"float32\")\n",
    "\n",
    "    datagen = generator.flow(X, y, batch_size=batch_size)\n",
    "\n",
    "    while True:\n",
    "        X, y = datagen.__next__()\n",
    "        X = np.array([cv2.resize(img, (224, 224)) for img in X[:, :, :, :]])\n",
    "        yield X, [y, y, y]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0ZkGA5pA8XKu"
   },
   "outputs": [],
   "source": [
    "def inception_module(\n",
    "    x,\n",
    "    filters_1x1,\n",
    "    filters_3x3_reduce,\n",
    "    filters_3x3,\n",
    "    filters_5x5_reduce,\n",
    "    filters_5x5,\n",
    "    filters_pool_proj,\n",
    "    name=None,\n",
    "):\n",
    "\n",
    "    conv_1x1 = Conv2D(\n",
    "        filters=filters_1x1,\n",
    "        kernel_size=(1, 1),\n",
    "        padding=\"same\",\n",
    "        activation=\"relu\",\n",
    "        kernel_initializer=kernel_init,\n",
    "        bias_initializer=bias_init,\n",
    "    )(x)\n",
    "\n",
    "    conv_3x3 = Conv2D(\n",
    "        filters=filters_3x3_reduce,\n",
    "        kernel_size=(1, 1),\n",
    "        padding=\"same\",\n",
    "        activation=\"relu\",\n",
    "        kernel_initializer=kernel_init,\n",
    "        bias_initializer=bias_init,\n",
    "    )(x)\n",
    "\n",
    "    conv_3x3 = Conv2D(\n",
    "        filters=filters_3x3,\n",
    "        kernel_size=(3, 3),\n",
    "        padding=\"same\",\n",
    "        activation=\"relu\",\n",
    "        kernel_initializer=kernel_init,\n",
    "        bias_initializer=bias_init,\n",
    "    )(conv_3x3)\n",
    "\n",
    "    conv_5x5 = Conv2D(\n",
    "        filters=filters_5x5_reduce,\n",
    "        kernel_size=(1, 1),\n",
    "        padding=\"same\",\n",
    "        activation=\"relu\",\n",
    "        kernel_initializer=kernel_init,\n",
    "        bias_initializer=bias_init,\n",
    "    )(x)\n",
    "\n",
    "    conv_5x5 = Conv2D(\n",
    "        filters=filters_5x5,\n",
    "        kernel_size=(5, 5),\n",
    "        padding=\"same\",\n",
    "        activation=\"relu\",\n",
    "        kernel_initializer=kernel_init,\n",
    "        bias_initializer=bias_init,\n",
    "    )(conv_5x5)\n",
    "\n",
    "    pool_proj = MaxPool2D(pool_size=(3, 3), strides=(1, 1), padding=\"same\")(x)\n",
    "\n",
    "    pool_proj = Conv2D(\n",
    "        filters=filters_pool_proj,\n",
    "        kernel_size=(1, 1),\n",
    "        padding=\"same\",\n",
    "        activation=\"relu\",\n",
    "        kernel_initializer=kernel_init,\n",
    "        bias_initializer=bias_init,\n",
    "    )(pool_proj)\n",
    "\n",
    "    output = concatenate(\n",
    "        inputs=[conv_1x1, conv_3x3, conv_5x5, pool_proj], axis=3, name=name\n",
    "    )\n",
    "\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1y73xcMu8XNa"
   },
   "outputs": [],
   "source": [
    "kernel_init = glorot_uniform()\n",
    "bias_init = Constant(value=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 332
    },
    "colab_type": "code",
    "id": "7ZYsSUO88XPu",
    "outputId": "a9035306-623d-419d-cf0e-819e72846bdc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:66: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:541: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4432: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4267: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4271: The name tf.nn.avg_pool is deprecated. Please use tf.nn.avg_pool2d instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:148: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3733: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:Large dropout rate: 0.7 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n",
      "WARNING:tensorflow:Large dropout rate: 0.7 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n"
     ]
    }
   ],
   "source": [
    "input_layer = Input(shape=(224, 224, 3))\n",
    "\n",
    "x = Conv2D(\n",
    "    64,\n",
    "    (7, 7),\n",
    "    padding=\"same\",\n",
    "    strides=(2, 2),\n",
    "    activation=\"relu\",\n",
    "    name=\"conv_1_7x7/2\",\n",
    "    kernel_initializer=kernel_init,\n",
    "    bias_initializer=bias_init,\n",
    ")(input_layer)\n",
    "x = MaxPool2D((3, 3), padding=\"same\", strides=(2, 2), name=\"max_pool_1_3x3/2\")(x)\n",
    "x = Conv2D(\n",
    "    64, (1, 1), padding=\"same\", strides=(1, 1), activation=\"relu\", name=\"conv_2a_3x3/1\"\n",
    ")(x)\n",
    "x = Conv2D(\n",
    "    192, (3, 3), padding=\"same\", strides=(1, 1), activation=\"relu\", name=\"conv_2b_3x3/1\"\n",
    ")(x)\n",
    "x = MaxPool2D((3, 3), padding=\"same\", strides=(2, 2), name=\"max_pool_2_3x3/2\")(x)\n",
    "\n",
    "x = inception_module(\n",
    "    x,\n",
    "    filters_1x1=64,\n",
    "    filters_3x3_reduce=96,\n",
    "    filters_3x3=128,\n",
    "    filters_5x5_reduce=16,\n",
    "    filters_5x5=32,\n",
    "    filters_pool_proj=32,\n",
    "    name=\"inception_3a\",\n",
    ")\n",
    "\n",
    "x = inception_module(\n",
    "    x,\n",
    "    filters_1x1=128,\n",
    "    filters_3x3_reduce=128,\n",
    "    filters_3x3=192,\n",
    "    filters_5x5_reduce=32,\n",
    "    filters_5x5=96,\n",
    "    filters_pool_proj=64,\n",
    "    name=\"inception_3b\",\n",
    ")\n",
    "\n",
    "x = MaxPool2D((3, 3), padding=\"same\", strides=(2, 2), name=\"max_pool_3_3x3/2\")(x)\n",
    "\n",
    "x = inception_module(\n",
    "    x,\n",
    "    filters_1x1=192,\n",
    "    filters_3x3_reduce=96,\n",
    "    filters_3x3=208,\n",
    "    filters_5x5_reduce=16,\n",
    "    filters_5x5=48,\n",
    "    filters_pool_proj=64,\n",
    "    name=\"inception_4a\",\n",
    ")\n",
    "\n",
    "\n",
    "x1 = AveragePooling2D((5, 5), strides=3)(x)\n",
    "x1 = Conv2D(128, (1, 1), padding=\"same\", activation=\"relu\")(x1)\n",
    "x1 = Flatten()(x1)\n",
    "x1 = Dense(1024, activation=\"relu\")(x1)\n",
    "x1 = Dropout(0.7)(x1)\n",
    "x1 = Dense(10, activation=\"softmax\", name=\"auxilliary_output_1\")(x1)\n",
    "\n",
    "x = inception_module(\n",
    "    x,\n",
    "    filters_1x1=160,\n",
    "    filters_3x3_reduce=112,\n",
    "    filters_3x3=224,\n",
    "    filters_5x5_reduce=24,\n",
    "    filters_5x5=64,\n",
    "    filters_pool_proj=64,\n",
    "    name=\"inception_4b\",\n",
    ")\n",
    "\n",
    "x = inception_module(\n",
    "    x,\n",
    "    filters_1x1=128,\n",
    "    filters_3x3_reduce=128,\n",
    "    filters_3x3=256,\n",
    "    filters_5x5_reduce=24,\n",
    "    filters_5x5=64,\n",
    "    filters_pool_proj=64,\n",
    "    name=\"inception_4c\",\n",
    ")\n",
    "\n",
    "x = inception_module(\n",
    "    x,\n",
    "    filters_1x1=112,\n",
    "    filters_3x3_reduce=144,\n",
    "    filters_3x3=288,\n",
    "    filters_5x5_reduce=32,\n",
    "    filters_5x5=64,\n",
    "    filters_pool_proj=64,\n",
    "    name=\"inception_4d\",\n",
    ")\n",
    "\n",
    "\n",
    "x2 = AveragePooling2D((5, 5), strides=3)(x)\n",
    "x2 = Conv2D(128, (1, 1), padding=\"same\", activation=\"relu\")(x2)\n",
    "x2 = Flatten()(x2)\n",
    "x2 = Dense(1024, activation=\"relu\")(x2)\n",
    "x2 = Dropout(0.7)(x2)\n",
    "x2 = Dense(10, activation=\"softmax\", name=\"auxilliary_output_2\")(x2)\n",
    "\n",
    "x = inception_module(\n",
    "    x,\n",
    "    filters_1x1=256,\n",
    "    filters_3x3_reduce=160,\n",
    "    filters_3x3=320,\n",
    "    filters_5x5_reduce=32,\n",
    "    filters_5x5=128,\n",
    "    filters_pool_proj=128,\n",
    "    name=\"inception_4e\",\n",
    ")\n",
    "\n",
    "x = MaxPool2D((3, 3), padding=\"same\", strides=(2, 2), name=\"max_pool_4_3x3/2\")(x)\n",
    "\n",
    "x = inception_module(\n",
    "    x,\n",
    "    filters_1x1=256,\n",
    "    filters_3x3_reduce=160,\n",
    "    filters_3x3=320,\n",
    "    filters_5x5_reduce=32,\n",
    "    filters_5x5=128,\n",
    "    filters_pool_proj=128,\n",
    "    name=\"inception_5a\",\n",
    ")\n",
    "\n",
    "x = inception_module(\n",
    "    x,\n",
    "    filters_1x1=384,\n",
    "    filters_3x3_reduce=192,\n",
    "    filters_3x3=384,\n",
    "    filters_5x5_reduce=48,\n",
    "    filters_5x5=128,\n",
    "    filters_pool_proj=128,\n",
    "    name=\"inception_5b\",\n",
    ")\n",
    "\n",
    "x = GlobalAveragePooling2D(name=\"avg_pool_5_3x3/1\")(x)\n",
    "\n",
    "x = Dropout(0.4)(x)\n",
    "\n",
    "x = Dense(10, activation=\"softmax\", name=\"output\")(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "K5gv4hDA8XSA"
   },
   "outputs": [],
   "source": [
    "model = Model(input_layer, [x, x1, x2], name=\"inception_v1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "oDaDZJCE8XUO"
   },
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 106
    },
    "colab_type": "code",
    "id": "UUAV2emp8XWL",
    "outputId": "ccd0abe1-7985-4edf-ba9d-56f47c1d7eed"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/optimizers.py:793: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3576: The name tf.log is deprecated. Please use tf.math.log instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "epochs = 25\n",
    "initial_lrate = 0.01\n",
    "\n",
    "\n",
    "def decay(epoch, steps=100):\n",
    "    initial_lrate = 0.01\n",
    "    drop = 0.96\n",
    "    epochs_drop = 8\n",
    "    lrate = initial_lrate * math.pow(drop, math.floor((1 + epoch) / epochs_drop))\n",
    "    return lrate\n",
    "\n",
    "\n",
    "sgd = SGD(lr=initial_lrate, momentum=0.9, nesterov=False)\n",
    "\n",
    "lr_sc = LearningRateScheduler(decay, verbose=1)\n",
    "\n",
    "model.compile(\n",
    "    loss=[\n",
    "        \"categorical_crossentropy\",\n",
    "        \"categorical_crossentropy\",\n",
    "        \"categorical_crossentropy\",\n",
    "    ],\n",
    "    loss_weights=[1, 0.3, 0.3],\n",
    "    optimizer=sgd,\n",
    "    metrics=[\"accuracy\"],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 610
    },
    "colab_type": "code",
    "id": "FpVMejKG7TDI",
    "outputId": "9745f1b1-ab2a-4366-ad74-2ea2195d63b2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/math_grad.py:1424: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1033: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1020: The name tf.assign is deprecated. Please use tf.compat.v1.assign instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3005: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
      "\n",
      "Epoch 1/25\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:190: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/keras/engine/training_generator.py:49: UserWarning: Using a generator with `use_multiprocessing=True` and multiple workers may duplicate your data. Please consider using the `keras.utils.Sequence class.\n",
      "  UserWarning('Using a generator with `use_multiprocessing=True`'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:197: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:207: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:216: The name tf.is_variable_initialized is deprecated. Please use tf.compat.v1.is_variable_initialized instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:223: The name tf.variables_initializer is deprecated. Please use tf.compat.v1.variables_initializer instead.\n",
      "\n",
      "\n",
      "Epoch 00001: LearningRateScheduler setting learning rate to 0.01.\n",
      "200/200 [==============================] - 218s 1s/step - loss: 3.7005 - output_loss: 2.3221 - auxilliary_output_1_loss: 2.2912 - auxilliary_output_2_loss: 2.3033 - output_acc: 0.1143 - auxilliary_output_1_acc: 0.1238 - auxilliary_output_2_acc: 0.1117\n",
      "Epoch 2/25\n",
      "\n",
      "Epoch 00002: LearningRateScheduler setting learning rate to 0.01.\n",
      "200/200 [==============================] - 195s 974ms/step - loss: 3.2905 - output_loss: 2.0650 - auxilliary_output_1_loss: 2.0379 - auxilliary_output_2_loss: 2.0472 - output_acc: 0.2155 - auxilliary_output_1_acc: 0.2431 - auxilliary_output_2_acc: 0.2337\n",
      "Epoch 3/25\n",
      "\n",
      "Epoch 00003: LearningRateScheduler setting learning rate to 0.01.\n",
      " 48/200 [======>.......................] - ETA: 2:28 - loss: 3.1024 - output_loss: 1.9481 - auxilliary_output_1_loss: 1.9233 - auxilliary_output_2_loss: 1.9243 - output_acc: 0.2620 - auxilliary_output_1_acc: 0.2888 - auxilliary_output_2_acc: 0.2900"
     ]
    }
   ],
   "source": [
    "model.fit_generator(\n",
    "    CustomImageDataGenerator(X_train, y_train, batch_size=256),\n",
    "    epochs=epochs,\n",
    "    steps_per_epoch=200,\n",
    "    use_multiprocessing=True,\n",
    "    workers=4,\n",
    "    callbacks=[lr_sc],\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "include_colab_link": true,
   "name": "Inception_v1_from_Scratch.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}