transfer_learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d6180458",
   "metadata": {},
   "source": [
    "## CNN Types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "095ab623",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "089ae5ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "vgg_model = tf.keras.applications.VGG16(input_shape=(224, 224, 3), include_top=True, weights=None)\n",
    "# vgg_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f16e9765",
   "metadata": {},
   "outputs": [],
   "source": [
    "resnet_model = tf.keras.applications.ResNet50(input_shape =(224, 224, 3), include_top=True, weights=None)\n",
    "# resnet_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "26f3c6e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "densenet_model = tf.keras.applications.DenseNet121(input_shape =(224, 224, 3), include_top=True, weights=None)\n",
    "# densenet_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "213fc7f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "mobilenet_model = tf.keras.applications.MobileNet(input_shape =(224, 224, 3), include_top=False, weights=\"imagenet\")\n",
    "# mobilenet_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6ad9031f",
   "metadata": {},
   "outputs": [],
   "source": [
    "mobilenetv2_model = tf.keras.applications.MobileNetV2(input_shape =(224, 224, 3), include_top=True, weights=None)\n",
    "# mobilenetv2_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ac4933ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "(train_data, train_labels), (test_data, test_labels) = tf.keras.datasets.cifar10.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f0ae512c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Layer block1_conv1 is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because it's dtype defaults to floatx.\n",
      "\n",
      "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
      "\n",
      "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
      "\n",
      "Run time for VGG =17.484537700000004\n"
     ]
    }
   ],
   "source": [
    "import timeit\n",
    "\n",
    "start_time = timeit.default_timer()\n",
    "for i in range(8):\n",
    "    vgg_model(np.random.rand(16, 224, 224, 3))\n",
    "end_time = timeit.default_timer()\n",
    "print(\"Run time for VGG ={}\".format(end_time - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0bc7d66d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "138357544"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainable_weights = 0\n",
    "for var in vgg_model.trainable_weights:\n",
    "    trainable_weights += var.numpy().size\n",
    "trainable_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b62ac5ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Layer conv1_pad is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because it's dtype defaults to floatx.\n",
      "\n",
      "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
      "\n",
      "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
      "\n",
      "Run time for Resnet =9.151308499999999\n"
     ]
    }
   ],
   "source": [
    "start_time = timeit.default_timer()\n",
    "for i in range(8):\n",
    "    resnet_model(np.random.rand(16, 224, 224, 3))\n",
    "end_time = timeit.default_timer()\n",
    "print(\"Run time for Resnet ={}\".format(end_time - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0c9d8412",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25583592"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainable_weights = 0\n",
    "for var in resnet_model.trainable_weights:\n",
    "    trainable_weights += var.numpy().size\n",
    "trainable_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "342c7819",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "177\n"
     ]
    }
   ],
   "source": [
    "print(len(resnet_model.layers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "52c1cb34",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Layer zero_padding2d is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because it's dtype defaults to floatx.\n",
      "\n",
      "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
      "\n",
      "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
      "\n",
      "Run time for Densenet =9.671390800000005\n"
     ]
    }
   ],
   "source": [
    "start_time = timeit.default_timer()\n",
    "for i in range(8):\n",
    "    densenet_model(np.random.rand(16, 224, 224, 3))\n",
    "end_time = timeit.default_timer()\n",
    "print(\"Run time for Densenet ={}\".format(end_time - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "aded9edc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Layer conv1_pad is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because it's dtype defaults to floatx.\n",
      "\n",
      "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
      "\n",
      "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
      "\n",
      "Run time for Mobilenet =3.0207960000000043\n"
     ]
    }
   ],
   "source": [
    "start_time = timeit.default_timer()\n",
    "for i in range(8):\n",
    "    mobilenet_model(np.random.rand(16, 224, 224, 3))\n",
    "end_time = timeit.default_timer()\n",
    "print(\"Run time for Mobilenet ={}\".format(end_time - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c64ec41d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Layer Conv1_pad is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because it's dtype defaults to floatx.\n",
      "\n",
      "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
      "\n",
      "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
      "\n",
      "Run time for Mobilenetv2 =3.6313497999999953\n"
     ]
    }
   ],
   "source": [
    "start_time = timeit.default_timer()\n",
    "for i in range(8):\n",
    "    mobilenetv2_model(np.random.rand(16, 224, 224, 3))\n",
    "end_time = timeit.default_timer()\n",
    "print(\"Run time for Mobilenetv2 ={}\".format(end_time - start_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f884aed",
   "metadata": {},
   "source": [
    "## Transfer Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5734a613",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, ReLU, Dense, Dropout, Flatten, GlobalAveragePooling2D\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.optimizers import Adam, SGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "86e1fc88",
   "metadata": {},
   "outputs": [],
   "source": [
    "(train_data, train_labels), (test_data, test_labels) = tf.keras.datasets.cifar10.load_data()\n",
    "train_data = tf.keras.applications.mobilenet.preprocess_input(train_data.astype(np.float32))\n",
    "test_data = tf.keras.applications.mobilenet.preprocess_input(test_data.astype(np.float32))\n",
    "train_labels = to_categorical(train_labels, num_classes=10)\n",
    "test_labels = to_categorical(test_labels, num_classes=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a7aac8a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\sevda\\anaconda3\\envs\\part_college\\lib\\site-packages\\keras_applications\\mobilenet.py:207: UserWarning: `input_shape` is undefined or non-square, or `rows` is not in [128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "  warnings.warn('`input_shape` is undefined or non-square, '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_6 (InputLayer)         [(None, 32, 32, 3)]       0         \n",
      "_________________________________________________________________\n",
      "conv1_pad (ZeroPadding2D)    (None, 33, 33, 3)         0         \n",
      "_________________________________________________________________\n",
      "conv1 (Conv2D)               (None, 16, 16, 32)        864       \n",
      "_________________________________________________________________\n",
      "conv1_bn (BatchNormalization (None, 16, 16, 32)        128       \n",
      "_________________________________________________________________\n",
      "conv1_relu (ReLU)            (None, 16, 16, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv_dw_1 (DepthwiseConv2D)  (None, 16, 16, 32)        288       \n",
      "_________________________________________________________________\n",
      "conv_dw_1_bn (BatchNormaliza (None, 16, 16, 32)        128       \n",
      "_________________________________________________________________\n",
      "conv_dw_1_relu (ReLU)        (None, 16, 16, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv_pw_1 (Conv2D)           (None, 16, 16, 64)        2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_1_bn (BatchNormaliza (None, 16, 16, 64)        256       \n",
      "_________________________________________________________________\n",
      "conv_pw_1_relu (ReLU)        (None, 16, 16, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv_pad_2 (ZeroPadding2D)   (None, 17, 17, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv_dw_2 (DepthwiseConv2D)  (None, 8, 8, 64)          576       \n",
      "_________________________________________________________________\n",
      "conv_dw_2_bn (BatchNormaliza (None, 8, 8, 64)          256       \n",
      "_________________________________________________________________\n",
      "conv_dw_2_relu (ReLU)        (None, 8, 8, 64)          0         \n",
      "_________________________________________________________________\n",
      "conv_pw_2 (Conv2D)           (None, 8, 8, 128)         8192      \n",
      "_________________________________________________________________\n",
      "conv_pw_2_bn (BatchNormaliza (None, 8, 8, 128)         512       \n",
      "_________________________________________________________________\n",
      "conv_pw_2_relu (ReLU)        (None, 8, 8, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv_dw_3 (DepthwiseConv2D)  (None, 8, 8, 128)         1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_3_bn (BatchNormaliza (None, 8, 8, 128)         512       \n",
      "_________________________________________________________________\n",
      "conv_dw_3_relu (ReLU)        (None, 8, 8, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_3 (Conv2D)           (None, 8, 8, 128)         16384     \n",
      "_________________________________________________________________\n",
      "conv_pw_3_bn (BatchNormaliza (None, 8, 8, 128)         512       \n",
      "_________________________________________________________________\n",
      "conv_pw_3_relu (ReLU)        (None, 8, 8, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv_pad_4 (ZeroPadding2D)   (None, 9, 9, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv_dw_4 (DepthwiseConv2D)  (None, 4, 4, 128)         1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_4_bn (BatchNormaliza (None, 4, 4, 128)         512       \n",
      "_________________________________________________________________\n",
      "conv_dw_4_relu (ReLU)        (None, 4, 4, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_4 (Conv2D)           (None, 4, 4, 256)         32768     \n",
      "_________________________________________________________________\n",
      "conv_pw_4_bn (BatchNormaliza (None, 4, 4, 256)         1024      \n",
      "_________________________________________________________________\n",
      "conv_pw_4_relu (ReLU)        (None, 4, 4, 256)         0         \n",
      "_________________________________________________________________\n",
      "conv_dw_5 (DepthwiseConv2D)  (None, 4, 4, 256)         2304      \n",
      "_________________________________________________________________\n",
      "conv_dw_5_bn (BatchNormaliza (None, 4, 4, 256)         1024      \n",
      "_________________________________________________________________\n",
      "conv_dw_5_relu (ReLU)        (None, 4, 4, 256)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_5 (Conv2D)           (None, 4, 4, 256)         65536     \n",
      "_________________________________________________________________\n",
      "conv_pw_5_bn (BatchNormaliza (None, 4, 4, 256)         1024      \n",
      "_________________________________________________________________\n",
      "conv_pw_5_relu (ReLU)        (None, 4, 4, 256)         0         \n",
      "_________________________________________________________________\n",
      "global_average_pooling2d_1 ( (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 10)                2570      \n",
      "=================================================================\n",
      "Total params: 139,722\n",
      "Trainable params: 2,570\n",
      "Non-trainable params: 137,152\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "mobilenet_model = tf.keras.applications.MobileNet(input_shape =(32, 32, 3), include_top=False, weights=\"imagenet\")\n",
    "\n",
    "my_smaller_model = tf.keras.models.Model(mobilenet_model.input, mobilenet_model.get_layer(\"conv_pw_5_relu\").output)\n",
    "my_smaller_model.trainable = False\n",
    "# mobilenet_model.trainable = False\n",
    "\n",
    "# for layer in mobilenet_model.layers[:-12]:\n",
    "#     mobilenet_model.trainable = False\n",
    "    \n",
    "    \n",
    "\n",
    "base_model_output = my_smaller_model.output\n",
    "\n",
    "gap = GlobalAveragePooling2D()(base_model_output)\n",
    "\n",
    "# fc = Dense(units=256, activation=\"relu\")(gap)\n",
    "\n",
    "output = Dense(units=10, activation=\"softmax\")(gap)\n",
    "\n",
    "\n",
    "transfer_model = tf.keras.models.Model(mobilenet_model.input, output)\n",
    "transfer_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "8a1dcde5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/20\n",
      "40000/40000 [==============================] - 46s 1ms/sample - loss: 1.4830 - categorical_accuracy: 0.4833 - val_loss: 4.8511 - val_categorical_accuracy: 0.2630\n",
      "Epoch 2/20\n",
      "40000/40000 [==============================] - 42s 1ms/sample - loss: 1.2386 - categorical_accuracy: 0.5687 - val_loss: 5.3665 - val_categorical_accuracy: 0.2463\n",
      "Epoch 3/20\n",
      "40000/40000 [==============================] - 43s 1ms/sample - loss: 1.1775 - categorical_accuracy: 0.5901 - val_loss: 5.7231 - val_categorical_accuracy: 0.2498\n",
      "Epoch 4/20\n",
      "40000/40000 [==============================] - 43s 1ms/sample - loss: 1.1467 - categorical_accuracy: 0.6046 - val_loss: 7.1915 - val_categorical_accuracy: 0.2571\n",
      "Epoch 5/20\n",
      "40000/40000 [==============================] - 42s 1ms/sample - loss: 1.1377 - categorical_accuracy: 0.6041 - val_loss: 7.7200 - val_categorical_accuracy: 0.2448\n",
      "Epoch 6/20\n",
      "40000/40000 [==============================] - 42s 1ms/sample - loss: 1.1128 - categorical_accuracy: 0.6132 - val_loss: 7.8365 - val_categorical_accuracy: 0.2061\n",
      "Epoch 7/20\n",
      "40000/40000 [==============================] - 43s 1ms/sample - loss: 1.0826 - categorical_accuracy: 0.6248 - val_loss: 6.6819 - val_categorical_accuracy: 0.2642\n",
      "Epoch 8/20\n",
      "40000/40000 [==============================] - 43s 1ms/sample - loss: 1.0785 - categorical_accuracy: 0.6260 - val_loss: 8.1497 - val_categorical_accuracy: 0.2207\n",
      "Epoch 9/20\n",
      "40000/40000 [==============================] - 44s 1ms/sample - loss: 1.0789 - categorical_accuracy: 0.6283 - val_loss: 7.1686 - val_categorical_accuracy: 0.2356\n",
      "Epoch 10/20\n",
      "40000/40000 [==============================] - 45s 1ms/sample - loss: 1.0674 - categorical_accuracy: 0.6315 - val_loss: 7.5850 - val_categorical_accuracy: 0.2211\n",
      "Epoch 11/20\n",
      "40000/40000 [==============================] - 43s 1ms/sample - loss: 1.0501 - categorical_accuracy: 0.6388 - val_loss: 8.1946 - val_categorical_accuracy: 0.2161\n",
      "Epoch 12/20\n",
      "40000/40000 [==============================] - 44s 1ms/sample - loss: 1.0547 - categorical_accuracy: 0.6374 - val_loss: 7.9994 - val_categorical_accuracy: 0.2249\n",
      "Epoch 13/20\n",
      "40000/40000 [==============================] - 43s 1ms/sample - loss: 1.0383 - categorical_accuracy: 0.6423 - val_loss: 7.2591 - val_categorical_accuracy: 0.2354\n",
      "Epoch 14/20\n",
      "40000/40000 [==============================] - 43s 1ms/sample - loss: 1.0378 - categorical_accuracy: 0.6416 - val_loss: 7.4277 - val_categorical_accuracy: 0.2282\n",
      "Epoch 15/20\n",
      "40000/40000 [==============================] - 44s 1ms/sample - loss: 1.0399 - categorical_accuracy: 0.6407 - val_loss: 8.6479 - val_categorical_accuracy: 0.2040\n",
      "Epoch 16/20\n",
      "40000/40000 [==============================] - 43s 1ms/sample - loss: 1.0391 - categorical_accuracy: 0.6421 - val_loss: 8.0343 - val_categorical_accuracy: 0.2235\n",
      "Epoch 17/20\n",
      "40000/40000 [==============================] - 44s 1ms/sample - loss: 1.0379 - categorical_accuracy: 0.6428 - val_loss: 7.5871 - val_categorical_accuracy: 0.2392\n",
      "Epoch 18/20\n",
      "40000/40000 [==============================] - 46s 1ms/sample - loss: 1.0422 - categorical_accuracy: 0.6436 - val_loss: 7.6111 - val_categorical_accuracy: 0.2410\n",
      "Epoch 19/20\n",
      "40000/40000 [==============================] - 44s 1ms/sample - loss: 1.0257 - categorical_accuracy: 0.6466 - val_loss: 8.8011 - val_categorical_accuracy: 0.2052\n",
      "Epoch 20/20\n",
      "40000/40000 [==============================] - 43s 1ms/sample - loss: 1.0374 - categorical_accuracy: 0.6439 - val_loss: 7.2610 - val_categorical_accuracy: 0.2269\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1e8a3ad9b08>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "save_best = tf.keras.callbacks.ModelCheckpoint(\"transfer_model.h5\", save_best_only=True)\n",
    "transfer_model.compile(optimizer=SGD(learning_rate=0.01, momentum=0.9),\n",
    "                       loss=\"categorical_crossentropy\", metrics=[\"categorical_accuracy\"])\n",
    "transfer_model.fit(x=train_data[:40000], y=train_labels[:40000], epochs=20, batch_size=32,\n",
    "                  validation_data=(train_data[40000:], train_labels[40000:]), callbacks=[save_best])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ea4a0730",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_6 (InputLayer)         [(None, 32, 32, 3)]       0         \n",
      "_________________________________________________________________\n",
      "conv1_pad (ZeroPadding2D)    (None, 33, 33, 3)         0         \n",
      "_________________________________________________________________\n",
      "conv1 (Conv2D)               (None, 16, 16, 32)        864       \n",
      "_________________________________________________________________\n",
      "conv1_bn (BatchNormalization (None, 16, 16, 32)        128       \n",
      "_________________________________________________________________\n",
      "conv1_relu (ReLU)            (None, 16, 16, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv_dw_1 (DepthwiseConv2D)  (None, 16, 16, 32)        288       \n",
      "_________________________________________________________________\n",
      "conv_dw_1_bn (BatchNormaliza (None, 16, 16, 32)        128       \n",
      "_________________________________________________________________\n",
      "conv_dw_1_relu (ReLU)        (None, 16, 16, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv_pw_1 (Conv2D)           (None, 16, 16, 64)        2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_1_bn (BatchNormaliza (None, 16, 16, 64)        256       \n",
      "_________________________________________________________________\n",
      "conv_pw_1_relu (ReLU)        (None, 16, 16, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv_pad_2 (ZeroPadding2D)   (None, 17, 17, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv_dw_2 (DepthwiseConv2D)  (None, 8, 8, 64)          576       \n",
      "_________________________________________________________________\n",
      "conv_dw_2_bn (BatchNormaliza (None, 8, 8, 64)          256       \n",
      "_________________________________________________________________\n",
      "conv_dw_2_relu (ReLU)        (None, 8, 8, 64)          0         \n",
      "_________________________________________________________________\n",
      "conv_pw_2 (Conv2D)           (None, 8, 8, 128)         8192      \n",
      "_________________________________________________________________\n",
      "conv_pw_2_bn (BatchNormaliza (None, 8, 8, 128)         512       \n",
      "_________________________________________________________________\n",
      "conv_pw_2_relu (ReLU)        (None, 8, 8, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv_dw_3 (DepthwiseConv2D)  (None, 8, 8, 128)         1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_3_bn (BatchNormaliza (None, 8, 8, 128)         512       \n",
      "_________________________________________________________________\n",
      "conv_dw_3_relu (ReLU)        (None, 8, 8, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_3 (Conv2D)           (None, 8, 8, 128)         16384     \n",
      "_________________________________________________________________\n",
      "conv_pw_3_bn (BatchNormaliza (None, 8, 8, 128)         512       \n",
      "_________________________________________________________________\n",
      "conv_pw_3_relu (ReLU)        (None, 8, 8, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv_pad_4 (ZeroPadding2D)   (None, 9, 9, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv_dw_4 (DepthwiseConv2D)  (None, 4, 4, 128)         1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_4_bn (BatchNormaliza (None, 4, 4, 128)         512       \n",
      "_________________________________________________________________\n",
      "conv_dw_4_relu (ReLU)        (None, 4, 4, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_4 (Conv2D)           (None, 4, 4, 256)         32768     \n",
      "_________________________________________________________________\n",
      "conv_pw_4_bn (BatchNormaliza (None, 4, 4, 256)         1024      \n",
      "_________________________________________________________________\n",
      "conv_pw_4_relu (ReLU)        (None, 4, 4, 256)         0         \n",
      "_________________________________________________________________\n",
      "conv_dw_5 (DepthwiseConv2D)  (None, 4, 4, 256)         2304      \n",
      "_________________________________________________________________\n",
      "conv_dw_5_bn (BatchNormaliza (None, 4, 4, 256)         1024      \n",
      "_________________________________________________________________\n",
      "conv_dw_5_relu (ReLU)        (None, 4, 4, 256)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_5 (Conv2D)           (None, 4, 4, 256)         65536     \n",
      "_________________________________________________________________\n",
      "conv_pw_5_bn (BatchNormaliza (None, 4, 4, 256)         1024      \n",
      "_________________________________________________________________\n",
      "conv_pw_5_relu (ReLU)        (None, 4, 4, 256)         0         \n",
      "=================================================================\n",
      "Total params: 137,152\n",
      "Trainable params: 134,208\n",
      "Non-trainable params: 2,944\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "my_smaller_model.trainable = True\n",
    "my_smaller_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "4b0e1539",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 40000 samples, validate on 10000 samples\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1e8a3b3d0c8>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "save_best = tf.keras.callbacks.ModelCheckpoint(\"transfer_model_stage2.h5\", save_best_only=True)\n",
    "transfer_model.compile(optimizer=SGD(learning_rate=0.0001, momentum=0.9),\n",
    "                       loss=\"categorical_crossentropy\", metrics=[\"categorical_accuracy\"])\n",
    "transfer_model.fit(x=train_data[:40000], y=train_labels[:40000], epochs=10, batch_size=32,\n",
    "                  validation_data=(train_data[40000:], train_labels[40000:]), callbacks=[save_best], initial_epoch=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a8e6ff72",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_7 (InputLayer)         [(None, 32, 32, 3)]       0         \n",
      "_________________________________________________________________\n",
      "conv2d (Conv2D)              (None, 32, 32, 16)        1216      \n",
      "_________________________________________________________________\n",
      "re_lu (ReLU)                 (None, 32, 32, 16)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 16, 16, 16)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 16, 16, 32)        4640      \n",
      "_________________________________________________________________\n",
      "re_lu_1 (ReLU)               (None, 16, 16, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 16, 16, 32)        9248      \n",
      "_________________________________________________________________\n",
      "re_lu_2 (ReLU)               (None, 16, 16, 32)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 8, 8, 32)          0         \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 8, 8, 64)          18496     \n",
      "_________________________________________________________________\n",
      "re_lu_3 (ReLU)               (None, 8, 8, 64)          0         \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 8, 8, 64)          36928     \n",
      "_________________________________________________________________\n",
      "re_lu_4 (ReLU)               (None, 8, 8, 64)          0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 4, 4, 64)          0         \n",
      "_________________________________________________________________\n",
      "global_average_pooling2d_2 ( (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 10)                650       \n",
      "=================================================================\n",
      "Total params: 71,178\n",
      "Trainable params: 71,178\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# INPUT\n",
    "i = Input((32, 32, 3))\n",
    "\n",
    "# CNN\n",
    "# BLOCK 1\n",
    "x = Conv2D(filters=16, kernel_size=(5, 5), strides=1, padding=\"same\")(i)\n",
    "x = ReLU()(x)\n",
    "x = MaxPooling2D(pool_size=(2, 2), strides=2, padding=\"same\")(x)\n",
    "\n",
    "# BLOCK 2\n",
    "x = Conv2D(filters=32, kernel_size=(3, 3), strides=1, padding=\"same\")(x)\n",
    "x = ReLU()(x)\n",
    "x = Conv2D(filters=32, kernel_size=(3, 3), strides=1, padding=\"same\")(x)\n",
    "x = ReLU()(x)\n",
    "x = MaxPooling2D(pool_size=(2, 2), strides=2, padding=\"same\")(x)\n",
    "\n",
    "# BLOCK 3\n",
    "x = Conv2D(filters=64, kernel_size=(3, 3), strides=1, padding=\"same\")(x)\n",
    "x = ReLU()(x)\n",
    "x = Conv2D(filters=64, kernel_size=(3, 3), strides=1, padding=\"same\")(x)\n",
    "x = ReLU()(x)\n",
    "x = MaxPooling2D(pool_size=(2, 2), strides=2, padding=\"same\")(x)\n",
    "\n",
    "# CLASSIFIER\n",
    "\n",
    "# Option 1\n",
    "# x = Flatten()(x)\n",
    "# c = Dense(units=256, activation=\"relu\")(x)\n",
    "\n",
    "# Option 2\n",
    "c = GlobalAveragePooling2D()(x)\n",
    "\n",
    "# DROPOUT\n",
    "c = Dropout(0.5)(c)\n",
    "\n",
    "# PREDICTIONS\n",
    "p = Dense(units=10, activation=\"softmax\")(c)\n",
    "\n",
    "# CREATE MODEL\n",
    "cnn_model = tf.keras.models.Model(i, p)\n",
    "cnn_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ec7922d6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 2000 samples, validate on 48000 samples\n",
      "Epoch 1/5\n",
      "2000/2000 [==============================] - 27s 13ms/sample - loss: 2.1971 - categorical_accuracy: 0.1780 - val_loss: 2.0542 - val_categorical_accuracy: 0.2178\n",
      "Epoch 2/5\n",
      "2000/2000 [==============================] - 22s 11ms/sample - loss: 2.0728 - categorical_accuracy: 0.1980 - val_loss: 1.9827 - val_categorical_accuracy: 0.2483\n",
      "Epoch 3/5\n",
      "2000/2000 [==============================] - 23s 12ms/sample - loss: 1.9850 - categorical_accuracy: 0.2200 - val_loss: 1.9955 - val_categorical_accuracy: 0.2107\n",
      "Epoch 4/5\n",
      "2000/2000 [==============================] - 21s 11ms/sample - loss: 1.9350 - categorical_accuracy: 0.2410 - val_loss: 1.7963 - val_categorical_accuracy: 0.3276\n",
      "Epoch 5/5\n",
      "2000/2000 [==============================] - 23s 11ms/sample - loss: 1.8546 - categorical_accuracy: 0.2710 - val_loss: 1.7481 - val_categorical_accuracy: 0.3461\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1e8a3ae70c8>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnn_model.compile(optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"categorical_accuracy\"])\n",
    "cnn_model.fit(x=train_data, y=train_labels, epochs=5, batch_size=16, validation_split=0.96)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2faccd8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}