Transfer_Learning_and_Fine_Tunning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Transfer Learning and Fine Tunning.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sergejhorvat/Tensorflow2.0_Udemy/blob/master/Transfer_Learning_and_Fine_Tunning.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "19QTXXYFazRX",
        "colab_type": "text"
      },
      "source": [
        "![alt text](https://live.staticflickr.com/4544/38228876666_3782386ca7_b.jpg)\n",
        "\n",
        "## Stage 1: Install dependencies and setting up GPU environment"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KmIh3NZAaEkC",
        "colab_type": "code",
        "outputId": "fe697df9-f6b2-4f3b-fb53-4ec8e377e968",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 550
        }
      },
      "source": [
        "!pip install tensorflow-gpu==2.0.0rc0"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting tensorflow-gpu==2.0.0rc0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/6a/12/8c64cc62149cc21c70c55018502831bbf4d42bd62bed196df7de6830d21b/tensorflow_gpu-2.0.0rc0-cp36-cp36m-manylinux2010_x86_64.whl (380.5MB)\n",
            "\u001b[K     |████████████████████████████████| 380.5MB 57kB/s \n",
            "\u001b[?25hRequirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0rc0) (0.2.2)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0rc0) (0.8.0)\n",
            "Collecting tf-estimator-nightly<1.14.0.dev2019080602,>=1.14.0.dev2019080601 (from tensorflow-gpu==2.0.0rc0)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/21/28/f2a27a62943d5f041e4a6fd404b2d21cb7c59b2242a4e73b03d9ba166552/tf_estimator_nightly-1.14.0.dev2019080601-py2.py3-none-any.whl (501kB)\n",
            "\u001b[K     |████████████████████████████████| 501kB 26.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy<2.0,>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0rc0) (1.16.5)\n",
            "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0rc0) (3.7.1)\n",
            "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0rc0) (0.1.7)\n",
            "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0rc0) (0.8.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0rc0) (1.1.0)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0rc0) (1.11.2)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0rc0) (3.0.1)\n",
            "Collecting tb-nightly<1.15.0a20190807,>=1.15.0a20190806 (from tensorflow-gpu==2.0.0rc0)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/bc/88/24b5fb7280e74c7cf65bde47c171547fd02afb3840cff41bcbe9270650f5/tb_nightly-1.15.0a20190806-py3-none-any.whl (4.3MB)\n",
            "\u001b[K     |████████████████████████████████| 4.3MB 30.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0rc0) (1.15.0)\n",
            "Requirement already satisfied: keras-applications>=1.0.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0rc0) (1.0.8)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0rc0) (1.1.0)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0rc0) (0.33.6)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0rc0) (1.12.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow-gpu==2.0.0rc0) (41.2.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.15.0a20190807,>=1.15.0a20190806->tensorflow-gpu==2.0.0rc0) (3.1.1)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.15.0a20190807,>=1.15.0a20190806->tensorflow-gpu==2.0.0rc0) (0.15.5)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.8->tensorflow-gpu==2.0.0rc0) (2.8.0)\n",
            "Installing collected packages: tf-estimator-nightly, tb-nightly, tensorflow-gpu\n",
            "Successfully installed tb-nightly-1.15.0a20190806 tensorflow-gpu-2.0.0rc0 tf-estimator-nightly-1.14.0.dev2019080601\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iu2mbI2Ha8UW",
        "colab_type": "code",
        "outputId": "3ec0ca3d-563d-484d-a85f-d38d5420dfe4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "!pip install tqdm"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (4.28.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jQl1fF9Ca80N",
        "colab_type": "text"
      },
      "source": [
        "### Downloading the Dogs vs Cats dataset "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mLWquLOxa-V5",
        "colab_type": "code",
        "outputId": "67e1101f-5567-4d79-e7b4-5903e43d26bd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 219
        }
      },
      "source": [
        "!wget --no-check-certificate \\\n",
        "    https://storage.googleapis.com/mledu-datasets/cats_and_dogs_filtered.zip \\\n",
        "    -O ./cats_and_dogs_filtered.zip"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2019-09-09 12:13:20--  https://storage.googleapis.com/mledu-datasets/cats_and_dogs_filtered.zip\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 173.194.76.128, 2a00:1450:400c:c08::80\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|173.194.76.128|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 68606236 (65M) [application/zip]\n",
            "Saving to: ‘./cats_and_dogs_filtered.zip’\n",
            "\n",
            "./cats_and_dogs_fil 100%[===================>]  65.43M  70.0MB/s    in 0.9s    \n",
            "\n",
            "2019-09-09 12:13:21 (70.0 MB/s) - ‘./cats_and_dogs_filtered.zip’ saved [68606236/68606236]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5-tk5DfRjXN3",
        "colab_type": "text"
      },
      "source": [
        "## Stage 2: Dataset preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LpbQm2i9jYrP",
        "colab_type": "text"
      },
      "source": [
        "### Import project dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9fOTP2XZimCM",
        "colab_type": "code",
        "outputId": "17ba3e80-4eda-4afb-f2e4-04c9a7a41eaf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "import os\n",
        "import zipfile\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from tqdm import tqdm_notebook\n",
        "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
        "\n",
        "%matplotlib inline\n",
        "tf.__version__"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'2.0.0-rc0'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MlsIOpGljdlS",
        "colab_type": "text"
      },
      "source": [
        "### Unzipping the Dogs vs Cats dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XsqDOC8WjiJ6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dataset_path = \"./cats_and_dogs_filtered.zip\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jm5kaTO6jjHA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "zip_object = zipfile.ZipFile(file=dataset_path, mode=\"r\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OI_eKR5-lhEy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "zip_object.extractall(\"./\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lj0UvhtwlhNx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "zip_object.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VSDt-MEVjlsA",
        "colab_type": "text"
      },
      "source": [
        "### Seting up dataset paths"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "glNaVdNmjmfr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dataset_path_new = \"./cats_and_dogs_filtered/\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VhfMF891jt0P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_dir = os.path.join(dataset_path_new, \"train\")\n",
        "validation_dir = os.path.join(dataset_path_new, \"validation\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-YLQb_WOmjwP",
        "colab_type": "text"
      },
      "source": [
        "## Building the model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PoMFwC51mmNa",
        "colab_type": "text"
      },
      "source": [
        "### Loading the pre-trained model (MobileNetV2)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "owkKMJiFmZhj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "IMG_SHAPE = (128, 128, 3)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pEemsnZinInr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "e9551e3a-85b8-4c05-bb86-5da89a379209"
      },
      "source": [
        "base_model = tf.keras.applications.MobileNetV2(input_shape=IMG_SHAPE, include_top=False, weights=\"imagenet\")"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://github.com/JonathanCMitchell/mobilenet_v2_keras/releases/download/v1.1/mobilenet_v2_weights_tf_dim_ordering_tf_kernels_1.0_128_no_top.h5\n",
            "9412608/9406464 [==============================] - 1s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "95RL7HQLoD8z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "base_model.summary()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iRDmVCtFofpc",
        "colab_type": "text"
      },
      "source": [
        "### Freezing the base model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kFlhPT4-oMxL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "base_model.trainable = False"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lZTcqkBipEEM",
        "colab_type": "text"
      },
      "source": [
        "### Defining the custom head for our network"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7U82_j0oo3wT",
        "colab_type": "code",
        "outputId": "f71a76df-d052-42a8-b8bc-50fadf14f1a2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "base_model.output"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor 'out_relu/Identity:0' shape=(None, 4, 4, 1280) dtype=float32>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vZlIXTCbpXEM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "global_average_layer = tf.keras.layers.GlobalAveragePooling2D()(base_model.output)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "al6yHSdEp2zL",
        "colab_type": "code",
        "outputId": "3de8f094-03ab-4237-c416-825f1ea5f28f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "global_average_layer"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor 'global_average_pooling2d/Identity:0' shape=(None, 1280) dtype=float32>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q8K2WenOp4Ob",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "prediction_layer = tf.keras.layers.Dense(units=1, activation='sigmoid')(global_average_layer)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SwzYZTTwqT8l",
        "colab_type": "text"
      },
      "source": [
        "### Defining the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qUHmyw4CqLTz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = tf.keras.models.Model(inputs=base_model.input, outputs=prediction_layer)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E-V9XI7_rBTi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qBrj4jkqrVfy",
        "colab_type": "text"
      },
      "source": [
        "### Compiling the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qt1ZOPKlsO87",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.compile(optimizer=tf.keras.optimizers.RMSprop(lr=0.0001), loss=\"binary_crossentropy\", metrics=[\"accuracy\"])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YJRlLGaotJ_J",
        "colab_type": "text"
      },
      "source": [
        "### Creating Data Generators\n",
        "\n",
        "Resizing images\n",
        "\n",
        "    Big pre-trained architecture support only certain input sizes.\n",
        "\n",
        "For example: MobileNet (architecture that we use) supports: (96, 96), (128, 128), (160, 160), (192, 192), (224, 224)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7QV5h3-OtC97",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data_gen_train = ImageDataGenerator(rescale=1/255.)\n",
        "data_gen_valid = ImageDataGenerator(rescale=1/255.)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Ms7kEkAutJb",
        "colab_type": "code",
        "outputId": "46f77ef6-5b50-493a-b81c-e13e696754d1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "train_generator = data_gen_train.flow_from_directory(train_dir, target_size=(128,128), batch_size=128, class_mode=\"binary\")"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 2000 images belonging to 2 classes.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WbJL_jtwvgGk",
        "colab_type": "code",
        "outputId": "55189e0c-caa8-4c4a-c7c9-75f9b89ffca4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "valid_generator = data_gen_valid.flow_from_directory(validation_dir, target_size=(128,128), batch_size=128, class_mode=\"binary\")"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 1000 images belonging to 2 classes.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eSzbQEcpv4C-",
        "colab_type": "text"
      },
      "source": [
        "### Training the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O6jUOnNBvuYD",
        "colab_type": "code",
        "outputId": "217cc8e8-e09c-423d-bba5-3e566152c708",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 219
        }
      },
      "source": [
        "model.fit_generator(train_generator, epochs=5, validation_data=valid_generator)"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/5\n",
            "16/16 [==============================] - 14s 866ms/step - loss: 0.3450 - accuracy: 0.8745 - val_loss: 0.3333 - val_accuracy: 0.8720\n",
            "Epoch 2/5\n",
            "16/16 [==============================] - 12s 779ms/step - loss: 0.3211 - accuracy: 0.8935 - val_loss: 0.3088 - val_accuracy: 0.8820\n",
            "Epoch 3/5\n",
            "16/16 [==============================] - 13s 782ms/step - loss: 0.3086 - accuracy: 0.8970 - val_loss: 0.2964 - val_accuracy: 0.8880\n",
            "Epoch 4/5\n",
            "16/16 [==============================] - 12s 779ms/step - loss: 0.2939 - accuracy: 0.9045 - val_loss: 0.2807 - val_accuracy: 0.8970\n",
            "Epoch 5/5\n",
            "16/16 [==============================] - 12s 776ms/step - loss: 0.2851 - accuracy: 0.9035 - val_loss: 0.2720 - val_accuracy: 0.8970\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f24aa1d9940>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t_Dx30HVxWr2",
        "colab_type": "text"
      },
      "source": [
        "### Transfer learning model evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bsc0GzhCxNrz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "valid_loss, valid_accuracy = model.evaluate_generator(valid_generator)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6XOGP0tExZvv",
        "colab_type": "code",
        "outputId": "a20efcf0-37ba-467c-e637-0204f2f442e5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "print(\"Accuracy after transfer learning: {}\".format(valid_accuracy))"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy after transfer learning: 0.8970000147819519\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3mmrlX05yhkU",
        "colab_type": "text"
      },
      "source": [
        "## Fine tuning\n",
        "\n",
        "\n",
        "There are a few pointers:\n",
        "\n",
        "- DO NOT use Fine tuning on the whole network; only a few top layers are enough. In most cases, they are more specialized. The goal of the Fine-tuning is to adopt that specific part of the network for our custom (new) dataset.\n",
        "- Start with the fine tunning AFTER you have finished with transfer learning step. If we try to perform Fine tuning immediately, gradients will be much different between our custom head layer and a few unfrozen layers from the base model. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vg-Na6Wmypx5",
        "colab_type": "text"
      },
      "source": [
        "### Un-freeze a few top layers from the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p9CHIFobyTVe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "base_model.trainable = True"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j_PpMM9xyuUW",
        "colab_type": "code",
        "outputId": "9f4e1414-a595-4d93-8936-3889608eb8f5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "print(\"Number of layersin the base model: {}\".format(len(base_model.layers)))"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of layersin the base model: 155\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kgijBsNS04AN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fine_tune_at = 100"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j8hnc35D1EOc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for layer in base_model.layers[:fine_tune_at]:\n",
        "    layer.trainable = False"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TIjXrxRa1alF",
        "colab_type": "text"
      },
      "source": [
        "### Compiling the model for fine-tuning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sOM615rt1POD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.compile(optimizer=tf.keras.optimizers.RMSprop(lr=0.0001),\n",
        "              loss='binary_crossentropy',\n",
        "              metrics=['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CD57phTH2QAI",
        "colab_type": "text"
      },
      "source": [
        "### Fine tuning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kPRnLqEu2G5z",
        "colab_type": "code",
        "outputId": "3cc8edf8-89f4-4819-9f41-9531f38e796c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 219
        }
      },
      "source": [
        "model.fit_generator(train_generator,  \n",
        "                    epochs=5, \n",
        "                    validation_data=valid_generator)"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/5\n",
            "16/16 [==============================] - 15s 952ms/step - loss: 0.1400 - accuracy: 0.9475 - val_loss: 0.1155 - val_accuracy: 0.9570\n",
            "Epoch 2/5\n",
            "16/16 [==============================] - 14s 855ms/step - loss: 0.0200 - accuracy: 0.9995 - val_loss: 0.1081 - val_accuracy: 0.9630\n",
            "Epoch 3/5\n",
            "16/16 [==============================] - 14s 851ms/step - loss: 0.0082 - accuracy: 0.9995 - val_loss: 0.1443 - val_accuracy: 0.9560\n",
            "Epoch 4/5\n",
            "16/16 [==============================] - 14s 849ms/step - loss: 0.0038 - accuracy: 1.0000 - val_loss: 0.1304 - val_accuracy: 0.9630\n",
            "Epoch 5/5\n",
            "16/16 [==============================] - 14s 853ms/step - loss: 0.0027 - accuracy: 0.9995 - val_loss: 0.1651 - val_accuracy: 0.9580\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f24adb8b908>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rutvwJBp6s2x",
        "colab_type": "text"
      },
      "source": [
        "### Evaluating the fine tuned model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FPTyisdx33u3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "valid_loss, valid_accuracy = model.evaluate_generator(valid_generator)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jijEA5n26yQu",
        "colab_type": "code",
        "outputId": "2a1e0717-b084-43dd-932c-b8755d456a5a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "print(\"Validation accuracy after fine tuning: {}\".format(valid_accuracy))"
      ],
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Validation accuracy after fine tuning: 0.9580000042915344\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ofPpHFcp7eHU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}