tf_batchnorm.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "0NZ87rXSLCrQ"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorflow import keras"
      ],
      "metadata": {
        "id": "hjSBXIjQLIrZ"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorflow.keras import layers"
      ],
      "metadata": {
        "id": "8Ub0f9KkLXpZ"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorflow.keras.datasets import cifar10"
      ],
      "metadata": {
        "id": "E9_hF0kYLa4s"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "(x_train, y_train), (x_test, y_test) = cifar10.load_data()"
      ],
      "metadata": {
        "id": "rM21qXSKLf3f"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x_train = x_train.astype(\"float32\") / 255.0\n",
        "x_test = x_test.astype(\"float32\") / 255.0"
      ],
      "metadata": {
        "id": "yG2K1xe0LwpX"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Model1 : Conv -> Relu -> BatchNorm**"
      ],
      "metadata": {
        "id": "QV8ByYJgREYi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def my_model1():\n",
        "    inputs = keras.Input(shape=(32,32,3))\n",
        "    x = layers.Conv2D(32, 3, activation='relu')(inputs)\n",
        "    x = layers.BatchNormalization()(x)\n",
        "    x = layers.MaxPooling2D()(x)\n",
        "\n",
        "    x = layers.Conv2D(64, 3, activation='relu')(x)\n",
        "    x = layers.BatchNormalization()(x)\n",
        "    x = layers.MaxPooling2D()(x)\n",
        "\n",
        "    x = layers.Conv2D(128, 3, activation='relu')(x)\n",
        "    x = layers.BatchNormalization()(x)\n",
        "    x = layers.MaxPooling2D()(x)\n",
        "\n",
        "    x = layers.Flatten()(x)\n",
        "\n",
        "    x = layers.Dense(64, activation='relu')(x)\n",
        "    outputs = layers.Dense(10)(x)\n",
        "\n",
        "    model = keras.Model(inputs=inputs, outputs=outputs)\n",
        "\n",
        "    return model\n"
      ],
      "metadata": {
        "id": "gCSf7b7pMHcJ"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Model2 : Conv -> BatchNorm -> Relu**"
      ],
      "metadata": {
        "id": "0wSREf1PRc7-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def my_model2():\n",
        "    inputs = keras.Input(shape=(32,32,3))\n",
        "    x = layers.Conv2D(32, 3)(inputs)\n",
        "    x = layers.BatchNormalization()(x)\n",
        "    x = keras.activations.relu(x)\n",
        "    x = layers.MaxPooling2D()(x)\n",
        "\n",
        "    x = layers.Conv2D(64, 3)(x)\n",
        "    x = layers.BatchNormalization()(x)\n",
        "    x = keras.activations.relu(x)\n",
        "    x = layers.MaxPooling2D()(x)\n",
        "\n",
        "    x = layers.Conv2D(128, 3)(x)\n",
        "    x = layers.BatchNormalization()(x)\n",
        "    x = keras.activations.relu(x)\n",
        "    x = layers.MaxPooling2D()(x)\n",
        "\n",
        "    x = layers.Flatten()(x)\n",
        "\n",
        "    x = layers.Dense(64, activation='relu')(x)\n",
        "    outputs = layers.Dense(10)(x)\n",
        "\n",
        "    model = keras.Model(inputs=inputs, outputs=outputs)\n",
        "\n",
        "    return model\n"
      ],
      "metadata": {
        "id": "U1QnBlDXNzb1"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model1 = my_model1()\n",
        "model2 = my_model2()"
      ],
      "metadata": {
        "id": "lyfE1f_IOIll"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(model1.summary())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qQkTP2mXOOaF",
        "outputId": "6f032e99-4787-4af7-c1b4-e9f015f0a7e8"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_1 (InputLayer)        [(None, 32, 32, 3)]       0         \n",
            "                                                                 \n",
            " conv2d (Conv2D)             (None, 30, 30, 32)        896       \n",
            "                                                                 \n",
            " batch_normalization (BatchN  (None, 30, 30, 32)       128       \n",
            " ormalization)                                                   \n",
            "                                                                 \n",
            " max_pooling2d (MaxPooling2D  (None, 15, 15, 32)       0         \n",
            " )                                                               \n",
            "                                                                 \n",
            " conv2d_1 (Conv2D)           (None, 13, 13, 64)        18496     \n",
            "                                                                 \n",
            " batch_normalization_1 (Batc  (None, 13, 13, 64)       256       \n",
            " hNormalization)                                                 \n",
            "                                                                 \n",
            " max_pooling2d_1 (MaxPooling  (None, 6, 6, 64)         0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " conv2d_2 (Conv2D)           (None, 4, 4, 128)         73856     \n",
            "                                                                 \n",
            " batch_normalization_2 (Batc  (None, 4, 4, 128)        512       \n",
            " hNormalization)                                                 \n",
            "                                                                 \n",
            " max_pooling2d_2 (MaxPooling  (None, 2, 2, 128)        0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " flatten (Flatten)           (None, 512)               0         \n",
            "                                                                 \n",
            " dense (Dense)               (None, 64)                32832     \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 10)                650       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 127,626\n",
            "Trainable params: 127,178\n",
            "Non-trainable params: 448\n",
            "_________________________________________________________________\n",
            "None\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(model2.summary())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GmySaQ51ORdG",
        "outputId": "1acbd371-b921-45e0-f989-8fe75c57ebe5"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_1\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_2 (InputLayer)        [(None, 32, 32, 3)]       0         \n",
            "                                                                 \n",
            " conv2d_3 (Conv2D)           (None, 30, 30, 32)        896       \n",
            "                                                                 \n",
            " batch_normalization_3 (Batc  (None, 30, 30, 32)       128       \n",
            " hNormalization)                                                 \n",
            "                                                                 \n",
            " tf.nn.relu (TFOpLambda)     (None, 30, 30, 32)        0         \n",
            "                                                                 \n",
            " max_pooling2d_3 (MaxPooling  (None, 15, 15, 32)       0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " conv2d_4 (Conv2D)           (None, 13, 13, 64)        18496     \n",
            "                                                                 \n",
            " batch_normalization_4 (Batc  (None, 13, 13, 64)       256       \n",
            " hNormalization)                                                 \n",
            "                                                                 \n",
            " tf.nn.relu_1 (TFOpLambda)   (None, 13, 13, 64)        0         \n",
            "                                                                 \n",
            " max_pooling2d_4 (MaxPooling  (None, 6, 6, 64)         0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " conv2d_5 (Conv2D)           (None, 4, 4, 128)         73856     \n",
            "                                                                 \n",
            " batch_normalization_5 (Batc  (None, 4, 4, 128)        512       \n",
            " hNormalization)                                                 \n",
            "                                                                 \n",
            " tf.nn.relu_2 (TFOpLambda)   (None, 4, 4, 128)         0         \n",
            "                                                                 \n",
            " max_pooling2d_5 (MaxPooling  (None, 2, 2, 128)        0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " flatten_1 (Flatten)         (None, 512)               0         \n",
            "                                                                 \n",
            " dense_2 (Dense)             (None, 64)                32832     \n",
            "                                                                 \n",
            " dense_3 (Dense)             (None, 10)                650       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 127,626\n",
            "Trainable params: 127,178\n",
            "Non-trainable params: 448\n",
            "_________________________________________________________________\n",
            "None\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.compile(\n",
        "    loss = keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "    optimizer = keras.optimizers.Adam(learning_rate=0.001),\n",
        "    metrics = [\"accuracy\"]\n",
        ")"
      ],
      "metadata": {
        "id": "0LuoPv0oOa4f"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model2.compile(\n",
        "    loss = keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "    optimizer = keras.optimizers.Adam(learning_rate=0.001),\n",
        "    metrics = [\"accuracy\"]\n",
        ")"
      ],
      "metadata": {
        "id": "Rs-8V7w9O8dx"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model1.fit(x_train, y_train, batch_size=64, epochs=10, verbose=2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "48rqndfXPNxN",
        "outputId": "0d11a06b-7486-48d1-dacc-1060e221df72"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "782/782 - 11s - loss: 1.3195 - accuracy: 0.5354 - 11s/epoch - 14ms/step\n",
            "Epoch 2/10\n",
            "782/782 - 4s - loss: 0.9503 - accuracy: 0.6678 - 4s/epoch - 5ms/step\n",
            "Epoch 3/10\n",
            "782/782 - 3s - loss: 0.7842 - accuracy: 0.7270 - 3s/epoch - 4ms/step\n",
            "Epoch 4/10\n",
            "782/782 - 3s - loss: 0.6760 - accuracy: 0.7641 - 3s/epoch - 4ms/step\n",
            "Epoch 5/10\n",
            "782/782 - 4s - loss: 0.5916 - accuracy: 0.7927 - 4s/epoch - 5ms/step\n",
            "Epoch 6/10\n",
            "782/782 - 3s - loss: 0.5229 - accuracy: 0.8174 - 3s/epoch - 4ms/step\n",
            "Epoch 7/10\n",
            "782/782 - 3s - loss: 0.4570 - accuracy: 0.8412 - 3s/epoch - 4ms/step\n",
            "Epoch 8/10\n",
            "782/782 - 3s - loss: 0.3977 - accuracy: 0.8593 - 3s/epoch - 4ms/step\n",
            "Epoch 9/10\n",
            "782/782 - 4s - loss: 0.3537 - accuracy: 0.8757 - 4s/epoch - 5ms/step\n",
            "Epoch 10/10\n",
            "782/782 - 3s - loss: 0.3051 - accuracy: 0.8917 - 3s/epoch - 4ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7c71ce72e800>"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model2.fit(x_train, y_train, batch_size=64, epochs=10, verbose=2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hy5A_4kIPXCM",
        "outputId": "6b3c8011-db1e-4105-8e0c-afeae3c1b080"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "782/782 - 7s - loss: 1.2616 - accuracy: 0.5504 - 7s/epoch - 9ms/step\n",
            "Epoch 2/10\n",
            "782/782 - 3s - loss: 0.9150 - accuracy: 0.6800 - 3s/epoch - 4ms/step\n",
            "Epoch 3/10\n",
            "782/782 - 4s - loss: 0.7773 - accuracy: 0.7280 - 4s/epoch - 5ms/step\n",
            "Epoch 4/10\n",
            "782/782 - 4s - loss: 0.6817 - accuracy: 0.7626 - 4s/epoch - 5ms/step\n",
            "Epoch 5/10\n",
            "782/782 - 3s - loss: 0.6014 - accuracy: 0.7899 - 3s/epoch - 4ms/step\n",
            "Epoch 6/10\n",
            "782/782 - 3s - loss: 0.5367 - accuracy: 0.8134 - 3s/epoch - 4ms/step\n",
            "Epoch 7/10\n",
            "782/782 - 3s - loss: 0.4789 - accuracy: 0.8344 - 3s/epoch - 4ms/step\n",
            "Epoch 8/10\n",
            "782/782 - 4s - loss: 0.4232 - accuracy: 0.8532 - 4s/epoch - 5ms/step\n",
            "Epoch 9/10\n",
            "782/782 - 3s - loss: 0.3769 - accuracy: 0.8675 - 3s/epoch - 4ms/step\n",
            "Epoch 10/10\n",
            "782/782 - 3s - loss: 0.3389 - accuracy: 0.8812 - 3s/epoch - 4ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7c718b96df00>"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1.evaluate(x_test, y_test, batch_size=64, verbose=2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uTGgd1WZQI4D",
        "outputId": "85da0312-2662-4ca6-dcdf-942c96905fb0"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "157/157 - 1s - loss: 1.0637 - accuracy: 0.7100 - 598ms/epoch - 4ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1.0636695623397827, 0.7099999785423279]"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model2.evaluate(x_test, y_test, batch_size=64, verbose=2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EYN2jHGGQR5s",
        "outputId": "6156ccf9-e131-4dcb-bb95-c5e8aef59ce0"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "157/157 - 1s - loss: 1.0174 - accuracy: 0.7049 - 579ms/epoch - 4ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1.017390251159668, 0.7049000263214111]"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "CqSvzEP3QTut"
      },
      "execution_count": 17,
      "outputs": []
    }
  ]
}