notebooks/deep_net_in_tensorflow.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jonkrohn/DLTFpT/blob/master/notebooks/deep_net_in_tensorflow.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rBc4jN01oxK6"
      },
      "source": [
        "# Deep Neural Network in TensorFlow"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jpqvWw3YoxK7"
      },
      "source": [
        "In this notebook, we improve on our [intermediate neural net](https://github.com/jonkrohn/DLTFpT/blob/master/notebooks/intermediate_net_in_tensorflow.ipynb) by incorporating the theory we've covered since."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xc8-tWyBoxK8"
      },
      "source": [
        "#### Load dependencies"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "scrolled": true,
        "id": "TiEdNZIXoxK8"
      },
      "outputs": [],
      "source": [
        "import tensorflow\n",
        "from tensorflow.keras.datasets import mnist\n",
        "from tensorflow.keras.utils import to_categorical\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense\n",
        "from tensorflow.keras.layers import Dropout # new!\n",
        "from tensorflow.keras.layers import BatchNormalization # new!\n",
        "from tensorflow.keras.optimizers import SGD\n",
        "from matplotlib import pyplot as plt"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_VQNSjckoxK9"
      },
      "source": [
        "#### Load data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "scrolled": true,
        "id": "-Anhi2R0oxK9",
        "outputId": "8652781a-e1cb-41ea-f5b5-8a905c158726",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
            "11490434/11490434 [==============================] - 2s 0us/step\n"
          ]
        }
      ],
      "source": [
        "(X_train, y_train), (X_valid, y_valid) = mnist.load_data()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1hlBAuuJoxK9"
      },
      "source": [
        "#### Preprocess data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "S6HH-sn-oxK9"
      },
      "outputs": [],
      "source": [
        "X_train = X_train.reshape(60000, 784).astype('float32')\n",
        "X_valid = X_valid.reshape(10000, 784).astype('float32')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mYtjpndBoxK9"
      },
      "outputs": [],
      "source": [
        "X_train /= 255\n",
        "X_valid /= 255"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Llrv68VzoxK9"
      },
      "outputs": [],
      "source": [
        "n_classes = 10\n",
        "y_train = to_categorical(y_train, n_classes)\n",
        "y_valid = to_categorical(y_valid, n_classes)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q2snEjw6oxK9"
      },
      "source": [
        "#### Design neural network architecture"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nP8VmY4ooxK-"
      },
      "outputs": [],
      "source": [
        "model = Sequential()\n",
        "\n",
        "model.add(Dense(64, activation='relu', input_shape=(784,)))\n",
        "model.add(BatchNormalization())\n",
        "\n",
        "model.add(Dense(64, activation='relu'))\n",
        "model.add(BatchNormalization())\n",
        "\n",
        "model.add(Dense(64, activation='relu'))\n",
        "model.add(BatchNormalization())\n",
        "model.add(Dropout(0.2))\n",
        "\n",
        "model.add(Dense(10, activation='softmax'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KdGXw3zvoxK-",
        "outputId": "0105342d-1bac-47c1-e909-cb24549bf29f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " dense (Dense)               (None, 64)                50240     \n",
            "                                                                 \n",
            " batch_normalization (BatchN  (None, 64)               256       \n",
            " ormalization)                                                   \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 64)                4160      \n",
            "                                                                 \n",
            " batch_normalization_1 (Batc  (None, 64)               256       \n",
            " hNormalization)                                                 \n",
            "                                                                 \n",
            " dense_2 (Dense)             (None, 64)                4160      \n",
            "                                                                 \n",
            " batch_normalization_2 (Batc  (None, 64)               256       \n",
            " hNormalization)                                                 \n",
            "                                                                 \n",
            " dropout (Dropout)           (None, 64)                0         \n",
            "                                                                 \n",
            " dense_3 (Dense)             (None, 10)                650       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 59,978\n",
            "Trainable params: 59,594\n",
            "Non-trainable params: 384\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "model.summary()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Elhdi6XpoxK-"
      },
      "source": [
        "#### Configure model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fdBVWyy2oxK-"
      },
      "outputs": [],
      "source": [
        "model.compile(loss='categorical_crossentropy', optimizer='nadam', metrics=['accuracy'])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AMXbqUOkoxK-"
      },
      "source": [
        "#### Train!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rPIiIVHIoxK-",
        "outputId": "daf361e5-351e-4245-cd0d-01f3772c4638",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/20\n",
            "469/469 [==============================] - 12s 9ms/step - loss: 0.3941 - accuracy: 0.8804 - val_loss: 0.1630 - val_accuracy: 0.9512\n",
            "Epoch 2/20\n",
            "469/469 [==============================] - 4s 8ms/step - loss: 0.1523 - accuracy: 0.9545 - val_loss: 0.1187 - val_accuracy: 0.9627\n",
            "Epoch 3/20\n",
            "469/469 [==============================] - 3s 7ms/step - loss: 0.1112 - accuracy: 0.9663 - val_loss: 0.1008 - val_accuracy: 0.9688\n",
            "Epoch 4/20\n",
            "469/469 [==============================] - 4s 8ms/step - loss: 0.0887 - accuracy: 0.9729 - val_loss: 0.1001 - val_accuracy: 0.9682\n",
            "Epoch 5/20\n",
            "469/469 [==============================] - 4s 8ms/step - loss: 0.0774 - accuracy: 0.9754 - val_loss: 0.1007 - val_accuracy: 0.9712\n",
            "Epoch 6/20\n",
            "469/469 [==============================] - 3s 7ms/step - loss: 0.0647 - accuracy: 0.9792 - val_loss: 0.0906 - val_accuracy: 0.9731\n",
            "Epoch 7/20\n",
            "469/469 [==============================] - 3s 7ms/step - loss: 0.0566 - accuracy: 0.9816 - val_loss: 0.0886 - val_accuracy: 0.9731\n",
            "Epoch 8/20\n",
            "469/469 [==============================] - 4s 8ms/step - loss: 0.0522 - accuracy: 0.9830 - val_loss: 0.0876 - val_accuracy: 0.9756\n",
            "Epoch 9/20\n",
            "469/469 [==============================] - 3s 7ms/step - loss: 0.0474 - accuracy: 0.9848 - val_loss: 0.0910 - val_accuracy: 0.9747\n",
            "Epoch 10/20\n",
            "469/469 [==============================] - 3s 7ms/step - loss: 0.0401 - accuracy: 0.9870 - val_loss: 0.0871 - val_accuracy: 0.9752\n",
            "Epoch 11/20\n",
            "469/469 [==============================] - 3s 7ms/step - loss: 0.0391 - accuracy: 0.9868 - val_loss: 0.0958 - val_accuracy: 0.9733\n",
            "Epoch 12/20\n",
            "469/469 [==============================] - 4s 8ms/step - loss: 0.0358 - accuracy: 0.9884 - val_loss: 0.0841 - val_accuracy: 0.9761\n",
            "Epoch 13/20\n",
            "469/469 [==============================] - 3s 7ms/step - loss: 0.0314 - accuracy: 0.9894 - val_loss: 0.0966 - val_accuracy: 0.9745\n",
            "Epoch 14/20\n",
            "469/469 [==============================] - 3s 7ms/step - loss: 0.0318 - accuracy: 0.9891 - val_loss: 0.0855 - val_accuracy: 0.9782\n",
            "Epoch 15/20\n",
            "469/469 [==============================] - 4s 8ms/step - loss: 0.0292 - accuracy: 0.9902 - val_loss: 0.0837 - val_accuracy: 0.9772\n",
            "Epoch 16/20\n",
            "469/469 [==============================] - 3s 7ms/step - loss: 0.0275 - accuracy: 0.9910 - val_loss: 0.0874 - val_accuracy: 0.9785\n",
            "Epoch 17/20\n",
            "469/469 [==============================] - 3s 7ms/step - loss: 0.0257 - accuracy: 0.9917 - val_loss: 0.1115 - val_accuracy: 0.9728\n",
            "Epoch 18/20\n",
            "469/469 [==============================] - 3s 7ms/step - loss: 0.0247 - accuracy: 0.9921 - val_loss: 0.0939 - val_accuracy: 0.9761\n",
            "Epoch 19/20\n",
            "469/469 [==============================] - 4s 8ms/step - loss: 0.0244 - accuracy: 0.9913 - val_loss: 0.0935 - val_accuracy: 0.9765\n",
            "Epoch 20/20\n",
            "469/469 [==============================] - 3s 7ms/step - loss: 0.0197 - accuracy: 0.9934 - val_loss: 0.1081 - val_accuracy: 0.9740\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f72b67aee80>"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ],
      "source": [
        "model.fit(X_train, y_train, batch_size=128, epochs=20, verbose=1, validation_data=(X_valid, y_valid))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ttxTyteyoxK-"
      },
      "source": [
        "#### Performing inference"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SGFH9CtloxK-"
      },
      "outputs": [],
      "source": [
        "valid_0 = X_valid[0].reshape(1, 784)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tjcfovmFoxK-",
        "outputId": "f1921dbb-8cda-46a2-ee5a-26caa808cd2f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1/1 [==============================] - 0s 175ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[1.2803646e-09, 7.9940527e-08, 4.3187218e-09, 7.9533340e-09,\n",
              "        1.2109734e-07, 5.8355193e-10, 2.5604714e-09, 9.9999893e-01,\n",
              "        1.6771040e-10, 8.1066821e-07]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ],
      "source": [
        "model.predict(valid_0)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "r4g7TQ7soxK-"
      },
      "outputs": [],
      "source": [
        "# model.predict_classes(valid_0)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "d9cyTg1joxK_",
        "outputId": "917592a1-3ac3-4792-df3c-61aeff5cd285",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1/1 [==============================] - 0s 29ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([7])"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ],
      "source": [
        "# The predict_classes() method no longer exists in recent TensorFlow releases.\n",
        "# Instead you could use:\n",
        "import numpy as np\n",
        "np.argmax(model.predict(valid_0), axis=-1)"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}