notebooks/deep_net_in_pytorch.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jonkrohn/DLTFpT/blob/master/notebooks/deep_net_in_pytorch.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VkIRIR7XxpP0"
      },
      "source": [
        "# Deep Neural Network in PyTorch\n",
        "\n",
        "In this notebook, we adapt our [TensorFlow Deep Net](https://github.com/jonkrohn/DLTFpT/blob/master/notebooks/deep_net_in_tensorflow.ipynb) to PyTorch."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XJv0N43uyVsy"
      },
      "source": [
        "#### Load dependencies"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "rAh1QLwWyYvY"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "\n",
        "from torchvision.datasets import MNIST\n",
        "from torchvision import transforms\n",
        "\n",
        "from torchsummary import summary\n",
        "\n",
        "import time # for tracking epoch duration"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g-Nh_zNUyarq"
      },
      "source": [
        "#### Load data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "6vBWZlFGydPY"
      },
      "outputs": [],
      "source": [
        "train = MNIST('data', train=True, transform=transforms.ToTensor(), download=True)\n",
        "test = MNIST('data', train=False, transform=transforms.ToTensor())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Bi5GT2I_yobV"
      },
      "source": [
        "#### Batch data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "wbqPKnEXy9i6"
      },
      "outputs": [],
      "source": [
        "train_loader = torch.utils.data.DataLoader(train, batch_size=128) \n",
        "test_loader = torch.utils.data.DataLoader(test, batch_size=128) "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SgJkRG_hzM1r"
      },
      "source": [
        "#### Design neural network architecture"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "7TnSFYGpzRMD"
      },
      "outputs": [],
      "source": [
        "n_input = 784\n",
        "n_dense_1 = 64\n",
        "n_dense_2 = 64\n",
        "n_dense_3 = 64\n",
        "n_out = 10"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "YAFJBtBYzYcc"
      },
      "outputs": [],
      "source": [
        "model = nn.Sequential(\n",
        "    \n",
        "    # first hidden layer: \n",
        "    nn.Linear(n_input, n_dense_1), \n",
        "    nn.ReLU(), \n",
        "    nn.BatchNorm1d(n_dense_1),\n",
        "    \n",
        "    # second hidden layer: \n",
        "    nn.Linear(n_dense_1, n_dense_2), \n",
        "    nn.ReLU(), \n",
        "    nn.BatchNorm1d(n_dense_2),\n",
        "    \n",
        "    # third hidden layer: \n",
        "    nn.Linear(n_dense_2, n_dense_3), \n",
        "    nn.ReLU(), \n",
        "    nn.BatchNorm1d(n_dense_3),\n",
        "    nn.Dropout(),  \n",
        "    \n",
        "    # output layer: \n",
        "    nn.Linear(n_dense_3, n_out) \n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bjfVGdDp0YDJ",
        "outputId": "c2baf3e7-e1a5-4ca0-b11e-3fca184db33e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "            Linear-1                   [-1, 64]          50,240\n",
            "              ReLU-2                   [-1, 64]               0\n",
            "       BatchNorm1d-3                   [-1, 64]             128\n",
            "            Linear-4                   [-1, 64]           4,160\n",
            "              ReLU-5                   [-1, 64]               0\n",
            "       BatchNorm1d-6                   [-1, 64]             128\n",
            "            Linear-7                   [-1, 64]           4,160\n",
            "              ReLU-8                   [-1, 64]               0\n",
            "       BatchNorm1d-9                   [-1, 64]             128\n",
            "          Dropout-10                   [-1, 64]               0\n",
            "           Linear-11                   [-1, 10]             650\n",
            "================================================================\n",
            "Total params: 59,594\n",
            "Trainable params: 59,594\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.00\n",
            "Forward/backward pass size (MB): 0.00\n",
            "Params size (MB): 0.23\n",
            "Estimated Total Size (MB): 0.24\n",
            "----------------------------------------------------------------\n"
          ]
        }
      ],
      "source": [
        "summary(model, input_size=(n_input,))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nzT_k2fF0gT1"
      },
      "source": [
        "#### Configure training hyperparameters"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "VS9m8bPd0j-T"
      },
      "outputs": [],
      "source": [
        "cost_fxn = nn.CrossEntropyLoss() # includes softmax activation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "9AKQugeh0lhM"
      },
      "outputs": [],
      "source": [
        "optimizer = torch.optim.Adam(model.parameters())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HVVGdPLI0oLk"
      },
      "source": [
        "#### Train"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "OxZMGQHd0s_b"
      },
      "outputs": [],
      "source": [
        "def accuracy_pct(pred_y, true_y):\n",
        "  _, prediction = torch.max(pred_y, 1) # returns maximum values, indices; fed tensor, dim to reduce\n",
        "  correct = (prediction == true_y).sum().item()\n",
        "  return (correct / true_y.shape[0]) * 100.0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "481O1w790pvJ",
        "outputId": "4d0aa182-c34a-40af-b113-0efb476c87e7"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "469"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ],
      "source": [
        "n_batches = len(train_loader)\n",
        "n_batches"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0K-f-uJb0x7A",
        "outputId": "2fd9de6f-8b7e-4e2e-9f79-62d701054934"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training for 10 epochs. \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 1/10 complete. Cost: 0.381, Accuracy: 89.7%, Time: 17.65 seconds \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 2/10 complete. Cost: 0.162, Accuracy: 95.5%, Time: 11.04 seconds \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 3/10 complete. Cost: 0.116, Accuracy: 96.7%, Time: 11.47 seconds \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 4/10 complete. Cost: 0.094, Accuracy: 97.3%, Time: 11.54 seconds \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 5/10 complete. Cost: 0.079, Accuracy: 97.7%, Time: 11.59 seconds \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 6/10 complete. Cost: 0.070, Accuracy: 97.9%, Time: 11.50 seconds \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 7/10 complete. Cost: 0.059, Accuracy: 98.3%, Time: 11.51 seconds \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 8/10 complete. Cost: 0.051, Accuracy: 98.4%, Time: 11.49 seconds \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 9/10 complete. Cost: 0.046, Accuracy: 98.6%, Time: 11.64 seconds \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 10/10 complete. Cost: 0.043, Accuracy: 98.6%, Time: 12.22 seconds \n",
            "\n",
            "Training complete.\n"
          ]
        }
      ],
      "source": [
        "n_epochs = 10 \n",
        "\n",
        "print('Training for {} epochs. \\n'.format(n_epochs))\n",
        "\n",
        "for epoch in range(n_epochs):\n",
        "\n",
        "  start_time = time.time()\n",
        "  \n",
        "  avg_cost = 0.0\n",
        "  avg_accuracy = 0.0\n",
        "  \n",
        "  for i, (X, y) in enumerate(train_loader): # enumerate() provides count of iterations  \n",
        "    \n",
        "    # forward propagation:\n",
        "    X_flat = X.view(X.shape[0], -1)\n",
        "    y_hat = model(X_flat)\n",
        "    cost = cost_fxn(y_hat, y)\n",
        "    avg_cost += cost / n_batches\n",
        "    \n",
        "    # backprop and optimization via gradient descent: \n",
        "    optimizer.zero_grad() # set gradients to zero; .backward() accumulates them in buffers\n",
        "    cost.backward()\n",
        "    optimizer.step()\n",
        "    \n",
        "    # calculate accuracy metric:\n",
        "    accuracy = accuracy_pct(y_hat, y)\n",
        "    avg_accuracy += accuracy / n_batches\n",
        "    \n",
        "    if (i + 1) % 100 == 0:\n",
        "      print('Step {}'.format(i + 1))\n",
        "\n",
        "  end_time = time.time()\n",
        "  time_delta = end_time - start_time\n",
        "    \n",
        "  print('Epoch {}/{} complete. Cost: {:.3f}, Accuracy: {:.1f}%, Time: {:.2f} seconds \\n'\n",
        "        .format(epoch + 1, n_epochs, avg_cost, avg_accuracy, time_delta)) \n",
        "\n",
        "print('Training complete.')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OKqQEbxY0ris"
      },
      "source": [
        "#### Test model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a5RxXcEy04UM",
        "outputId": "71d33051-f23c-4ee7-8f41-afe09a71a34b"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "79"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ],
      "source": [
        "n_test_batches = len(test_loader)\n",
        "n_test_batches"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DqeluNIm06pT",
        "outputId": "9b770339-7240-48aa-dd49-b7583b0c7848"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test cost: 0.105, Test accuracy: 97.3%\n"
          ]
        }
      ],
      "source": [
        "model.eval() # disables dropout and batch norm\n",
        "\n",
        "with torch.no_grad(): # disables autograd, reducing memory consumption\n",
        "  \n",
        "  avg_test_cost = 0.0\n",
        "  avg_test_acc = 0.0\n",
        "  \n",
        "  for X, y in test_loader:\n",
        "    \n",
        "    # make predictions: \n",
        "    X_flat = X.view(X.shape[0], -1)\n",
        "    y_hat = model(X_flat)\n",
        "    \n",
        "    # calculate cost: \n",
        "    cost = cost_fxn(y_hat, y)\n",
        "    avg_test_cost += cost / n_test_batches\n",
        "    \n",
        "    # calculate accuracy:\n",
        "    test_accuracy = accuracy_pct(y_hat, y)\n",
        "    avg_test_acc += test_accuracy / n_test_batches\n",
        "\n",
        "print('Test cost: {:.3f}, Test accuracy: {:.1f}%'.format(avg_test_cost, avg_test_acc))\n",
        "\n",
        "# model.train() # 'undoes' model.eval()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "name": "deep_net_in_pytorch.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}