notebooks/shallow_net_in_pytorch.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LcrCNS6jc5-e"
      },
      "source": [
        "# Shallow Neural Network in PyTorch\n",
        "\n",
        "In this notebook, we adapt our [TensorFlow Shallow Net](https://github.com/jonkrohn/DLTFpT/blob/master/notebooks/shallow_net_in_tensorflow.ipynb) to PyTorch."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jonkrohn/DLTFpT/blob/master/notebooks/shallow_net_in_pytorch.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mEygUgLVKJ7x"
      },
      "source": [
        "#### Load dependencies"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "nfFVh41odSXB"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "\n",
        "from torchvision.datasets import MNIST\n",
        "from torchvision import transforms\n",
        "\n",
        "from torchsummary import summary\n",
        "\n",
        "import matplotlib.pyplot as plt"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6QCoX-S6dj54"
      },
      "source": [
        "#### Load data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "UXk2_Pazef4W",
        "outputId": "6f2f5633-f4fe-455e-f704-c79183298809",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to data/MNIST/raw/train-images-idx3-ubyte.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 9912422/9912422 [00:00<00:00, 61642777.86it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting data/MNIST/raw/train-images-idx3-ubyte.gz to data/MNIST/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to data/MNIST/raw/train-labels-idx1-ubyte.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 28881/28881 [00:00<00:00, 65798855.96it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting data/MNIST/raw/train-labels-idx1-ubyte.gz to data/MNIST/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to data/MNIST/raw/t10k-images-idx3-ubyte.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "100%|██████████| 1648877/1648877 [00:00<00:00, 27069339.45it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting data/MNIST/raw/t10k-images-idx3-ubyte.gz to data/MNIST/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to data/MNIST/raw/t10k-labels-idx1-ubyte.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 4542/4542 [00:00<00:00, 11359885.97it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting data/MNIST/raw/t10k-labels-idx1-ubyte.gz to data/MNIST/raw\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "train = MNIST('data', train=True, transform=transforms.ToTensor(), download=True)\n",
        "test = MNIST('data', train=False, transform=transforms.ToTensor())\n",
        "# ...toTensor() scales pixels from [0, 255] to [0, 1]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EdMVpcOHetgf",
        "outputId": "42c390db-bd29-47db-9aa0-b51ce21e0921"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([60000, 28, 28])"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "train.data.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yZ8hKPF6exuw",
        "outputId": "ed25b96e-ed2c-4c8e-8d39-2a2608e5042e"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
              "           0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
              "           0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
              "           0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
              "           0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
              "           0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   3,  18,\n",
              "          18,  18, 126, 136, 175,  26, 166, 255, 247, 127,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,  30,  36,  94, 154, 170, 253,\n",
              "         253, 253, 253, 253, 225, 172, 253, 242, 195,  64,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,  49, 238, 253, 253, 253, 253, 253,\n",
              "         253, 253, 253, 251,  93,  82,  82,  56,  39,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,  18, 219, 253, 253, 253, 253, 253,\n",
              "         198, 182, 247, 241,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,  80, 156, 107, 253, 253, 205,\n",
              "          11,   0,  43, 154,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,   0,  14,   1, 154, 253,  90,\n",
              "           0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0, 139, 253, 190,\n",
              "           2,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  11, 190, 253,\n",
              "          70,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  35, 241,\n",
              "         225, 160, 108,   1,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  81,\n",
              "         240, 253, 253, 119,  25,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
              "          45, 186, 253, 253, 150,  27,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
              "           0,  16,  93, 252, 253, 187,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
              "           0,   0,   0, 249, 253, 249,  64,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
              "          46, 130, 183, 253, 253, 207,   2,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  39, 148,\n",
              "         229, 253, 253, 253, 250, 182,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  24, 114, 221, 253,\n",
              "         253, 253, 253, 201,  78,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,  23,  66, 213, 253, 253, 253,\n",
              "         253, 198,  81,   2,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,  18, 171, 219, 253, 253, 253, 253, 195,\n",
              "          80,   9,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,  55, 172, 226, 253, 253, 253, 253, 244, 133,  11,\n",
              "           0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0, 136, 253, 253, 253, 212, 135, 132,  16,   0,   0,\n",
              "           0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
              "           0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
              "           0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
              "        [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
              "           0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0]],\n",
              "       dtype=torch.uint8)"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "train.data[0] # not scaled! "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 430
        },
        "id": "vzITvYebe6NQ",
        "outputId": "e25150cd-54c3-4737-bd14-d39ccfdc19cd"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGdCAYAAABU0qcqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAbz0lEQVR4nO3df2zU9R3H8dcV6AnYXq21vZ4UVlBhitSJ0DUoojSULmGgZPHXNjAGhRUdIuo6f6CbSTfMnFGZ/rGNzkzwVwSC2Vig2BJnYVIhjG02tKmjBFomS+9KkULoZ38Qb54U4Xve9d0rz0dyib27d+/t10uffrnj6nPOOQEA0MfSrBcAAJyfCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADAx2HqBL+vp6dGBAweUkZEhn89nvQ4AwCPnnDo7OxUKhZSWdubznH4XoAMHDqigoMB6DQDA19Ta2qoRI0ac8fZ+F6CMjAxJpxbPzMw03gYA4FUkElFBQUH05/mZJC1AK1eu1LPPPqu2tjYVFRXpxRdf1OTJk8869/kfu2VmZhIgAEhhZ3sZJSlvQnjjjTe0dOlSLV++XB999JGKiopUVlamQ4cOJePhAAApKCkBeu6557RgwQLdfffduvLKK/XKK69o2LBh+v3vf5+MhwMApKCEB+j48eNqaGhQaWnp/x8kLU2lpaWqr68/7f7d3d2KRCIxFwDAwJfwAH366ac6efKk8vLyYq7Py8tTW1vbafevqqpSIBCIXngHHACcH8z/ImplZaXC4XD00traar0SAKAPJPxdcDk5ORo0aJDa29tjrm9vb1cwGDzt/n6/X36/P9FrAAD6uYSfAaWnp2vixImqqamJXtfT06OamhqVlJQk+uEAACkqKX8PaOnSpZo3b56uu+46TZ48Wc8//7y6urp09913J+PhAAApKCkBuu222/Sf//xHTz75pNra2nTNNddo48aNp70xAQBw/vI555z1El8UiUQUCAQUDof5JAQASEHn+nPc/F1wAIDzEwECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGBisPUCQH9y8uRJzzPhcDgJmyTGSy+9FNfc0aNHPc80NjZ6nlm5cqXnmWXLlnmeWbNmjecZSbrgggs8z/zkJz/xPLN8+XLPMwMBZ0AAABMECABgIuEBeuqpp+Tz+WIu48aNS/TDAABSXFJeA7rqqqu0efPm/z/IYF5qAgDESkoZBg8erGAwmIxvDQAYIJLyGtDevXsVCoU0evRo3XXXXdq3b98Z79vd3a1IJBJzAQAMfAkPUHFxsaqrq7Vx40a9/PLLamlp0Q033KDOzs5e719VVaVAIBC9FBQUJHolAEA/lPAAlZeX63vf+54mTJigsrIy/elPf1JHR4fefPPNXu9fWVmpcDgcvbS2tiZ6JQBAP5T0dwdkZWXpiiuuUFNTU6+3+/1++f3+ZK8BAOhnkv73gI4cOaLm5mbl5+cn+6EAACkk4QFatmyZ6urq9Mknn+iDDz7QLbfcokGDBumOO+5I9EMBAFJYwv8Ibv/+/brjjjt0+PBhXXLJJbr++uu1bds2XXLJJYl+KABACkt4gF5//fVEf0v0U1/19vozOX78uOeZDz74wPPM+++/73lGkjo6OjzPvP3223E91kATzztY77//fs8za9eu9TyTkZHheUaSioqKPM/ceOONcT3W+YjPggMAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATCT9F9Kh/9u5c2dcczfffLPnmXA4HNdjoW8NGjTI88wzzzzjeWb48OGeZ+666y7PM6FQyPOMJF100UWeZ8aOHRvXY52POAMCAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACT4NGxo1alRcczk5OZ5n+DTsU4qLiz3PxPPJzO+9957nGUlKT0/3PPODH/wgrsfC+YszIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABB9GCmVnZ8c19+yzz3qe2bBhg+eZb33rW55nHnjgAc8z8brmmms8z2zevNnzzPDhwz3P7Nmzx/OMJL3wwgtxzQFecAYEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJjwOeec9RJfFIlEFAgEFA6HlZmZab0OEiwSiXieycjI8Dxz3333eZ6RpN/+9reeZ/74xz96nrnzzjs9zwCp4lx/jnMGBAAwQYAAACY8B2jr1q2aNWuWQqGQfD6f1q1bF3O7c05PPvmk8vPzNXToUJWWlmrv3r2J2hcAMEB4DlBXV5eKioq0cuXKXm9fsWKFXnjhBb3yyivavn27hg8frrKyMh07duxrLwsAGDg8/0bU8vJylZeX93qbc07PP/+8Hn/8cc2ePVuS9OqrryovL0/r1q3T7bff/vW2BQAMGAl9DailpUVtbW0qLS2NXhcIBFRcXKz6+vpeZ7q7uxWJRGIuAICBL6EBamtrkyTl5eXFXJ+Xlxe97cuqqqoUCASil4KCgkSuBADop8zfBVdZWalwOBy9tLa2Wq8EAOgDCQ1QMBiUJLW3t8dc397eHr3ty/x+vzIzM2MuAICBL6EBKiwsVDAYVE1NTfS6SCSi7du3q6SkJJEPBQBIcZ7fBXfkyBE1NTVFv25padGuXbuUnZ2tkSNHasmSJXrmmWd0+eWXq7CwUE888YRCoZDmzJmTyL0BACnOc4B27Nihm266Kfr10qVLJUnz5s1TdXW1HnnkEXV1denee+9VR0eHrr/+em3cuFEXXHBB4rYGAKQ8PowUA9LDDz8c19yvfvUrzzPTpk3zPLN582bPM2lp5u8ZAs4JH0YKAOjXCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYMLzr2MAUsFTTz0V11xDQ4PnmdraWs8z8Xwa9owZMzzPAP0ZZ0AAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAmfc85ZL/FFkUhEgUBA4XBYmZmZ1uvgPNPc3Ox55tprr/U8k5WV5Xnmpptu8jxz3XXXeZ6RpIqKCs8zPp8vrsfCwHOuP8c5AwIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATAy2XgDoT8aMGeN5prq62vPM3Xff7Xnm1Vdf7ZMZSerq6vI888Mf/tDzTH5+vucZDBycAQEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJnzOOWe9xBdFIhEFAgGFw2FlZmZarwMkxd///nfPMw899JDnmc2bN3ueidfChQs9zzz22GOeZy699FLPM+hb5/pznDMgAIAJAgQAMOE5QFu3btWsWbMUCoXk8/m0bt26mNvnz58vn88Xc5k5c2ai9gUADBCeA9TV1aWioiKtXLnyjPeZOXOmDh48GL2sWbPmay0JABh4PP9G1PLycpWXl3/lffx+v4LBYNxLAQAGvqS8BlRbW6vc3FyNHTtWixYt0uHDh8943+7ubkUikZgLAGDgS3iAZs6cqVdffVU1NTX65S9/qbq6OpWXl+vkyZO93r+qqkqBQCB6KSgoSPRKAIB+yPMfwZ3N7bffHv3nq6++WhMmTNCYMWNUW1ur6dOnn3b/yspKLV26NPp1JBIhQgBwHkj627BHjx6tnJwcNTU19Xq73+9XZmZmzAUAMPAlPUD79+/X4cOHlZ+fn+yHAgCkEM9/BHfkyJGYs5mWlhbt2rVL2dnZys7O1tNPP625c+cqGAyqublZjzzyiC677DKVlZUldHEAQGrzHKAdO3bopptuin79+es38+bN08svv6zdu3frD3/4gzo6OhQKhTRjxgz9/Oc/l9/vT9zWAICUx4eRAimio6PD88yGDRvieqz58+d7nonnR0lvb0w6m02bNnmeQd/iw0gBAP0aAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATPBp2ABOE8+vTzlx4oTnmSFDhnie+ctf/uJ5Ztq0aZ5nED8+DRsA0K8RIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYGWy8AnI92797teebtt9/2PPPhhx96npHi+2DReFx55ZWeZ6ZOnZqETWCBMyAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQfRgp8QWNjo+eZF1980fPMO++843mmra3N80xfGjzY+4+T/Px8zzNpafx/80DBf0kAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQfRop+L54P4Vy9enVcj/XSSy95nvnkk0/ieqz+bNKkSZ5nHnvsMc8z3/3udz3PYODgDAgAYIIAAQBMeApQVVWVJk2apIyMDOXm5mrOnDmn/f6UY8eOqaKiQhdffLEuvPBCzZ07V+3t7QldGgCQ+jwFqK6uThUVFdq2bZs2bdqkEydOaMaMGerq6ore58EHH9SGDRv01ltvqa6uTgcOHNCtt96a8MUBAKnN05sQNm7cGPN1dXW1cnNz1dDQoKlTpyocDut3v/udVq9erZtvvlmStGrVKn3zm9/Utm3b9O1vfztxmwMAUtrXeg0oHA5LkrKzsyVJDQ0NOnHihEpLS6P3GTdunEaOHKn6+vpev0d3d7cikUjMBQAw8MUdoJ6eHi1ZskRTpkzR+PHjJZ16u2x6erqysrJi7puXl3fGt9JWVVUpEAhELwUFBfGuBABIIXEHqKKiQnv27NHrr7/+tRaorKxUOByOXlpbW7/W9wMApIa4/iLq4sWL9e6772rr1q0aMWJE9PpgMKjjx4+ro6Mj5iyovb1dwWCw1+/l9/vl9/vjWQMAkMI8nQE557R48WKtXbtWW7ZsUWFhYcztEydO1JAhQ1RTUxO9rrGxUfv27VNJSUliNgYADAiezoAqKiq0evVqrV+/XhkZGdHXdQKBgIYOHapAIKB77rlHS5cuVXZ2tjIzM3X//ferpKSEd8ABAGJ4CtDLL78sSZo2bVrM9atWrdL8+fMlSb/+9a+VlpamuXPnqru7W2VlZfrNb36TkGUBAAOHzznnrJf4okgkokAgoHA4rMzMTOt18BXi+YSLf/zjH55nFi9e7Hnm448/9jzT3xUXF3ueeeSRR+J6rNmzZ3ueSUvjk71wyrn+HOcZAwAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABNx/UZU9F///e9/Pc/cd999cT3Wrl27PM80NzfH9Vj92ZQpUzzPPPTQQ55nysrKPM8MHTrU8wzQVzgDAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBM8GGkfWT79u2eZ1asWOF55sMPP/Q8s3//fs8z/d2wYcPimnvggQc8zzz22GOeZ4YPH+55BhhoOAMCAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEzwYaR9ZO3atX0y05euvPJKzzOzZs3yPDNo0CDPM8uWLfM8I0lZWVlxzQHwjjMgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMCEzznnrJf4okgkokAgoHA4rMzMTOt1AAAenevPcc6AAAAmCBAAwISnAFVVVWnSpEnKyMhQbm6u5syZo8bGxpj7TJs2TT6fL+aycOHChC4NAEh9ngJUV1eniooKbdu2TZs2bdKJEyc0Y8YMdXV1xdxvwYIFOnjwYPSyYsWKhC4NAEh9nn4j6saNG2O+rq6uVm5urhoaGjR16tTo9cOGDVMwGEzMhgCAAelrvQYUDoclSdnZ2THXv/baa8rJydH48eNVWVmpo0ePnvF7dHd3KxKJxFwAAAOfpzOgL+rp6dGSJUs0ZcoUjR8/Pnr9nXfeqVGjRikUCmn37t169NFH1djYqHfeeafX71NVVaWnn3463jUAACkq7r8HtGjRIv35z3/W+++/rxEjRpzxflu2bNH06dPV1NSkMWPGnHZ7d3e3uru7o19HIhEVFBTw94AAIEWd698DiusMaPHixXr33Xe1devWr4yPJBUXF0vSGQPk9/vl9/vjWQMAkMI8Bcg5p/vvv19r165VbW2tCgsLzzqza9cuSVJ+fn5cCwIABiZPAaqoqNDq1au1fv16ZWRkqK2tTZIUCAQ0dOhQNTc3a/Xq1frOd76jiy++WLt379aDDz6oqVOnasKECUn5FwAApCZPrwH5fL5er1+1apXmz5+v1tZWff/739eePXvU1dWlgoIC3XLLLXr88cfP+fUcPgsOAFJbUl4DOlurCgoKVFdX5+VbAgDOU3wWHADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADAxGDrBb7MOSdJikQixpsAAOLx+c/vz3+en0m/C1BnZ6ckqaCgwHgTAMDX0dnZqUAgcMbbfe5siepjPT09OnDggDIyMuTz+WJui0QiKigoUGtrqzIzM402tMdxOIXjcArH4RSOwyn94Tg459TZ2alQKKS0tDO/0tPvzoDS0tI0YsSIr7xPZmbmef0E+xzH4RSOwykch1M4DqdYH4evOvP5HG9CAACYIEAAABMpFSC/36/ly5fL7/dbr2KK43AKx+EUjsMpHIdTUuk49Ls3IQAAzg8pdQYEABg4CBAAwAQBAgCYIEAAABMpE6CVK1fqG9/4hi644AIVFxfrb3/7m/VKfe6pp56Sz+eLuYwbN856raTbunWrZs2apVAoJJ/Pp3Xr1sXc7pzTk08+qfz8fA0dOlSlpaXau3evzbJJdLbjMH/+/NOeHzNnzrRZNkmqqqo0adIkZWRkKDc3V3PmzFFjY2PMfY4dO6aKigpdfPHFuvDCCzV37ly1t7cbbZwc53Icpk2bdtrzYeHChUYb9y4lAvTGG29o6dKlWr58uT766CMVFRWprKxMhw4dsl6tz1111VU6ePBg9PL+++9br5R0XV1dKioq0sqVK3u9fcWKFXrhhRf0yiuvaPv27Ro+fLjKysp07NixPt40uc52HCRp5syZMc+PNWvW9OGGyVdXV6eKigpt27ZNmzZt0okTJzRjxgx1dXVF7/Pggw9qw4YNeuutt1RXV6cDBw7o1ltvNdw68c7lOEjSggULYp4PK1asMNr4DFwKmDx5squoqIh+ffLkSRcKhVxVVZXhVn1v+fLlrqioyHoNU5Lc2rVro1/39PS4YDDonn322eh1HR0dzu/3uzVr1hhs2De+fBycc27evHlu9uzZJvtYOXTokJPk6urqnHOn/tsPGTLEvfXWW9H7/Otf/3KSXH19vdWaSffl4+CcczfeeKP78Y9/bLfUOej3Z0DHjx9XQ0ODSktLo9elpaWptLRU9fX1hpvZ2Lt3r0KhkEaPHq277rpL+/bts17JVEtLi9ra2mKeH4FAQMXFxefl86O2tla5ubkaO3asFi1apMOHD1uvlFThcFiSlJ2dLUlqaGjQiRMnYp4P48aN08iRIwf08+HLx+Fzr732mnJycjR+/HhVVlbq6NGjFuudUb/7MNIv+/TTT3Xy5Enl5eXFXJ+Xl6ePP/7YaCsbxcXFqq6u1tixY3Xw4EE9/fTTuuGGG7Rnzx5lZGRYr2eira1Nknp9fnx+2/li5syZuvXWW1VYWKjm5mb99Kc/VXl5uerr6zVo0CDr9RKup6dHS5Ys0ZQpUzR+/HhJp54P6enpysrKirnvQH4+9HYcJOnOO+/UqFGjFAqFtHv3bj366KNqbGzUO++8Y7htrH4fIPxfeXl59J8nTJig4uJijRo1Sm+++abuuecew83QH9x+++3Rf7766qs1YcIEjRkzRrW1tZo+fbrhZslRUVGhPXv2nBevg36VMx2He++9N/rPV199tfLz8zV9+nQ1NzdrzJgxfb1mr/r9H8Hl5ORo0KBBp72Lpb29XcFg0Gir/iErK0tXXHGFmpqarFcx8/lzgOfH6UaPHq2cnJwB+fxYvHix3n33Xb333nsxv74lGAzq+PHj6ujoiLn/QH0+nOk49Ka4uFiS+tXzod8HKD09XRMnTlRNTU30up6eHtXU1KikpMRwM3tHjhxRc3Oz8vPzrVcxU1hYqGAwGPP8iEQi2r59+3n//Ni/f78OHz48oJ4fzjktXrxYa9eu1ZYtW1RYWBhz+8SJEzVkyJCY50NjY6P27ds3oJ4PZzsOvdm1a5ck9a/ng/W7IM7F66+/7vx+v6uurnb//Oc/3b333uuysrJcW1ub9Wp96qGHHnK1tbWupaXF/fWvf3WlpaUuJyfHHTp0yHq1pOrs7HQ7d+50O3fudJLcc88953bu3On+/e9/O+ec+8UvfuGysrLc+vXr3e7du93s2bNdYWGh++yzz4w3T6yvOg6dnZ1u2bJlrr6+3rW0tLjNmze7a6+91l1++eXu2LFj1qsnzKJFi1wgEHC1tbXu4MGD0cvRo0ej91m4cKEbOXKk27Jli9uxY4crKSlxJSUlhlsn3tmOQ1NTk/vZz37mduzY4VpaWtz69evd6NGj3dSpU403j5USAXLOuRdffNGNHDnSpaenu8mTJ7tt27ZZr9TnbrvtNpefn+/S09PdpZde6m677TbX1NRkvVbSvffee07SaZd58+Y55069FfuJJ55weXl5zu/3u+nTp7vGxkbbpZPgq47D0aNH3YwZM9wll1zihgwZ4kaNGuUWLFgw4P4nrbd/f0lu1apV0ft89tln7kc/+pG76KKL3LBhw9wtt9ziDh48aLd0EpztOOzbt89NnTrVZWdnO7/f7y677DL38MMPu3A4bLv4l/DrGAAAJvr9a0AAgIGJAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADDxPwVDG1RxUx1zAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "_ = plt.imshow(train.data[0].numpy().squeeze(), cmap='gray_r') # squeeze() removes dims of length one"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kf4i8Q09fCpw",
        "outputId": "1164ec3a-35ca-45ef-b805-8e7d5b14286c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([5, 0, 4, 1, 9, 2, 1, 3, 1, 4, 3, 5, 3, 6, 1, 7, 2, 8, 6, 9, 4, 0, 9, 1,\n",
              "        1, 2, 4, 3, 2, 7, 3, 8, 6, 9, 0, 5, 6, 0, 7, 6, 1, 8, 7, 9, 3, 9, 8, 5,\n",
              "        9, 3, 3, 0, 7, 4, 9, 8, 0, 9, 4, 1, 4, 4, 6, 0, 4, 5, 6, 1, 0, 0, 1, 7,\n",
              "        1, 6, 3, 0, 2, 1, 1, 7, 9, 0, 2, 6, 7, 8, 3, 9, 0, 4, 6, 7, 4, 6, 8, 0,\n",
              "        7, 8, 3, 1])"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ],
      "source": [
        "train.targets[0:100]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e6d0sd_8fQPH",
        "outputId": "3e05f124-4b09-4753-c6c8-337e7d978df3"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([60000])"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ],
      "source": [
        "train.targets.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0qST6tDffR9v",
        "outputId": "584143dc-d81c-4c7f-a6dd-955af194d61d"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([10000, 28, 28])"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "test.data.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WyPpoB3dfiwo",
        "outputId": "f603adf2-85b9-408b-f6e6-2407661f1d17"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([10000])"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ],
      "source": [
        "test.targets.shape"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yYMXOGJvfkCo"
      },
      "source": [
        "#### Batch data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "8IySNK8Ef6Eq"
      },
      "outputs": [],
      "source": [
        "train_loader = torch.utils.data.DataLoader(train, batch_size=128, shuffle=True) \n",
        "test_loader = torch.utils.data.DataLoader(test, batch_size=128) \n",
        "# ...DataLoader() can also sample and run multithreaded over a set number of workers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "SLfWTrjshEsh"
      },
      "outputs": [],
      "source": [
        "X_sample, y_sample = next(iter(train_loader))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5fc-M4JCiRTW",
        "outputId": "b9408cf6-d414-4062-bbb2-f36e65737d61"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([128, 1, 28, 28])"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ],
      "source": [
        "X_sample.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2TCZnZR-iWvo",
        "outputId": "bd34ac75-7869-4428-e15e-bbad6eb30678"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([128])"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ],
      "source": [
        "y_sample.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8uk1_iPDiXR7",
        "outputId": "70600b6d-fcba-469b-bb6a-39de857e36da"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([8, 7, 5, 3, 9, 2, 9, 8, 2, 3, 1, 1, 6, 9, 8, 1, 1, 0, 9, 5, 9, 4, 1, 4,\n",
              "        0, 3, 3, 7, 3, 4, 2, 1, 4, 4, 0, 1, 5, 0, 8, 4, 1, 6, 1, 6, 2, 9, 9, 4,\n",
              "        7, 4, 7, 6, 7, 3, 5, 0, 6, 9, 9, 7, 4, 3, 0, 4, 0, 1, 4, 0, 8, 3, 9, 3,\n",
              "        3, 2, 4, 9, 0, 6, 1, 1, 4, 8, 2, 9, 6, 8, 6, 2, 6, 9, 1, 4, 1, 8, 5, 9,\n",
              "        1, 5, 0, 0, 5, 2, 5, 3, 7, 2, 7, 4, 2, 4, 7, 3, 1, 8, 3, 8, 2, 7, 4, 8,\n",
              "        1, 9, 1, 0, 3, 6, 0, 8])"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ],
      "source": [
        "y_sample"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Sf4z56RKi1ja",
        "outputId": "73bb1f00-42bc-4f27-c0c4-88c0c1858015"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[[0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000, 0.0392, 0.3098, 0.5529, 0.8471,\n",
              "          0.6980, 0.1137, 0.0510, 0.0275, 0.1137, 0.0627, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0275, 0.3216, 0.7725, 0.9882, 0.9922, 0.9882,\n",
              "          0.9882, 0.9882, 0.8118, 0.7333, 0.9882, 0.8431, 0.1020, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.6039, 0.9882, 0.9882, 0.9882, 0.9922, 0.9882,\n",
              "          0.9882, 0.9882, 0.9922, 0.9882, 0.9882, 0.9882, 0.4431, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.2000, 0.9922, 0.9882, 0.9882, 0.5922, 0.5020, 0.1098,\n",
              "          0.1098, 0.3059, 0.5490, 0.8431, 0.9882, 0.9882, 0.6392, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.6902, 1.0000, 0.9686, 0.3922, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0157, 0.7020, 0.9922, 0.9922, 0.2000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.8824, 0.9922, 0.7686, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.5020, 0.9882, 0.9882, 0.5922, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.5373, 0.9922, 0.8431, 0.0745, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.2980, 0.9922, 0.9882, 0.8784, 0.0745, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.2471, 0.9922, 0.9882, 0.6118, 0.0000, 0.0000, 0.0000,\n",
              "          0.4941, 0.9765, 0.9922, 0.8902, 0.1686, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.5059, 0.9922, 0.8471, 0.1098, 0.0627, 0.5020,\n",
              "          0.9922, 0.9922, 0.8392, 0.1490, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0392, 0.7686, 0.9882, 0.6471, 0.8471, 0.9882,\n",
              "          0.9882, 0.8902, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.4431, 0.9176, 0.9882, 0.9922, 0.9882,\n",
              "          0.8784, 0.1686, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.2980, 0.8706, 0.9882, 0.9922, 0.9882,\n",
              "          0.2667, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.1137, 0.8471, 0.9922, 0.9922, 0.9961, 0.8941,\n",
              "          0.1608, 0.0157, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.8471, 0.9882, 0.9882, 0.5922, 0.7961, 0.9882,\n",
              "          0.8431, 0.4039, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.1490, 0.9922, 0.9882, 0.7294, 0.0235, 0.0745, 0.8784,\n",
              "          0.9882, 0.8941, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.4431, 0.9922, 0.8902, 0.1686, 0.0000, 0.0000, 0.6627,\n",
              "          0.9882, 0.7922, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.4471, 0.9961, 0.8941, 0.1216, 0.0000, 0.1137, 0.7020,\n",
              "          0.9922, 0.9922, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.1490, 0.9922, 0.9882, 0.5843, 0.4196, 0.9922, 0.9882,\n",
              "          0.9882, 0.8902, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.9922, 0.9882, 0.9882, 0.9882, 0.9922, 0.9882,\n",
              "          0.8784, 0.1686, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.1569, 0.8431, 0.9882, 0.9882, 0.5490, 0.2549,\n",
              "          0.0745, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000],\n",
              "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "          0.0000, 0.0000, 0.0000, 0.0000]]])"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ],
      "source": [
        "X_sample[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "zlj29yCske1c"
      },
      "outputs": [],
      "source": [
        "X_flat_sample = X_sample.view(X_sample.shape[0], -1) # view() reshapes Tensor (confusingly)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8VtnEEvLi4UD",
        "outputId": "6c9a79df-3fb4-4304-8eb3-4d753e3802f9"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([128, 784])"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ],
      "source": [
        "X_flat_sample.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Koet4ZSgkSHu",
        "outputId": "262431bd-78bc-4c3e-c517-ad6cf38fd500"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0392,\n",
              "        0.3098, 0.5529, 0.8471, 0.6980, 0.1137, 0.0510, 0.0275, 0.1137, 0.0627,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0275, 0.3216,\n",
              "        0.7725, 0.9882, 0.9922, 0.9882, 0.9882, 0.9882, 0.8118, 0.7333, 0.9882,\n",
              "        0.8431, 0.1020, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.6039,\n",
              "        0.9882, 0.9882, 0.9882, 0.9922, 0.9882, 0.9882, 0.9882, 0.9922, 0.9882,\n",
              "        0.9882, 0.9882, 0.4431, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.2000,\n",
              "        0.9922, 0.9882, 0.9882, 0.5922, 0.5020, 0.1098, 0.1098, 0.3059, 0.5490,\n",
              "        0.8431, 0.9882, 0.9882, 0.6392, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.6902, 1.0000, 0.9686, 0.3922, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0157, 0.7020, 0.9922, 0.9922, 0.2000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.8824, 0.9922, 0.7686, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.5020, 0.9882, 0.9882, 0.5922, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.5373, 0.9922, 0.8431, 0.0745, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.2980, 0.9922, 0.9882, 0.8784, 0.0745, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.2471, 0.9922, 0.9882, 0.6118, 0.0000, 0.0000,\n",
              "        0.0000, 0.4941, 0.9765, 0.9922, 0.8902, 0.1686, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.5059, 0.9922, 0.8471, 0.1098,\n",
              "        0.0627, 0.5020, 0.9922, 0.9922, 0.8392, 0.1490, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0392, 0.7686, 0.9882,\n",
              "        0.6471, 0.8471, 0.9882, 0.9882, 0.8902, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.4431,\n",
              "        0.9176, 0.9882, 0.9922, 0.9882, 0.8784, 0.1686, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.2980, 0.8706, 0.9882, 0.9922, 0.9882, 0.2667, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.1137, 0.8471, 0.9922, 0.9922, 0.9961, 0.8941, 0.1608, 0.0157, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.8471, 0.9882, 0.9882, 0.5922, 0.7961, 0.9882, 0.8431, 0.4039,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.1490, 0.9922, 0.9882, 0.7294, 0.0235, 0.0745, 0.8784, 0.9882,\n",
              "        0.8941, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.4431, 0.9922, 0.8902, 0.1686, 0.0000, 0.0000, 0.6627,\n",
              "        0.9882, 0.7922, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.4471, 0.9961, 0.8941, 0.1216, 0.0000, 0.1137,\n",
              "        0.7020, 0.9922, 0.9922, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.1490, 0.9922, 0.9882, 0.5843, 0.4196,\n",
              "        0.9922, 0.9882, 0.9882, 0.8902, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.9922, 0.9882, 0.9882,\n",
              "        0.9882, 0.9922, 0.9882, 0.8784, 0.1686, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.1569, 0.8431,\n",
              "        0.9882, 0.9882, 0.5490, 0.2549, 0.0745, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "        0.0000])"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ],
      "source": [
        "X_flat_sample[0]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s-lTFFhdkogC"
      },
      "source": [
        "#### Design neural network architecture"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "mkwmSP10k8Y7"
      },
      "outputs": [],
      "source": [
        "n_input = 784\n",
        "n_dense = 64\n",
        "n_out = 10"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "XLAYTQcrk208"
      },
      "outputs": [],
      "source": [
        "model = nn.Sequential(\n",
        "    nn.Linear(n_input, n_dense), # hidden layer\n",
        "    nn.Sigmoid(), # activation function\n",
        "    nn.Linear(n_dense, n_out) # output layer\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ELhOOEzcmMjN",
        "outputId": "6d67c1c3-1ebb-4643-997a-a7883eabb5eb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "            Linear-1                [-1, 1, 64]          50,240\n",
            "           Sigmoid-2                [-1, 1, 64]               0\n",
            "            Linear-3                [-1, 1, 10]             650\n",
            "================================================================\n",
            "Total params: 50,890\n",
            "Trainable params: 50,890\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.00\n",
            "Forward/backward pass size (MB): 0.00\n",
            "Params size (MB): 0.19\n",
            "Estimated Total Size (MB): 0.20\n",
            "----------------------------------------------------------------\n"
          ]
        }
      ],
      "source": [
        "summary(model, (1, n_input))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ad12qUaumQNF"
      },
      "source": [
        "#### Configure training hyperparameters"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "id": "MApHSXQGmfdE"
      },
      "outputs": [],
      "source": [
        "cost_fxn = nn.CrossEntropyLoss() # includes softmax activation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "K0c1Iqnpm32m"
      },
      "outputs": [],
      "source": [
        "optimizer = torch.optim.SGD(model.parameters(), lr=0.01)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V3836ImZnQ6_"
      },
      "source": [
        "#### Train"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "p9iTkEv6nVHB"
      },
      "outputs": [],
      "source": [
        "def accuracy_pct(pred_y, true_y):\n",
        "  _, prediction = torch.max(pred_y, 1) # returns maximum values, indices; fed tensor, dim to reduce\n",
        "  correct = (prediction == true_y).sum().item()\n",
        "  return (correct / true_y.shape[0]) * 100.0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qkd4YuS4xBaF",
        "outputId": "eebfb27b-934b-4963-bc74-2db9b9f17dd6"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "469"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ],
      "source": [
        "n_batches = len(train_loader)\n",
        "n_batches"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ka2rxjOcnY_1",
        "outputId": "ee219fe6-2ae4-4ec6-864a-e770572a1c98"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training for 20 epochs. \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 1/20 complete. Cost: 2.252, Accuracy: 29.2% \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 2/20 complete. Cost: 2.092, Accuracy: 57.5% \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 3/20 complete. Cost: 1.843, Accuracy: 64.4% \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 4/20 complete. Cost: 1.543, Accuracy: 69.6% \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 5/20 complete. Cost: 1.282, Accuracy: 74.1% \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 6/20 complete. Cost: 1.086, Accuracy: 77.6% \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 7/20 complete. Cost: 0.944, Accuracy: 80.3% \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 8/20 complete. Cost: 0.840, Accuracy: 81.9% \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 9/20 complete. Cost: 0.760, Accuracy: 83.4% \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 10/20 complete. Cost: 0.699, Accuracy: 84.4% \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 11/20 complete. Cost: 0.650, Accuracy: 85.2% \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 12/20 complete. Cost: 0.610, Accuracy: 85.9% \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 13/20 complete. Cost: 0.577, Accuracy: 86.5% \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 14/20 complete. Cost: 0.549, Accuracy: 86.9% \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 15/20 complete. Cost: 0.525, Accuracy: 87.4% \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 16/20 complete. Cost: 0.504, Accuracy: 87.6% \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 17/20 complete. Cost: 0.486, Accuracy: 87.9% \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 18/20 complete. Cost: 0.471, Accuracy: 88.2% \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 19/20 complete. Cost: 0.457, Accuracy: 88.5% \n",
            "\n",
            "Step 100\n",
            "Step 200\n",
            "Step 300\n",
            "Step 400\n",
            "Epoch 20/20 complete. Cost: 0.444, Accuracy: 88.7% \n",
            "\n",
            "Training complete.\n"
          ]
        }
      ],
      "source": [
        "n_epochs = 20 \n",
        "\n",
        "print('Training for {} epochs. \\n'.format(n_epochs))\n",
        "\n",
        "for epoch in range(n_epochs):\n",
        "  \n",
        "  avg_cost = 0.0\n",
        "  avg_accuracy = 0.0\n",
        "  \n",
        "  for i, (X, y) in enumerate(train_loader): # enumerate() provides count of iterations  \n",
        "    \n",
        "    # forward propagation:\n",
        "    X_flat = X.view(X.shape[0], -1)\n",
        "    y_hat = model(X_flat)\n",
        "    cost = cost_fxn(y_hat, y)\n",
        "    avg_cost += cost / n_batches\n",
        "    \n",
        "    # backprop and optimization via gradient descent: \n",
        "    optimizer.zero_grad() # set gradients to zero; .backward() accumulates them in buffers\n",
        "    cost.backward()\n",
        "    optimizer.step()\n",
        "    \n",
        "    # calculate accuracy metric:\n",
        "    accuracy = accuracy_pct(y_hat, y)\n",
        "    avg_accuracy += accuracy / n_batches\n",
        "    \n",
        "    if (i + 1) % 100 == 0:\n",
        "      print('Step {}'.format(i + 1))\n",
        "    \n",
        "  print('Epoch {}/{} complete. Cost: {:.3f}, Accuracy: {:.1f}% \\n'\n",
        "        .format(epoch + 1, n_epochs, avg_cost, avg_accuracy)) \n",
        "\n",
        "print('Training complete.')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xmUI1Z7jj0XO"
      },
      "source": [
        "#### Test model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r1-3E_YfkIL7",
        "outputId": "5712ccc3-93a6-4130-e850-2b787a0b348c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "79"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ],
      "source": [
        "n_test_batches = len(test_loader)\n",
        "n_test_batches"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DPU-rP_7jtk_",
        "outputId": "3ad7b7c4-72a9-472d-8754-a9cc0e68fa5b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test cost: 0.421, Test accuracy: 89.5%\n"
          ]
        }
      ],
      "source": [
        "model.eval() # disables dropout (and batch norm)\n",
        "\n",
        "with torch.no_grad(): # disables autograd, reducing memory consumption\n",
        "  \n",
        "  avg_test_cost = 0.0\n",
        "  avg_test_acc = 0.0\n",
        "  \n",
        "  for X, y in test_loader:\n",
        "    \n",
        "    # make predictions: \n",
        "    X_flat = X.view(X.shape[0], -1)\n",
        "    y_hat = model(X_flat)\n",
        "    \n",
        "    # calculate cost: \n",
        "    cost = cost_fxn(y_hat, y)\n",
        "    avg_test_cost += cost / n_test_batches\n",
        "    \n",
        "    # calculate accuracy:\n",
        "    test_accuracy = accuracy_pct(y_hat, y)\n",
        "    avg_test_acc += test_accuracy / n_test_batches\n",
        "\n",
        "print('Test cost: {:.3f}, Test accuracy: {:.1f}%'.format(avg_test_cost, avg_test_acc))\n",
        "\n",
        "# model.train() # 'undoes' model.eval()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "TPU",
    "colab": {
      "name": "shallow_net_in_pytorch.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}