SGD_MNIST_Putting_it_all_together.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "SGD-MNIST - Putting it all together.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JprzQyR_SXuT"
      },
      "source": [
        "THIS code is a summary of the code from the book FASTAI for practionersPractical deep learning for devlopers)\n",
        "Chapter 4: MNIST basics \n",
        "https://colab.research.google.com/github/fastai/fastbook/blob/master/04_mnist_basics.ipynb\n",
        "\n",
        "This code is to build a Gradient descent from Scratch based on the MNIST data set.\n",
        "In this code we are only interested in identifying whether an image is more likely to be a 3 or a 7 ( limited set of the MNIST dataset)\n",
        "As the code is dispersed along the text book with deeper explanation. I decided to put it all together in one block.\n",
        "\n",
        "I strongly recommend to go through the whole chapter for more in depth explanation and keep this code as summary"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9987OHN8H7uS",
        "outputId": "2e8140c5-d992-4f2c-800f-4337c85c05bb"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XfzJOJmAICcE",
        "outputId": "6442ee5e-e5ce-4d8b-b247-ad4bac2af356"
      },
      "source": [
        "!pip install -Uqq fastbook\n",
        "import fastbook\n",
        "fastbook.setup_book()"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 727kB 6.9MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.2MB 10.5MB/s \n",
            "\u001b[K     |████████████████████████████████| 194kB 14.4MB/s \n",
            "\u001b[K     |████████████████████████████████| 51kB 5.2MB/s \n",
            "\u001b[K     |████████████████████████████████| 776.8MB 24kB/s \n",
            "\u001b[K     |████████████████████████████████| 12.8MB 27.8MB/s \n",
            "\u001b[K     |████████████████████████████████| 61kB 6.2MB/s \n",
            "\u001b[K     |████████████████████████████████| 51kB 5.5MB/s \n",
            "\u001b[31mERROR: torchtext 0.9.1 has requirement torch==1.8.1, but you'll have torch 1.7.1 which is incompatible.\u001b[0m\n",
            "\u001b[?25hMounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lezxrNGkIRJC"
      },
      "source": [
        "from fastai.vision.all import *\n",
        "from fastbook import *\n",
        "\n",
        "matplotlib.rc('image', cmap='Greys')"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125
        },
        "id": "9EvWcyIoNihg",
        "outputId": "d1e08d92-e0bf-46af-8c2d-1924aa72ce7a"
      },
      "source": [
        "#id gradient_descent\n",
        "#caption The gradient descent process\n",
        "#alt Graph showing the steps for Gradient Descent\n",
        "gv('''\n",
        "init->predict->loss->gradient->step->stop\n",
        "step->predict[label=repeat]\n",
        "''')"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<graphviz.files.Source at 0x7f8d0ac66f10>"
            ],
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n -->\n<!-- Title: G Pages: 1 -->\n<svg width=\"597pt\" height=\"78pt\"\n viewBox=\"0.00 0.00 596.69 78.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 74)\">\n<title>G</title>\n<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-74 592.6863,-74 592.6863,4 -4,4\"/>\n<!-- init -->\n<g id=\"node1\" class=\"node\">\n<title>init</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"27\" cy=\"-18\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"27\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">init</text>\n</g>\n<!-- predict -->\n<g id=\"node2\" class=\"node\">\n<title>predict</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"127.3968\" cy=\"-18\" rx=\"36.2938\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"127.3968\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">predict</text>\n</g>\n<!-- init&#45;&gt;predict -->\n<g id=\"edge1\" class=\"edge\">\n<title>init&#45;&gt;predict</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M54.1688,-18C62.3543,-18 71.5827,-18 80.6596,-18\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"80.7795,-21.5001 90.7795,-18 80.7795,-14.5001 80.7795,-21.5001\"/>\n</g>\n<!-- loss -->\n<g id=\"node3\" class=\"node\">\n<title>loss</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"227.7935\" cy=\"-52\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"227.7935\" y=\"-48.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">loss</text>\n</g>\n<!-- predict&#45;&gt;loss -->\n<g id=\"edge2\" class=\"edge\">\n<title>predict&#45;&gt;loss</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M157.5191,-28.2011C168.9806,-32.0826 182.1139,-36.5303 193.9014,-40.5222\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"192.8259,-43.8532 203.4202,-43.7458 195.0713,-37.2231 192.8259,-43.8532\"/>\n</g>\n<!-- gradient -->\n<g id=\"node4\" class=\"node\">\n<title>gradient</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"365.7399\" cy=\"-52\" rx=\"40.8928\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"365.7399\" y=\"-48.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">gradient</text>\n</g>\n<!-- loss&#45;&gt;gradient -->\n<g id=\"edge3\" class=\"edge\">\n<title>loss&#45;&gt;gradient</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M255.0473,-52C272.0415,-52 294.4481,-52 314.6545,-52\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"314.671,-55.5001 324.671,-52 314.671,-48.5001 314.671,-55.5001\"/>\n</g>\n<!-- step -->\n<g id=\"node5\" class=\"node\">\n<title>step</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"470.6863\" cy=\"-18\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"470.6863\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">step</text>\n</g>\n<!-- gradient&#45;&gt;step -->\n<g id=\"edge4\" class=\"edge\">\n<title>gradient&#45;&gt;step</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M398.9456,-41.2422C410.9558,-37.3512 424.5297,-32.9536 436.6132,-29.0388\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"437.9112,-32.2975 446.3457,-25.8857 435.7537,-25.6382 437.9112,-32.2975\"/>\n</g>\n<!-- step&#45;&gt;predict -->\n<g id=\"edge6\" class=\"edge\">\n<title>step&#45;&gt;predict</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M443.4266,-18C384.9297,-18 246.7861,-18 174.0495,-18\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"173.8098,-14.5001 163.8098,-18 173.8097,-21.5001 173.8098,-14.5001\"/>\n<text text-anchor=\"middle\" x=\"289.7935\" y=\"-21.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">repeat</text>\n</g>\n<!-- stop -->\n<g id=\"node6\" class=\"node\">\n<title>stop</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"561.6863\" cy=\"-18\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"561.6863\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">stop</text>\n</g>\n<!-- step&#45;&gt;stop -->\n<g id=\"edge5\" class=\"edge\">\n<title>step&#45;&gt;stop</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M497.9893,-18C506.2676,-18 515.508,-18 524.3268,-18\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"524.4026,-21.5001 534.4025,-18 524.4025,-14.5001 524.4026,-21.5001\"/>\n</g>\n</g>\n</svg>\n"
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J8JpEgu6Tjz_"
      },
      "source": [
        "Initialization phase"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "WEsfzrSsI04Z",
        "outputId": "1180bc29-bd05-40bc-bc95-64e401246967"
      },
      "source": [
        "#Load MNIST Data\n",
        "path = untar_data(URLs.MNIST_SAMPLE)\n",
        "Path.BASE_PATH = path\n",
        "\n",
        "#Specify Folders for Number 3 & 7\n",
        "threes = (path/'train'/'3').ls().sorted()\n",
        "sevens = (path/'train'/'7').ls().sorted()\n",
        "\n",
        "#Create Tensors for 3 & 7\n",
        "seven_tensors = [tensor(Image.open(o)) for o in sevens]\n",
        "three_tensors = [tensor(Image.open(o)) for o in threes]\n"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SZ6mPlq1MFmu"
      },
      "source": [
        "#prepare training DataSet\n",
        "#Normalize the Tensors Limit the values between 0 to1\n",
        "stacked_sevens = torch.stack(seven_tensors).float()/255\n",
        "stacked_threes = torch.stack(three_tensors).float()/255\n",
        "\n",
        "train_x = torch.cat([stacked_threes, stacked_sevens]).view(-1, 28*28)\n",
        "train_y = tensor([1]*len(threes) + [0]*len(sevens)).unsqueeze(1)\n",
        "dset = list(zip(train_x,train_y))"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BSQywBbxMgwr"
      },
      "source": [
        "#prepare Validation Dataset\n",
        "\n",
        "#normalize\n",
        "valid_3_tens = torch.stack([tensor(Image.open(o)) \n",
        "                            for o in (path/'valid'/'3').ls()])\n",
        "valid_3_tens = valid_3_tens.float()/255\n",
        "valid_7_tens = torch.stack([tensor(Image.open(o)) \n",
        "                            for o in (path/'valid'/'7').ls()])\n",
        "valid_7_tens = valid_7_tens.float()/255\n",
        "\n",
        "#Specify Dependent and Independent\n",
        "valid_x = torch.cat([valid_3_tens, valid_7_tens]).view(-1, 28*28)\n",
        "valid_y = tensor([1]*len(valid_3_tens) + [0]*len(valid_7_tens)).unsqueeze(1)\n",
        "\n",
        "#Validation dataset\n",
        "valid_dset = list(zip(valid_x,valid_y))"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6SIQord1o5Hq"
      },
      "source": [
        "PARAMETERS\n",
        "\n",
        "Based on the equation y=wx+b where w is a vector of weights, b biases Y is the  predicted result( Whether it is more likely to be a 3 or a 7) and x is the actual picture being presented to the model\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NuHCUPaOOvED"
      },
      "source": [
        "def init_params(size, std=1.0): return (torch.randn(size)*std).requires_grad_()\n",
        "\n",
        "weights = init_params((28*28,1))\n",
        "bias = init_params(1)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "78vmzkhKoyiT"
      },
      "source": [
        "The Model\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "16c--qwSO2RY"
      },
      "source": [
        "#Initialize weights and Biases tensors\n",
        "def linear1(xb): return xb@weights + bias"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SYDP61-Sox9S"
      },
      "source": [
        "Loss Function\n",
        "\n",
        "Helps us to identify how far away our predictions based on the current set of parameters are from the true values (the targets)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O2nvBwPeQBAK"
      },
      "source": [
        "def mnist_loss(predictions, targets):\n",
        "    predictions = predictions.sigmoid()\n",
        "    return torch.where(targets==1, 1-predictions, predictions).mean()"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZFxsVyw1QhZa"
      },
      "source": [
        "#put data into Dataloaders\n",
        "\n",
        "dl = DataLoader(dset, batch_size=256)\n",
        "xb,yb = first(dl)\n",
        "\n",
        "valid_dl = DataLoader(valid_dset, batch_size=256)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aINZpOMTUuCI"
      },
      "source": [
        "Calculate the gradient of the parameters so we can modify them and get closer to the right targets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QC-ID5bFk6ZC"
      },
      "source": [
        "# calculate predictions followed by loss function and compute the gradient for parameters\n",
        "def calc_grad(xb, yb, model):\n",
        "    preds = model(xb)\n",
        "    loss = mnist_loss(preds, yb)\n",
        "    loss.backward()"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6RPce7U2lAXq"
      },
      "source": [
        "# calculate predictions followed by loss function and compute the gradient for parameters\n",
        "#Apply the learning rate\n",
        "#Reset the gradients for the following epoch\n",
        "\n",
        "def train_epoch(model, lr, params):\n",
        "    for xb,yb in dl:\n",
        "        calc_grad(xb, yb, model)\n",
        "        for p in params:\n",
        "            p.data -= p.grad*lr\n",
        "            p.grad.zero_()"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yp0S9WYVl9SC"
      },
      "source": [
        "#Determines how many of the predictions are actually correct \n",
        "def batch_accuracy(xb, yb):\n",
        "    preds = xb.sigmoid()\n",
        "    correct = (preds>0.5) == yb\n",
        "    return correct.float().mean()"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BNKLyet2mfNh"
      },
      "source": [
        "#Collects the accuracy from each element in the dataloader and displays it\n",
        "def validate_epoch(model):\n",
        "    accs = [batch_accuracy(model(xb), yb) for xb,yb in valid_dl]\n",
        "    return round(torch.stack(accs).mean().item(), 4)"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G5NrUs12m11a",
        "outputId": "0562631e-cf80-4add-bf15-5bb0156b7266"
      },
      "source": [
        "#Bringing it all together\n",
        "lr = 1.\n",
        "params = weights,bias\n",
        "train_epoch(linear1, lr, params)\n",
        "validate_epoch(linear1)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7304"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lvOjTHKenGDC",
        "outputId": "250f1e2c-db52-44f9-f891-84598f7c810a"
      },
      "source": [
        "#Running 20 epochs\n",
        "for i in range(30):\n",
        "    train_epoch(linear1, lr, params)\n",
        "    print(validate_epoch(linear1), end=' ')"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.9681 0.9681 0.9676 0.9681 0.9686 0.9701 0.9706 0.9716 0.9721 0.9726 0.9731 0.9736 0.974 0.9736 0.9736 0.9736 0.9736 0.9736 0.9731 0.9731 0.9731 0.9736 0.974 0.9745 0.9745 0.9745 0.9745 0.9745 0.976 0.976 "
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0xizDECdnKQ4"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}