08.4 - Backpropagation with MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Lab 8-4: Backpropagation with MNIST\r\n",
    "\r\n",
    "Edited By Steve Ive\r\n",
    "\r\n",
    "Here, we are going to learn how to backpropagation works at the deep inside. Before we used ```cost.backward()```, but here we are going to implement backward at the low level.\r\n",
    "\r\n",
    "Reference from\r\n",
    "\r\n",
    "https://github.com/deeplearningzerotoall/PyTorch/blob/master/lab-08_4_mnist_back_prop.ipynb"
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Imports"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "source": [
    "import torch\r\n",
    "import torchvision.datasets as datasets\r\n",
    "import torchvision.transforms as transforms"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\r\n",
    "\r\n",
    "torch.manual_seed(1)\r\n",
    "if device == 'cuda':\r\n",
    "    torch.cuda.manual_seed_all(1)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Set Hyperparameters"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "source": [
    "learning_rate = 0.5\r\n",
    "batch_size = 10"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## MNIST datasets"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "source": [
    "mnist_train = datasets.MNIST(root='MNIST_data/',\r\n",
    "                             download=True,\r\n",
    "                             train=True,\r\n",
    "                             transform=transforms.ToTensor())\r\n",
    "mnist_test = datasets.MNIST(root='MNIST_data/',\r\n",
    "                            train = False,\r\n",
    "                            transform=transforms.ToTensor(),\r\n",
    "                            download=True)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "source": [
    "data_loader = torch.utils.data.DataLoader(dataset=mnist_train,\r\n",
    "                                          batch_size = batch_size,\r\n",
    "                                          shuffle = True,\r\n",
    "                                          drop_last = True)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Set Parameters"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "source": [
    "w1 = torch.nn.Parameter(torch.Tensor(784, 30)).to(device)\r\n",
    "b1 = torch.nn.Parameter(torch.Tensor(30)).to(device)\r\n",
    "w2 = torch.nn.Parameter(torch.Tensor(30, 10)).to(device)\r\n",
    "b2 = torch.nn.Parameter(torch.Tensor(10)).to(device)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "source": [
    "torch.nn.init.normal_(w1)\r\n",
    "torch.nn.init.normal_(b1)\r\n",
    "torch.nn.init.normal_(w2)\r\n",
    "torch.nn.init.normal_(b2)"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "tensor([ 0.1696, -1.2966,  0.3153,  0.9196, -0.1853, -1.0896, -0.2633,  0.3830,\n",
       "        -0.6385,  1.4271], device='cuda:0', grad_fn=<CopyBackwards>)"
      ]
     },
     "metadata": {},
     "execution_count": 18
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Set Activation Functions and Its derivative terms"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "source": [
    "def sigmoid(x):\r\n",
    "    return 1.0 / (1.0 + torch.exp(-x))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "source": [
    "def sigmoid_prime(x):\r\n",
    "    #derivative of the sigmoid function\r\n",
    "    return sigmoid(x) * (1 - sigmoid(x))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "source": [
    "X_test = mnist_test.test_data.view(-1, 28 * 28).float().to(device)[:1000]\r\n",
    "Y_test = mnist_test.test_labels.to(device)[:1000]\r\n",
    "\r\n",
    "i = 0\r\n",
    "\r\n",
    "while not i == 10000:\r\n",
    "    for X, Y in data_loader:\r\n",
    "        i += 1\r\n",
    "\r\n",
    "        #forward\r\n",
    "        X = X.view(-1 ,28* 28).to(device)\r\n",
    "        Y = torch.zeros((batch_size , 10)).scatter_(1, Y.unsqueeze(1), 1).to(device)\r\n",
    "\r\n",
    "        #one-hot\r\n",
    "        layer1 = torch.add(torch.matmul(X, w1), b1)\r\n",
    "        activation1 = sigmoid(layer1)\r\n",
    "\r\n",
    "        layer2 = torch.add(torch.matmul(activation1, w2), b2)\r\n",
    "        y_pred = sigmoid(layer2)\r\n",
    "\r\n",
    "        diff = y_pred - Y\r\n",
    "\r\n",
    "        #backward (back prop: Chain Rule)\r\n",
    "        diff_layer2 = diff * sigmoid_prime(layer2)\r\n",
    "        diff_b2 = diff_layer2\r\n",
    "        diff_w2 = torch.matmul(torch.transpose(activation1, 0, 1), diff_layer2)\r\n",
    "\r\n",
    "        diff_activation1 = torch.matmul(diff_layer2, torch.transpose(w2, 0, 1))\r\n",
    "        diff_layer1 = diff_activation1 * sigmoid_prime(layer1)\r\n",
    "        diff_b1 = diff_layer1\r\n",
    "        diff_w1 = torch.matmul(torch.transpose(X, 0, 1), diff_layer1)\r\n",
    "\r\n",
    "        w1 = w1 - learning_rate * diff_w1\r\n",
    "        b1 = b1 - learning_rate * torch.mean(diff_b1, 0)\r\n",
    "        w2 = w2 - learning_rate * diff_w2\r\n",
    "        b2 = b2 - learning_rate * torch.mean(diff_b2, 0)\r\n",
    "\r\n",
    "        if i % 1000 == 0:\r\n",
    "            layer1 = torch.add(torch.matmul(X_test, w1), b1)\r\n",
    "            activation1 = sigmoid(layer1)\r\n",
    "            layer2 = torch.add(torch.matmul(activation1, w2), b2)\r\n",
    "            y_pred = sigmoid(layer2)\r\n",
    "            accuracy_mat = torch.argmax(y_pred, 1) == Y_test\r\n",
    "            accuracy_res = accuracy_mat.sum()\r\n",
    "            print(accuracy_res.item())\r\n",
    "\r\n",
    "        if i == 10000:\r\n",
    "            break\r\n"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "752\n",
      "858\n",
      "882\n",
      "868\n",
      "891\n",
      "889\n",
      "905\n",
      "896\n",
      "897\n"
     ]
    }
   ],
   "metadata": {}
  }
 ],
 "metadata": {
  "orig_nbformat": 4,
  "language_info": {
   "name": "python",
   "version": "3.8.8",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.8 64-bit ('buddhalight': conda)"
  },
  "interpreter": {
   "hash": "38ed4d61829b01de31b0fe0651719916120d9f7e023a62cbbfea93b7d24a50a0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}