guided_backprop_practice/Guided_Backpropagation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Guided Backpropagation\n",
    "\n",
    "- ReLU\n",
    "- Deconvnet ReLU\n",
    "- Guided Backpropagation ReLU\n",
    "\n",
    "![alt_text](./guided_backprop.jpg)\n",
    "\n",
    "## 1) Import Required Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from torch.autograd import Function \n",
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2) Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input X:\n",
      " 3  1  5\n",
      "-2  5 -7\n",
      " 3  2 -4\n",
      "[torch.FloatTensor of size 3x3]\n",
      " \n",
      " Target Y:\n",
      " 2  2  2\n",
      " 2  2  2\n",
      " 2  2  2\n",
      "[torch.FloatTensor of size 3x3]\n",
      "\n",
      "Gradient of X: None\n",
      "\n"
     ]
    }
   ],
   "source": [
    "x = torch.Tensor([[3,1,5],[-2,5,-7],[3,2,-4]])\n",
    "\n",
    "x = Variable(x,requires_grad=True)\n",
    "y = Variable(2*torch.ones(3,3))\n",
    "\n",
    "print(\"Input X:{} \\n Target Y:{}\".format(x.data,y.data))\n",
    "print(\"Gradient of X: {}\\n\".format(x.grad))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3-1) ReLU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input X:\n",
      " 3  1  5\n",
      "-2  5 -7\n",
      " 3  2 -4\n",
      "[torch.FloatTensor of size 3x3]\n",
      " \n",
      " Target Y:\n",
      " 2  2  2\n",
      " 2  2  2\n",
      " 2  2  2\n",
      "[torch.FloatTensor of size 3x3]\n",
      "\n",
      "Output: \n",
      " 3  1  5\n",
      " 0  5  0\n",
      " 3  2  0\n",
      "[torch.FloatTensor of size 3x3]\n",
      "\n",
      "Loss: \n",
      " 1  1  3\n",
      " 2  3  2\n",
      " 1  0  2\n",
      "[torch.FloatTensor of size 3x3]\n",
      "\n",
      "Variable containing:\n",
      " 1 -1  1\n",
      " 0  1  0\n",
      " 1  0  0\n",
      "[torch.FloatTensor of size 3x3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "x = torch.Tensor([[3,1,5],[-2,5,-7],[3,2,-4]])\n",
    "\n",
    "x = Variable(x,requires_grad=True)\n",
    "y = Variable(2*torch.ones(3,3))\n",
    "\n",
    "print(\"Input X:{} \\n Target Y:{}\".format(x.data,y.data))\n",
    "\n",
    "out = F.relu(x) \n",
    "\n",
    "print(\"Output: {}\".format(out.data))\n",
    "\n",
    "loss = torch.abs(out-y)\n",
    "print(\"Loss: {}\".format(loss.data))\n",
    "loss = torch.sum(loss)\n",
    "loss.backward()\n",
    "\n",
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3-2) MyReLU\n",
    "\n",
    "http://pytorch.org/tutorials/beginner/examples_autograd/two_layer_net_custom_function.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class MyReLU(torch.autograd.Function):\n",
    "    \"\"\"\n",
    "    We can implement our own custom autograd Functions by subclassing\n",
    "    torch.autograd.Function and implementing the forward and backward passes\n",
    "    which operate on Tensors.\n",
    "    \"\"\"\n",
    "\n",
    "    @staticmethod\n",
    "    def forward(ctx, input):\n",
    "        \"\"\"\n",
    "        In the forward pass we receive a Tensor containing the input and return\n",
    "        a Tensor containing the output. ctx is a context object that can be used\n",
    "        to stash information for backward computation. You can cache arbitrary\n",
    "        objects for use in the backward pass using the ctx.save_for_backward method.\n",
    "        \"\"\"\n",
    "        ctx.save_for_backward(input)\n",
    "        return input.clamp(min=0)\n",
    "\n",
    "    @staticmethod\n",
    "    def backward(ctx, grad_output):\n",
    "        \"\"\"\n",
    "        In the backward pass we receive a Tensor containing the gradient of the loss\n",
    "        with respect to the output, and we need to compute the gradient of the loss\n",
    "        with respect to the input.\n",
    "        \"\"\"\n",
    "        input, = ctx.saved_tensors\n",
    "        grad_input = grad_output.clone()\n",
    "        print(\"grad_input: \",grad_input)\n",
    "        grad_input[input < 0] = 0\n",
    "        return grad_input\n",
    "    \n",
    "my_relu = MyReLU.apply"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input X:\n",
      " 3  1  5\n",
      "-2  5 -7\n",
      " 3  2 -4\n",
      "[torch.FloatTensor of size 3x3]\n",
      "\n",
      "grad_input:  Variable containing:\n",
      " 1 -1  1\n",
      "-1  1 -1\n",
      " 1  0 -1\n",
      "[torch.FloatTensor of size 3x3]\n",
      "\n",
      "Variable containing:\n",
      " 1 -1  1\n",
      " 0  1  0\n",
      " 1  0  0\n",
      "[torch.FloatTensor of size 3x3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "x = torch.Tensor([[3,1,5],[-2,5,-7],[3,2,-4]])\n",
    "\n",
    "x = Variable(x,requires_grad=True)\n",
    "y = Variable(2*torch.ones(3,3))\n",
    "\n",
    "print(\"Input X:{}\".format(x.data))\n",
    "\n",
    "out = my_relu(x) \n",
    "\n",
    "loss = torch.abs(out-y)\n",
    "#print(\"Loss: {}\".format(loss.data))\n",
    "loss = torch.sum(loss)\n",
    "loss.backward()\n",
    "\n",
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3-3) Deconvnet ReLU\n",
    "\n",
    "http://pytorch.org/docs/0.3.1/notes/extending.html#extending-torch-autograd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "class DeconvReLU(Function):\n",
    "    @staticmethod\n",
    "    def forward(ctx,input):\n",
    "        #ctx.save_for_backward(input)\n",
    "        return input.clamp(min=0)\n",
    "    \n",
    "    @staticmethod\n",
    "    def backward(ctx,grad_output):\n",
    "        #input = ctx.saved_tensors\n",
    "        grad_input = grad_output.clone()\n",
    "        print(\"grad input: \",grad_input)\n",
    "        grad_input[grad_input<0] = 0\n",
    "        return grad_input\n",
    "        \n",
    "deconv_relu = DeconvReLU.apply"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input X:\n",
      " 3  1  5\n",
      "-2  5 -7\n",
      " 3  2 -4\n",
      "[torch.FloatTensor of size 3x3]\n",
      "\n",
      "grad input:  Variable containing:\n",
      " 1 -1  1\n",
      "-1  1 -1\n",
      " 1  0 -1\n",
      "[torch.FloatTensor of size 3x3]\n",
      "\n",
      "Variable containing:\n",
      " 1  0  1\n",
      " 0  1  0\n",
      " 1  0  0\n",
      "[torch.FloatTensor of size 3x3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "x = torch.Tensor([[3,1,5],[-2,5,-7],[3,2,-4]])\n",
    "\n",
    "x = Variable(x,requires_grad=True)\n",
    "y = Variable(2*torch.ones(3,3))\n",
    "\n",
    "print(\"Input X:{}\".format(x.data))\n",
    "\n",
    "out = deconv_relu(x)\n",
    "\n",
    "#print(\"Output: {}\".format(out.data))\n",
    "\n",
    "loss = torch.abs(out-y)\n",
    "#print(\"Loss: {}\".format(loss.data))\n",
    "loss = torch.sum(loss)\n",
    "loss.backward()\n",
    "\n",
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3-4) Guided Backpropagation ReLU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class GuidedBackpropRelu(Function):\n",
    "    @staticmethod\n",
    "    def forward(ctx,input):\n",
    "        ctx.save_for_backward(input)\n",
    "        return input.clamp(min=0)\n",
    "    \n",
    "    @staticmethod\n",
    "    def backward(ctx,grad_output):\n",
    "        input = ctx.saved_tensors[0]\n",
    "        grad_input = grad_output.clone()\n",
    "        print(\"grad input: \",grad_input)\n",
    "        grad_input[grad_input<0] = 0\n",
    "        grad_input[input<0]=0\n",
    "        return grad_input\n",
    "    \n",
    "guided_backprop_relu = GuidedBackpropRelu.apply"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input X:\n",
      " 3  1  5\n",
      "-2  5 -7\n",
      " 3  2 -4\n",
      "[torch.FloatTensor of size 3x3]\n",
      "\n",
      "grad input:  Variable containing:\n",
      " 1 -1  1\n",
      "-1  1 -1\n",
      " 1  0 -1\n",
      "[torch.FloatTensor of size 3x3]\n",
      "\n",
      "Variable containing:\n",
      " 1  0  1\n",
      " 0  1  0\n",
      " 1  0  0\n",
      "[torch.FloatTensor of size 3x3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "x = torch.Tensor([[3,1,5],[-2,5,-7],[3,2,-4]])\n",
    "\n",
    "x = Variable(x,requires_grad=True)\n",
    "y = Variable(2*torch.ones(3,3))\n",
    "\n",
    "print(\"Input X:{}\".format(x.data))\n",
    "\n",
    "out = guided_backprop_relu(x)\n",
    "\n",
    "#print(\"Output: {}\".format(out.data))\n",
    "\n",
    "loss = torch.abs(out-y)\n",
    "#print(\"Loss: {}\".format(loss.data))\n",
    "loss = torch.sum(loss)\n",
    "loss.backward()\n",
    "\n",
    "print(x.grad)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}