Attention.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "require 'nn';\n",
    "require 'nngraph';"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vh\n",
    "\n",
    "* nInputPlane = 1\n",
    "* nOutputPlane = scoreDepth\n",
    "* height = L\n",
    "* width = annotationDepth\n",
    "* input = nInputPlane x height x width\n",
    "* module = nn.SpatialConvolution(nInputPlane, nOutputPlane, kW, kH, [dW], [dH], [padW], [padH])\n",
    "* kW = annotationDepth\n",
    "* kH = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "  4\n",
       " 10\n",
       "  1\n",
       "[torch.LongStorage of size 3]\n",
       "\n",
       " 6  6  6  6  6  6  6  6  6  6\n",
       " 6  6  6  6  6  6  6  6  6  6\n",
       " 6  6  6  6  6  6  6  6  6  6\n",
       " 6  6  6  6  6  6  6  6  6  6\n",
       "[torch.DoubleTensor of size 4x10]\n",
       "\n"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "annotationDepth = 5\n",
    "scoreDepth = 4\n",
    "L = 10\n",
    "\n",
    "m = nn.SpatialConvolutionMM(1,scoreDepth,annotationDepth,1)\n",
    "p,dp = m:getParameters()\n",
    "p:fill(1)\n",
    "\n",
    "h = torch.ones(1,L,annotationDepth)\n",
    "Vh = m:forward(h)\n",
    "print(Vh:size())\n",
    "print(Vh:resize(scoreDepth,L))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* inputFrameSize = annotationDepth\n",
    "* nOutputPlane = scoreDepth\n",
    "* input = L x inputFrameSize\n",
    "* module = nn.TemporalConvolution(inputFrameSize, outputFrameSize, kW, [dW])\n",
    "* kW = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 10\n",
       "  4\n",
       "[torch.LongStorage of size 2]\n",
       "\n",
       "\n",
       "  8\n",
       " 10\n",
       "  4\n",
       "[torch.LongStorage of size 3]\n",
       "\n"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "annotationDepth = 5\n",
    "scoreDepth = 4\n",
    "L = 10\n",
    "\n",
    "m = nn.TemporalConvolution(annotationDepth,scoreDepth,1)\n",
    "w = m:parameters()[1]\n",
    "b = m:parameters()[2]\n",
    "i=0\n",
    "w:apply(function() i=i+1 return i end)\n",
    "b:fill(0)\n",
    "\n",
    "h = torch.ones(L,annotationDepth)\n",
    "Vh = m:forward(h)\n",
    "print(Vh:size())\n",
    "\n",
    "h = torch.ones(8,L,annotationDepth)\n",
    "Vh = m:forward(h)\n",
    "print(Vh:size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 15  40  65  90\n",
       " 15  40  65  90\n",
       " 15  40  65  90\n",
       " 15  40  65  90\n",
       " 15  40  65  90\n",
       " 15  40  65  90\n",
       " 15  40  65  90\n",
       " 15  40  65  90\n",
       " 15  40  65  90\n",
       " 15  40  65  90\n",
       "[torch.DoubleTensor of size 10x4]\n",
       "\n"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Vh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "  1   2   3   4   5\n",
       "  6   7   8   9  10\n",
       " 11  12  13  14  15\n",
       " 16  17  18  19  20\n",
       "[torch.DoubleTensor of size 4x5]\n",
       "\n"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# UF\n",
    "\n",
    "* module = nn.Padding(dim, pad [, nInputDim, value])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 12\n",
       "  1\n",
       "[torch.LongStorage of size 2]\n",
       "\n",
       "\n",
       "  5\n",
       " 12\n",
       "  1\n",
       "[torch.LongStorage of size 3]\n",
       "\n",
       " 0\n",
       " 0\n",
       " 1\n",
       " 1\n",
       " 1\n",
       " 1\n",
       " 1\n",
       " 1\n",
       " 1\n",
       " 1\n",
       " 1\n",
       " 1\n",
       " 0\n",
       " 0\n",
       "[torch.DoubleTensor of size 14x1]\n",
       "\n"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hybridAttendFilterSize = 3\n",
    "hybridAttendFeatureMaps = 5\n",
    "L = 10\n",
    "\n",
    "--print(nn.SpatialZeroPadding(0,0,1,1):forward(nn.Reshape(1,L,1):forward(torch.ones(1,10,1))):size())\n",
    "\n",
    "print(nn.Padding(1,-2,2):forward(torch.ones(10,1)):size())\n",
    "print(nn.Padding(1,2,2):forward(torch.ones(5,10,1)):size())\n",
    "print(nn.Padding(1,2,2):forward(nn.Padding(1,-2,2):forward(torch.ones(10,1))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* inputFrameSize = 1\n",
    "* nOutputPlane = hybridAttendFeatureMaps\n",
    "* input = L x inputFrameSize\n",
    "* module = nn.TemporalConvolution(inputFrameSize, outputFrameSize, kW, [dW])\n",
    "* kW = hybridAttendFilterSize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### odd filter size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 10\n",
       "  5\n",
       "[torch.LongStorage of size 2]\n",
       "\n",
       "\n",
       " 11\n",
       "  5\n",
       "[torch.LongStorage of size 2]\n",
       "\n"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hybridAttendFilterSize = 3\n",
    "hybridAttendFeatureMaps = 5\n",
    "L = 10\n",
    "\n",
    "pad_left = math.floor((hybridAttendFilterSize-1)/2)\n",
    "pad_right = math.floor((hybridAttendFilterSize-1)/2)\n",
    "\n",
    "nn.Padding(1,2,2):forward(nn.Padding(1,-2,2):forward(torch.ones(10,1)))\n",
    "\n",
    "m = nn.Sequential()\n",
    "m:add(nn.Padding(1,-pad_left,2))\n",
    "m:add(nn.Padding(1,pad_right,2))\n",
    "m:add(nn.TemporalConvolution(1,hybridAttendFeatureMaps,hybridAttendFilterSize))\n",
    "\n",
    "x = torch.ones(10,1)\n",
    "print(m:forward(x):size())\n",
    "\n",
    "x = torch.ones(11,1)\n",
    "print(m:forward(x):size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## even filter size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 10\n",
       "  5\n",
       "[torch.LongStorage of size 2]\n",
       "\n",
       "\n",
       " 11\n",
       "  5\n",
       "[torch.LongStorage of size 2]\n",
       "\n"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hybridAttendFilterSize = 4\n",
    "hybridAttendFeatureMaps = 5\n",
    "scoreDepth = 4\n",
    "L = 10\n",
    "\n",
    "pad_left = math.floor((hybridAttendFilterSize)/2)\n",
    "pad_right = math.floor((hybridAttendFilterSize)/2)-1\n",
    "\n",
    "nn.Padding(1,2,2):forward(nn.Padding(1,-2,2):forward(torch.ones(10,1)))\n",
    "\n",
    "m = nn.Sequential()\n",
    "m:add(nn.Padding(1,-pad_left,2))\n",
    "m:add(nn.Padding(1,pad_right,2))\n",
    "m:add(nn.TemporalConvolution(1,hybridAttendFeatureMaps,hybridAttendFilterSize))\n",
    "\n",
    "x = torch.ones(10,1)\n",
    "print(m:forward(x):size())\n",
    "\n",
    "x = torch.ones(11,1)\n",
    "print(m:forward(x):size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## We have calculated F, now do UF ... actually (UF)^T\n",
    "\n",
    "* module = nn.TemporalConvolution(inputFrameSize, outputFrameSize, kW, [dW])\n",
    "* input = L x inputFrameSize\n",
    "#### F\n",
    "* inputFrameSize = 1\n",
    "* nOutputPlane = hybridAttendFeatureMaps\n",
    "* kW = hybridAttendFilterSize\n",
    "#### UF\n",
    "* inputFrameSize = hybridAttendFeatureMaps\n",
    "* nOutputPlane = scoreDepth\n",
    "* kW = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 10\n",
       "  6\n",
       "[torch.LongStorage of size 2]\n",
       "\n"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hybridAttendFilterSize = 3\n",
    "hybridAttendFeatureMaps = 5\n",
    "scoreDepth = 4\n",
    "L = 10\n",
    "\n",
    "pad_left = math.floor((hybridAttendFilterSize-1)/2)\n",
    "pad_right = math.floor((hybridAttendFilterSize-1)/2)\n",
    "\n",
    "nn.Padding(1,2,2):forward(nn.Padding(1,-2,2):forward(torch.ones(10,1)))\n",
    "\n",
    "m = nn.Sequential()\n",
    "m:add(nn.Padding(1,-pad_left,2))\n",
    "m:add(nn.Padding(1,pad_right,2))\n",
    "m:add(nn.TemporalConvolution(1,hybridAttendFeatureMaps,hybridAttendFilterSize))\n",
    "m:add(nn.TemporalConvolution(hybridAttendFeatureMaps,scoreDepth,1))\n",
    "\n",
    "x = torch.ones(10,1)\n",
    "print(m:forward(x):size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ws\n",
    "\n",
    "\n",
    "* module = nn.TemporalConvolution(inputFrameSize, outputFrameSize, kW, [dW])\n",
    "* input = L x inputFrameSize\n",
    "* L = stateDepth\n",
    "* inputFrameSize = 1\n",
    "* nOutputPlane = scoreDepth\n",
    "* kW = stateDepth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 1\n",
       " 4\n",
       "[torch.LongStorage of size 2]\n",
       "\n",
       "\n",
       " 8\n",
       " 1\n",
       " 4\n",
       "[torch.LongStorage of size 3]\n",
       "\n"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stateDepth = 5\n",
    "scoreDepth = 4\n",
    "\n",
    "m = nn.Sequential()\n",
    "m:add(nn.TemporalConvolution(1,scoreDepth,stateDepth))\n",
    "\n",
    "x = torch.ones(stateDepth,1)\n",
    "print(m:forward(x):size())\n",
    "\n",
    "x = torch.ones(8,stateDepth,1)\n",
    "print(m:forward(x):size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### replicate L times\n",
    "\n",
    "module = nn.Replicate(nFeature [, dim, ndim])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 0.2889 -0.1294  1.4467  0.0965\n",
       "[torch.DoubleTensor of size 1x4]\n",
       "\n",
       " 0.2889 -0.1294  1.4467  0.0965\n",
       " 0.2889 -0.1294  1.4467  0.0965\n",
       " 0.2889 -0.1294  1.4467  0.0965\n",
       " 0.2889 -0.1294  1.4467  0.0965\n",
       " 0.2889 -0.1294  1.4467  0.0965\n",
       " 0.2889 -0.1294  1.4467  0.0965\n",
       "[torch.DoubleTensor of size 6x4]\n",
       "\n"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.randn(1,4)\n",
    "print(x)\n",
    "print(nn.Reshape(6,4):forward(nn.Replicate(6,1,2):forward(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 10\n",
       "  4\n",
       "[torch.LongStorage of size 2]\n",
       "\n",
       "\n",
       "  8\n",
       " 10\n",
       "  4\n",
       "[torch.LongStorage of size 3]\n",
       "\n"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stateDepth = 5\n",
    "scoreDepth = 4\n",
    "L = 10\n",
    "\n",
    "m = nn.Sequential()\n",
    "m:add(nn.TemporalConvolution(1,scoreDepth,stateDepth))\n",
    "m:add(nn.Replicate(L,1,2))\n",
    "m:add(nn.Reshape(L,scoreDepth))\n",
    "\n",
    "x = torch.ones(stateDepth,1)\n",
    "print(m:forward(x):size())\n",
    "\n",
    "x = torch.ones(8,stateDepth,1)\n",
    "print(m:forward(x):size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# add bias to input to tanh(Z+b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1,.,.) = \n",
       "  0.3949 -0.9719 -0.2226 -0.0188\n",
       "  0.3949 -0.9719 -0.2226 -0.0188\n",
       "  0.3949 -0.9719 -0.2226 -0.0188\n",
       "  0.3949 -0.9719 -0.2226 -0.0188\n",
       "  0.3949 -0.9719 -0.2226 -0.0188\n",
       "  0.3949 -0.9719 -0.2226 -0.0188\n",
       "  0.3949 -0.9719 -0.2226 -0.0188\n",
       "  0.3949 -0.9719 -0.2226 -0.0188\n",
       "  0.3949 -0.9719 -0.2226 -0.0188\n",
       "  0.3949 -0.9719 -0.2226 -0.0188\n",
       "\n",
       "(2,.,.) = \n",
       "  0.3949 -0.9719 -0.2226 -0.0188\n",
       "  0.3949 -0.9719 -0.2226 -0.0188\n",
       "  0.3949 -0.9719 -0.2226 -0.0188\n",
       "  0.3949 -0.9719 -0.2226 -0.0188\n",
       "  0.3949 -0.9719 -0.2226 -0.0188\n",
       "  0.3949 -0.9719 -0.2226 -0.0188\n",
       "  0.3949 -0.9719 -0.2226 -0.0188\n",
       "  0.3949 -0.9719 -0.2226 -0.0188\n",
       "  0.3949 -0.9719 -0.2226 -0.0188\n",
       "  0.3949 -0.9719 -0.2226 -0.0188\n",
       "[torch.DoubleTensor of size 2x10x4]\n",
       "\n"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "L = 10\n",
    "scoreDepth = 4\n",
    "batchSize = 2\n",
    "\n",
    "Z = torch.ones(batchSize,L,scoreDepth)\n",
    "\n",
    "bias = torch.randn(scoreDepth)\n",
    "ones = torch.ones(Z:size(2))\n",
    "bias_grid = torch.Tensor():resizeAs(Z[1]):fill(0):addr(1, ones, bias)\n",
    "--bias_grid:expandAs(Z)\n",
    "\n",
    "print(bias_grid:resize(1,Z:size(2),Z:size(3)):expandAs(Z))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "require 'nn'\n",
    "require 'AddBias'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 0.5298  0.5552  1.0446  0.5357\n",
       " 0.5298  0.5552  1.0446  0.5357\n",
       " 0.5298  0.5552  1.0446  0.5357\n",
       " 0.5298  0.5552  1.0446  0.5357\n",
       " 0.5298  0.5552  1.0446  0.5357\n",
       " 0.5298  0.5552  1.0446  0.5357\n",
       " 0.5298  0.5552  1.0446  0.5357\n",
       " 0.5298  0.5552  1.0446  0.5357\n",
       " 0.5298  0.5552  1.0446  0.5357\n",
       " 0.5298  0.5552  1.0446  0.5357\n",
       "[torch.DoubleTensor of size 10x4]\n",
       "\n",
       "(1,.,.) = \n",
       "  0.5298  0.5552  1.0446  0.5357\n",
       "  0.5298  0.5552  1.0446  0.5357\n",
       "  0.5298  0.5552  1.0446  0.5357\n",
       "  0.5298  0.5552  1.0446  0.5357\n",
       "  0.5298  0.5552  1.0446  0.5357\n",
       "  0.5298  0.5552  1.0446  0.5357\n",
       "  0.5298  0.5552  1.0446  0.5357\n",
       "  0.5298  0.5552  1.0446  0.5357\n",
       "  0.5298  0.5552  1.0446  0.5357\n",
       "  0.5298  0.5552  1.0446  0.5357\n",
       "\n",
       "(2,.,.) = \n",
       "  0.5298  0.5552  1.0446  0.5357\n",
       "  0.5298  0.5552  1.0446  0.5357\n",
       "  0.5298  0.5552  1.0446  0.5357\n",
       "  0.5298  0.5552  1.0446  0.5357\n",
       "  0.5298  0.5552  1.0446  0.5357\n",
       "  0.5298  0.5552  1.0446  0.5357\n",
       "  0.5298  0.5552  1.0446  0.5357\n",
       "  0.5298  0.5552  1.0446  0.5357\n",
       "  0.5298  0.5552  1.0446  0.5357\n",
       "  0.5298  0.5552  1.0446  0.5357\n",
       "[torch.DoubleTensor of size 2x10x4]\n",
       "\n"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "L = 10\n",
    "scoreDepth = 4\n",
    "batchSize = 2\n",
    "\n",
    "m = nn.AddBias(scoreDepth,false)\n",
    "\n",
    "Z = torch.ones(L,scoreDepth)\n",
    "print(m:forward(Z))\n",
    "\n",
    "Z = torch.ones(batchSize,L,scoreDepth)\n",
    "print(m:forward(Z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 9\n",
       " 9\n",
       " 9\n",
       " 9\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       " 27\n",
       " 27\n",
       " 27\n",
       " 27\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "L = 9\n",
    "scoreDepth = 4\n",
    "batchSize = 3\n",
    "\n",
    "m = nn.AddBias(scoreDepth,false)\n",
    "p,dp = m:getParameters()\n",
    "\n",
    "m:zeroGradParameters()\n",
    "Z = torch.ones(L,scoreDepth)\n",
    "m:forward(Z)\n",
    "m:backward(Z,Z)\n",
    "print(dp)\n",
    "\n",
    "m:zeroGradParameters()\n",
    "Z = torch.ones(batchSize,L,scoreDepth)\n",
    "m:forward(Z)\n",
    "m:backward(Z,Z)\n",
    "print(dp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# e_t\n",
    "\n",
    "* module = nn.TemporalConvolution(inputFrameSize, outputFrameSize, kW, [dW])\n",
    "* input = L x inputFrameSize\n",
    "* inputFrameSize = scoreDepth\n",
    "* nOutputPlane = 1\n",
    "* kW = scoreDepth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 10\n",
       "  1\n",
       "[torch.LongStorage of size 2]\n",
       "\n",
       "\n",
       "  8\n",
       " 10\n",
       "  1\n",
       "[torch.LongStorage of size 3]\n",
       "\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scoreDepth = 4\n",
    "L = 10\n",
    "batchSize = 8\n",
    "\n",
    "m = nn.Sequential()\n",
    "m:add(nn.TemporalConvolution(scoreDepth,1,1))\n",
    "\n",
    "x = torch.ones(L,scoreDepth)\n",
    "print(m:forward(x):size())\n",
    "\n",
    "x = torch.ones(batchSize,L,scoreDepth)\n",
    "print(m:forward(x):size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 0.1876\n",
       " 0.0429\n",
       " 0.0349\n",
       " 0.0166\n",
       " 0.1256\n",
       " 0.0318\n",
       " 0.0867\n",
       " 0.1422\n",
       " 0.1153\n",
       " 0.2165\n",
       "[torch.DoubleTensor of size 10]\n",
       "\n",
       " 0.1207  0.0105  0.3167  0.1983  0.0944  0.0229  0.0645  0.0531  0.0356  0.0833\n",
       " 0.1076  0.1684  0.0765  0.3354  0.0376  0.0366  0.0160  0.0203  0.0839  0.1178\n",
       " 0.1586  0.0483  0.0071  0.0304  0.0805  0.0524  0.0232  0.2953  0.0432  0.2609\n",
       " 0.0759  0.0856  0.0637  0.0380  0.2950  0.0699  0.0438  0.1340  0.0080  0.1861\n",
       " 0.1196  0.1007  0.0803  0.1347  0.1056  0.1594  0.0205  0.1102  0.1072  0.0618\n",
       " 0.0295  0.0583  0.0845  0.0745  0.0453  0.2239  0.2827  0.0532  0.0698  0.0781\n",
       " 0.1176  0.0311  0.1571  0.0354  0.0123  0.0875  0.3124  0.0516  0.1290  0.0660\n",
       " 0.0550  0.0396  0.0650  0.0101  0.5254  0.0365  0.0564  0.0132  0.1088  0.0899\n",
       "[torch.DoubleTensor of size 8x10]\n",
       "\n"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "L = 10\n",
    "batchSize = 8\n",
    "\n",
    "m = nn.Sequential()\n",
    "m:add(nn.Reshape(L))\n",
    "m:add(nn.SoftMax())\n",
    "\n",
    "e = torch.randn(L,1)\n",
    "print(m:forward(e))\n",
    "\n",
    "e = torch.randn(batchSize,L,1)\n",
    "print(m:forward(e))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 10  10  10  10  10\n",
       "[torch.DoubleTensor of size 1x5]\n",
       "\n",
       "(1,.,.) = \n",
       "  10  10  10  10  10\n",
       "\n",
       "(2,.,.) = \n",
       "  10  10  10  10  10\n",
       "\n",
       "(3,.,.) = \n",
       "  10  10  10  10  10\n",
       "\n",
       "(4,.,.) = \n",
       "  10  10  10  10  10\n",
       "\n",
       "(5,.,.) = \n",
       "  10  10  10  10  10\n",
       "\n",
       "(6,.,.) = \n",
       "  10  10  10  10  10\n",
       "\n",
       "(7,.,.) = \n",
       "  10  10  10  10  10\n",
       "\n",
       "(8,.,.) = \n",
       "  10  10  10  10  10\n",
       "[torch.DoubleTensor of size 8x1x5]\n",
       "\n"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "L = 10\n",
    "annotationDepth = 5\n",
    "batchSize = 8\n",
    "\n",
    "m = nn.MM()\n",
    "\n",
    "h = torch.ones(L,annotationDepth)\n",
    "alpha = torch.ones(1,L)\n",
    "print(m:forward({alpha,h}))\n",
    "\n",
    "h = torch.ones(batchSize,L,annotationDepth)\n",
    "alpha = torch.ones(batchSize,1,L)\n",
    "print(m:forward({alpha,h}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RNN with nonrecurrent inputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### First, test to make sure LSTM still works with it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "require 'nn';\n",
    "require 'RNN2';\n",
    "require 'RNN';\n",
    "require 'LSTM';"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "./RNN2.lua:10: recurrent must specify dimoutput\nstack traceback:\n\t[C]: in function 'assert'\n\t./RNN2.lua:10: in function '__init'\n\t...ustinmaojones/torch/install/share/lua/5.1/torch/init.lua:54: in function <...ustinmaojones/torch/install/share/lua/5.1/torch/init.lua:50>\n\t[C]: in function 'RNN2'\n\t[string \"diminput = 4...\"]:10: in main chunk\n\t[C]: in function 'xpcall'\n\t...stinmaojones/torch/install/share/lua/5.1/itorch/main.lua:179: in function <...stinmaojones/torch/install/share/lua/5.1/itorch/main.lua:143>\n\t...stinmaojones/torch/install/share/lua/5.1/lzmq/poller.lua:75: in function 'poll'\n\t...nmaojones/torch/install/share/lua/5.1/lzmq/impl/loop.lua:307: in function 'poll'\n\t...nmaojones/torch/install/share/lua/5.1/lzmq/impl/loop.lua:325: in function 'sleep_ex'\n\t...nmaojones/torch/install/share/lua/5.1/lzmq/impl/loop.lua:370: in function 'start'\n\t...stinmaojones/torch/install/share/lua/5.1/itorch/main.lua:350: in main chunk\n\t[C]: in function 'require'\n\t[string \"arg={'/Users/justinmaojones/Library/Jupyter/r...\"]:1: in main chunk",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "./RNN2.lua:10: recurrent must specify dimoutput\nstack traceback:\n\t[C]: in function 'assert'\n\t./RNN2.lua:10: in function '__init'\n\t...ustinmaojones/torch/install/share/lua/5.1/torch/init.lua:54: in function <...ustinmaojones/torch/install/share/lua/5.1/torch/init.lua:50>\n\t[C]: in function 'RNN2'\n\t[string \"diminput = 4...\"]:10: in main chunk\n\t[C]: in function 'xpcall'\n\t...stinmaojones/torch/install/share/lua/5.1/itorch/main.lua:179: in function <...stinmaojones/torch/install/share/lua/5.1/itorch/main.lua:143>\n\t...stinmaojones/torch/install/share/lua/5.1/lzmq/poller.lua:75: in function 'poll'\n\t...nmaojones/torch/install/share/lua/5.1/lzmq/impl/loop.lua:307: in function 'poll'\n\t...nmaojones/torch/install/share/lua/5.1/lzmq/impl/loop.lua:325: in function 'sleep_ex'\n\t...nmaojones/torch/install/share/lua/5.1/lzmq/impl/loop.lua:370: in function 'start'\n\t...stinmaojones/torch/install/share/lua/5.1/itorch/main.lua:350: in main chunk\n\t[C]: in function 'require'\n\t[string \"arg={'/Users/justinmaojones/Library/Jupyter/r...\"]:1: in main chunk"
     ]
    }
   ],
   "source": [
    "diminput = 4\n",
    "dimoutput = 3\n",
    "T = 10\n",
    "batchSize = 4\n",
    "peepholes = false\n",
    "lstm1 = nn.LSTM(diminput,dimoutput,peepholes)\n",
    "rnn1 = nn.RNN(lstm1,T)\n",
    "\n",
    "lstm2 = nn.LSTM(diminput,dimoutput,peepholes)\n",
    "rnn2 = nn.RNN2(lstm2,T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "[string \"local f = function() return rnn2.gradRecurren...\"]:1: attempt to index global 'rnn2' (a nil value)\nstack traceback:\n\t[string \"local f = function() return rnn2.gradRecurren...\"]:1: in function 'f'\n\t[string \"local f = function() return rnn2.gradRecurren...\"]:1: in main chunk\n\t[C]: in function 'xpcall'\n\t...stinmaojones/torch/install/share/lua/5.1/itorch/main.lua:179: in function <...stinmaojones/torch/install/share/lua/5.1/itorch/main.lua:143>\n\t...stinmaojones/torch/install/share/lua/5.1/lzmq/poller.lua:75: in function 'poll'\n\t...nmaojones/torch/install/share/lua/5.1/lzmq/impl/loop.lua:307: in function 'poll'\n\t...nmaojones/torch/install/share/lua/5.1/lzmq/impl/loop.lua:325: in function 'sleep_ex'\n\t...nmaojones/torch/install/share/lua/5.1/lzmq/impl/loop.lua:370: in function 'start'\n\t...stinmaojones/torch/install/share/lua/5.1/itorch/main.lua:350: in main chunk\n\t[C]: in function 'require'\n\t[string \"arg={'/Users/justinmaojones/Library/Jupyter/r...\"]:1: in main chunk",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "[string \"local f = function() return rnn2.gradRecurren...\"]:1: attempt to index global 'rnn2' (a nil value)\nstack traceback:\n\t[string \"local f = function() return rnn2.gradRecurren...\"]:1: in function 'f'\n\t[string \"local f = function() return rnn2.gradRecurren...\"]:1: in main chunk\n\t[C]: in function 'xpcall'\n\t...stinmaojones/torch/install/share/lua/5.1/itorch/main.lua:179: in function <...stinmaojones/torch/install/share/lua/5.1/itorch/main.lua:143>\n\t...stinmaojones/torch/install/share/lua/5.1/lzmq/poller.lua:75: in function 'poll'\n\t...nmaojones/torch/install/share/lua/5.1/lzmq/impl/loop.lua:307: in function 'poll'\n\t...nmaojones/torch/install/share/lua/5.1/lzmq/impl/loop.lua:325: in function 'sleep_ex'\n\t...nmaojones/torch/install/share/lua/5.1/lzmq/impl/loop.lua:370: in function 'start'\n\t...stinmaojones/torch/install/share/lua/5.1/itorch/main.lua:350: in main chunk\n\t[C]: in function 'require'\n\t[string \"arg={'/Users/justinmaojones/Library/Jupyter/r...\"]:1: in main chunk"
     ]
    }
   ],
   "source": [
    "rnn2.gradRecurrentInput"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "p1,dp1 = rnn1:getParameters()\n",
    "p2,dp2 = rnn2:getParameters()\n",
    "\n",
    "dp1:zero();\n",
    "dp2:zero()\n",
    "\n",
    "p2:copy(p1)\n",
    "\n",
    "if batchSize == 1 then\n",
    "    x = torch.randn(T,diminput)\n",
    "    dy = torch.randn(T,dimoutput)\n",
    "else\n",
    "    x = torch.randn(batchSize,T,diminput)\n",
    "    dy = torch.randn(batchSize,T,dimoutput)\n",
    "end\n",
    "\n",
    "f1 = rnn1:forward(x)\n",
    "b1 = rnn1:backward(x,dy)\n",
    "\n",
    "f2 = rnn2:forward(x)\n",
    "b2 = rnn2:backward(x,dy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0\t\n"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(f1-f2):norm()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0\t\n"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(b1-b2):norm()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Attention!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "require 'nn';\n",
    "require 'Attention';\n",
    "require 'LSTM';\n",
    "require 'nngraph';\n",
    "nngraph.setDebug(true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "diminput = 4\n",
    "dimoutput = 3\n",
    "scoreDepth = 5\n",
    "hybridAttendFilterSize = 3\n",
    "hybridAttendFeatureMaps = 4\n",
    "stateDepth = 4\n",
    "annotationDepth = 6\n",
    "outputDepth = 2\n",
    "L = 10\n",
    "T = 9\n",
    "batchSize = 4\n",
    "peepholes = false\n",
    "mlpDepth = 7\n",
    "\n",
    "\n",
    "dec_rec_inp = nn.Identity()()\n",
    "inp, prev_s, prev_mem = dec_rec_inp:split(3)\n",
    "c, prev_y = inp:split(2)\n",
    "\n",
    "lstm_inp = nn.CAddTable()({nn.Linear(annotationDepth,stateDepth)(c),nn.Linear(outputDepth,stateDepth)(prev_y)})\n",
    "lstm = nn.LSTM(stateDepth,stateDepth,peepholes)({lstm_inp,prev_s,prev_mem})\n",
    "\n",
    "nngraph.annotateNodes()\n",
    "decoder_recurrent = nn.gModule({dec_rec_inp},{lstm})\n",
    "decoder_recurrent.name = \"decoder_recurrent\"\n",
    "\n",
    "dec_mlp_inp = nn.Identity()()\n",
    "mlp_inp = nn.JoinTable(1,1)(dec_mlp_inp)\n",
    "mlp     = nn.Sequential()\n",
    "mlp:add(nn.Linear(stateDepth+annotationDepth,mlpDepth))\n",
    "mlp:add(nn.ReLU())\n",
    "mlp:add(nn.Linear(mlpDepth,outputDepth))\n",
    "mlp:add(nn.LogSoftMax())\n",
    "decoder_mlp = nn.gModule({dec_mlp_inp},{mlp(mlp_inp)})\n",
    "decoder_mlp.name = \"decoder_mlp\"\n",
    "\n",
    "\n",
    "m = nn.Attention(decoder_recurrent,decoder_mlp,scoreDepth,hybridAttendFilterSize,hybridAttendFeatureMaps,stateDepth,annotationDepth,outputDepth,L,T)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RNN3 type(input) ==\ttable\t\n",
       "RNN3 input = \t{\n",
       "  1 : DoubleTensor - size: 10x5\n",
       "  2 : DoubleTensor - size: 10x6\n",
       "}\n",
       "==================================> RNN t =\t1\t\n",
       "Recurrent type(inp) ==\ttable\t\n",
       "Recurrent inp = \t{\n",
       "  1 : DoubleTensor - size: 10x5\n",
       "  2 : DoubleTensor - size: 10x6\n",
       "}\n",
       "Recurrent\t\n",
       "inp\t{\n",
       "  1 : DoubleTensor - size: 10x5\n",
       "  2 : DoubleTensor - size: 10x6\n",
       "}\n",
       "prev_y\t 0\n",
       " 0\n",
       "[torch.DoubleTensor of size 2]\n",
       "\n",
       "prev_h\t{\n",
       "  1 : DoubleTensor - size: 10\n",
       "  2 : DoubleTensor - size: 4\n",
       "  3 : DoubleTensor - size: 4\n",
       "}\n"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "RNN3 output\t\n",
       "y\t-0.8086\n",
       "-0.5897\n",
       "[torch.DoubleTensor of size 2]\n",
       "\n",
       "h\t{\n",
       "  1 : DoubleTensor - size: 10\n",
       "  2 : DoubleTensor - size: 4\n",
       "  3 : DoubleTensor - size: 4\n",
       "}\n",
       "==================================> RNN t =\t2\t\n",
       "Recurrent type(inp) ==\ttable\t\n",
       "Recurrent inp = \t{\n",
       "  1 : DoubleTensor - size: 10x5\n",
       "  2 : DoubleTensor - size: 10x6\n",
       "}\n",
       "Recurrent\t\n",
       "inp\t{\n",
       "  1 : DoubleTensor - size: 10x5\n",
       "  2 : DoubleTensor - size: 10x6\n",
       "}\n",
       "prev_y\t-0.8086\n",
       "-0.5897\n",
       "[torch.DoubleTensor of size 2]\n",
       "\n",
       "prev_h\t{\n",
       "  1 : DoubleTensor - size: 10\n",
       "  2 : DoubleTensor - size: 4\n",
       "  3 : DoubleTensor - size: 4\n",
       "}\n",
       "RNN3 output\t\n",
       "y\t-0.8141\n",
       "-0.5853\n",
       "[torch.DoubleTensor of size 2]\n",
       "\n",
       "h\t{\n",
       "  1 : DoubleTensor - size: 10\n",
       "  2 : DoubleTensor - size: 4\n",
       "  3 : DoubleTensor - size: 4\n",
       "}\n",
       "==================================> RNN t =\t3\t\n",
       "Recurrent type(inp) ==\ttable\t\n",
       "Recurrent inp = \t{\n",
       "  1 : DoubleTensor - size: 10x5\n",
       "  2 : DoubleTensor - size: 10x6\n",
       "}\n",
       "Recurrent\t\n",
       "inp\t{\n",
       "  1 : DoubleTensor - size: 10x5\n",
       "  2 : DoubleTensor - size: 10x6\n",
       "}\n",
       "prev_y\t-0.8141\n",
       "-0.5853\n",
       "[torch.DoubleTensor of size 2]\n",
       "\n",
       "prev_h\t{\n",
       "  1 : DoubleTensor - size: 10\n",
       "  2 : DoubleTensor - size: 4\n",
       "  3 : DoubleTensor - size: 4\n",
       "}\n"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "RNN3 output\t\n",
       "y\t-0.8176\n",
       "-0.5825\n",
       "[torch.DoubleTensor of size 2]\n",
       "\n",
       "h\t{\n",
       "  1 : DoubleTensor - size: 10\n",
       "  2 : DoubleTensor - size: 4\n",
       "  3 : DoubleTensor - size: 4\n",
       "}\n",
       "==================================> RNN t =\t4\t\n",
       "Recurrent type(inp) ==\ttable\t\n",
       "Recurrent inp = \t{\n",
       "  1 : DoubleTensor - size: 10x5\n",
       "  2 : DoubleTensor - size: 10x6\n",
       "}\n",
       "Recurrent\t\n",
       "inp\t{\n",
       "  1 : DoubleTensor - size: 10x5\n",
       "  2 : DoubleTensor - size: 10x6\n",
       "}\n",
       "prev_y\t-0.8176\n",
       "-0.5825\n",
       "[torch.DoubleTensor of size 2]\n",
       "\n",
       "prev_h\t{\n",
       "  1 : DoubleTensor - size: 10\n",
       "  2 : DoubleTensor - size: 4\n",
       "  3 : DoubleTensor - size: 4\n",
       "}\n",
       "RNN3 output\t\n",
       "y\t-0.8199\n",
       "-0.5807\n",
       "[torch.DoubleTensor of size 2]\n",
       "\n",
       "h\t{\n",
       "  1 : DoubleTensor - size: 10\n",
       "  2 : DoubleTensor - size: 4\n",
       "  3 : DoubleTensor - size: 4\n",
       "}\n",
       "==================================> RNN t =\t5\t\n",
       "Recurrent type(inp) ==\ttable\t\n",
       "Recurrent inp = \t{\n",
       "  1 : DoubleTensor - size: 10x5\n",
       "  2 : DoubleTensor - size: 10x6\n",
       "}\n",
       "Recurrent\t\n"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "inp\t{\n",
       "  1 : DoubleTensor - size: 10x5\n",
       "  2 : DoubleTensor - size: 10x6\n",
       "}\n",
       "prev_y\t-0.8199\n",
       "-0.5807\n",
       "[torch.DoubleTensor of size 2]\n",
       "\n",
       "prev_h\t{\n",
       "  1 : DoubleTensor - size: 10\n",
       "  2 : DoubleTensor - size: 4\n",
       "  3 : DoubleTensor - size: 4\n",
       "}\n",
       "RNN3 output\t\n",
       "y\t-0.8215\n",
       "-0.5794\n",
       "[torch.DoubleTensor of size 2]\n",
       "\n",
       "h\t{\n",
       "  1 : DoubleTensor - size: 10\n",
       "  2 : DoubleTensor - size: 4\n",
       "  3 : DoubleTensor - size: 4\n",
       "}\n",
       "==================================> RNN t =\t6\t\n",
       "Recurrent type(inp) ==\ttable\t\n",
       "Recurrent inp = \t{\n",
       "  1 : DoubleTensor - size: 10x5\n",
       "  2 : DoubleTensor - size: 10x6\n",
       "}\n",
       "Recurrent\t\n",
       "inp\t{\n",
       "  1 : DoubleTensor - size: 10x5\n",
       "  2 : DoubleTensor - size: 10x6\n",
       "}\n",
       "prev_y\t-0.8215\n",
       "-0.5794\n",
       "[torch.DoubleTensor of size 2]\n",
       "\n",
       "prev_h\t{\n",
       "  1 : DoubleTensor - size: 10\n",
       "  2 : DoubleTensor - size: 4\n",
       "  3 : DoubleTensor - size: 4\n",
       "}\n"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "RNN3 output\t\n",
       "y\t-0.8227\n",
       "-0.5785\n",
       "[torch.DoubleTensor of size 2]\n",
       "\n",
       "h\t{\n",
       "  1 : DoubleTensor - size: 10\n",
       "  2 : DoubleTensor - size: 4\n",
       "  3 : DoubleTensor - size: 4\n",
       "}\n",
       "==================================> RNN t =\t7\t\n",
       "Recurrent type(inp) ==\ttable\t\n",
       "Recurrent inp = \t{\n",
       "  1 : DoubleTensor - size: 10x5\n",
       "  2 : DoubleTensor - size: 10x6\n",
       "}\n",
       "Recurrent\t\n",
       "inp\t{\n",
       "  1 : DoubleTensor - size: 10x5\n",
       "  2 : DoubleTensor - size: 10x6\n",
       "}\n",
       "prev_y\t-0.8227\n",
       "-0.5785\n",
       "[torch.DoubleTensor of size 2]\n",
       "\n",
       "prev_h\t{\n",
       "  1 : DoubleTensor - size: 10\n",
       "  2 : DoubleTensor - size: 4\n",
       "  3 : DoubleTensor - size: 4\n",
       "}\n",
       "RNN3 output\t\n",
       "y\t-0.8235\n",
       "-0.5778\n",
       "[torch.DoubleTensor of size 2]\n",
       "\n",
       "h\t{\n",
       "  1 : DoubleTensor - size: 10\n",
       "  2 : DoubleTensor - size: 4\n",
       "  3 : DoubleTensor - size: 4\n",
       "}\n",
       "==================================> RNN t =\t8\t\n",
       "Recurrent type(inp) ==\ttable\t\n",
       "Recurrent inp = \t{\n",
       "  1 : DoubleTensor - size: 10x5\n",
       "  2 : DoubleTensor - size: 10x6\n",
       "}\n",
       "Recurrent\t\n",
       "inp\t{\n",
       "  1 : DoubleTensor - size: 10x5\n",
       "  2 : DoubleTensor - size: 10x6\n",
       "}\n",
       "prev_y\t-0.8235\n",
       "-0.5778\n",
       "[torch.DoubleTensor of size 2]\n",
       "\n",
       "prev_h\t{\n",
       "  1 : DoubleTensor - size: 10\n",
       "  2 : DoubleTensor - size: 4\n",
       "  3 : DoubleTensor - size: 4\n",
       "}\n"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "RNN3 output\t\n",
       "y\t-0.8241\n",
       "-0.5773\n",
       "[torch.DoubleTensor of size 2]\n",
       "\n",
       "h\t{\n",
       "  1 : DoubleTensor - size: 10\n",
       "  2 : DoubleTensor - size: 4\n",
       "  3 : DoubleTensor - size: 4\n",
       "}\n",
       "==================================> RNN t =\t9\t\n",
       "Recurrent type(inp) ==\ttable\t\n",
       "Recurrent inp = \t{\n",
       "  1 : DoubleTensor - size: 10x5\n",
       "  2 : DoubleTensor - size: 10x6\n",
       "}\n",
       "Recurrent\t\n",
       "inp\t{\n",
       "  1 : DoubleTensor - size: 10x5\n",
       "  2 : DoubleTensor - size: 10x6\n",
       "}\n",
       "prev_y\t-0.8241\n",
       "-0.5773\n",
       "[torch.DoubleTensor of size 2]\n",
       "\n",
       "prev_h\t{\n",
       "  1 : DoubleTensor - size: 10\n",
       "  2 : DoubleTensor - size: 4\n",
       "  3 : DoubleTensor - size: 4\n",
       "}\n",
       "RNN3 output\t\n",
       "y\t-0.8246\n",
       "-0.5770\n",
       "[torch.DoubleTensor of size 2]\n",
       "\n",
       "h\t{\n",
       "  1 : DoubleTensor - size: 10\n",
       "  2 : DoubleTensor - size: 4\n",
       "  3 : DoubleTensor - size: 4\n",
       "}\n",
       "LSTM\t\n",
       "inp\t 1.6939\n",
       "-0.1172\n",
       " 1.2906\n",
       " 1.8179\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "prev_h\t-0.2173\n",
       " 0.1906\n",
       "-0.1768\n",
       "-0.2238\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "prev_c\t-0.5136\n",
       " 0.3187\n",
       "-0.5425\n",
       "-0.6642\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "LSTM\t\n",
       "inp\t 1.6938\n",
       "-0.1170\n",
       " 1.2906\n",
       " 1.8176\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "prev_h\t-0.2171\n",
       " 0.1783\n",
       "-0.1736\n",
       "-0.2119\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "prev_c\t-0.5116\n",
       " 0.2976\n",
       "-0.5282\n",
       "-0.6176\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "LSTM\t\n",
       "inp\t 1.6935\n",
       "-0.1168\n",
       " 1.2906\n",
       " 1.8172\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "prev_h\t-0.2157\n",
       " 0.1629\n",
       "-0.1683\n",
       "-0.1979\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "prev_c\t-0.5060\n",
       " 0.2716\n",
       "-0.5063\n",
       "-0.5653\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "LSTM\t\n",
       "inp\t 1.6932\n",
       "-0.1164\n",
       " 1.2907\n",
       " 1.8166\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "prev_h\t-0.2118\n",
       " 0.1436\n",
       "-0.1600\n",
       "-0.1811\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "prev_c\t-0.4937\n",
       " 0.2393\n",
       "-0.4738\n",
       "-0.5059\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "LSTM\t\n",
       "inp\t 1.6927\n",
       "-0.1158\n",
       " 1.2908\n",
       " 1.8159\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "prev_h\t"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "-0.2034\n",
       " 0.1193\n",
       "-0.1469\n",
       "-0.1602\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "prev_c\t-0.4694\n",
       " 0.1991\n",
       "-0.4266\n",
       "-0.4367\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "LSTM\t\n",
       "inp\t 1.6921\n",
       "-0.1151\n",
       " 1.2909\n",
       " 1.8147\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "prev_h\t-0.1868\n",
       " 0.0889\n",
       "-0.1268\n",
       "-0.1333\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "prev_c\t-0.4248\n",
       " 0.1494\n",
       "-0.3598\n",
       "-0.3529\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "LSTM\t\n",
       "inp\t 1.6911\n",
       "-0.1139\n",
       " 1.2911\n",
       " 1.8131\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "prev_h\t-0.1555\n",
       " 0.0521\n",
       "-0.0965\n",
       "-0.0962\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "prev_c\t-0.3460\n",
       " 0.0889\n",
       "-0.2674\n",
       "-0.2458\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "LSTM\t\n",
       "inp\t 1.6895\n",
       "-0.1121\n",
       " 1.2913\n",
       " 1.8105\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "prev_h\t0.01 *\n",
       "-9.2555\n",
       " 0.9338\n",
       "-5.3572\n",
       "-4.5063\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "prev_c\t-0.2114\n",
       " 0.0170\n",
       "-0.1436\n",
       "-0.0974\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "LSTM\t\n",
       "inp\t 1.6365\n",
       " 0.1245\n",
       " 0.9352\n",
       " 1.3752\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "prev_h\t 0\n",
       " 0\n",
       " 0\n",
       " 0\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n",
       "prev_c\t 0\n",
       " 0\n",
       " 0\n",
       " 0\n",
       "[torch.DoubleTensor of size 4]\n",
       "\n"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h = torch.ones(L,annotationDepth)\n",
    "m:forward(h)\n",
    "m:backward(h,torch.ones(T,outputDepth))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RNN3 type(input) ==\ttable\t\n",
       "RNN3 input = \t{\n",
       "  1 : DoubleTensor - size: 4x10x5\n",
       "  2 : DoubleTensor - size: 4x10x6\n",
       "}\n",
       "==================================> RNN t =\t1\t\n",
       "Recurrent type(inp) ==\ttable\t\n",
       "Recurrent inp = \t{\n",
       "  1 : DoubleTensor - size: 4x10x5\n",
       "  2 : DoubleTensor - size: 4x10x6\n",
       "}\n",
       "Recurrent\t\n",
       "inp\t{\n",
       "  1 : DoubleTensor - size: 4x10x5\n",
       "  2 : DoubleTensor - size: 4x10x6\n",
       "}\n",
       "prev_y\t 0  0\n",
       " 0  0\n",
       " 0  0\n",
       " 0  0\n",
       "[torch.DoubleTensor of size 4x2]\n",
       "\n",
       "prev_h\t{\n",
       "  1 : DoubleTensor - size: 4x10\n",
       "  2 : DoubleTensor - size: 4x4\n",
       "  3 : DoubleTensor - size: 4x4\n",
       "}\n"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "RNN3 output\t\n",
       "y\t-0.8086 -0.5897\n",
       "-0.8086 -0.5897\n",
       "-0.8086 -0.5897\n",
       "-0.8086 -0.5897\n",
       "[torch.DoubleTensor of size 4x2]\n",
       "\n",
       "h\t{\n",
       "  1 : DoubleTensor - size: 4x10\n",
       "  2 : DoubleTensor - size: 4x4\n",
       "  3 : DoubleTensor - size: 4x4\n",
       "}\n",
       "==================================> RNN t =\t2\t\n",
       "Recurrent type(inp) ==\ttable\t\n",
       "Recurrent inp = \t{\n",
       "  1 : DoubleTensor - size: 4x10x5\n",
       "  2 : DoubleTensor - size: 4x10x6\n",
       "}\n",
       "Recurrent\t\n",
       "inp\t{\n",
       "  1 : DoubleTensor - size: 4x10x5\n",
       "  2 : DoubleTensor - size: 4x10x6\n",
       "}\n",
       "prev_y\t-0.8086 -0.5897\n",
       "-0.8086 -0.5897\n",
       "-0.8086 -0.5897\n",
       "-0.8086 -0.5897\n",
       "[torch.DoubleTensor of size 4x2]\n",
       "\n",
       "prev_h\t{\n",
       "  1 : DoubleTensor - size: 4x10\n",
       "  2 : DoubleTensor - size: 4x4\n",
       "  3 : DoubleTensor - size: 4x4\n",
       "}\n",
       "RNN3 output\t\n",
       "y\t-0.8141 -0.5853\n",
       "-0.8141 -0.5853\n",
       "-0.8141 -0.5853\n",
       "-0.8141 -0.5853\n",
       "[torch.DoubleTensor of size 4x2]\n",
       "\n",
       "h\t{\n"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "  1 : DoubleTensor - size: 4x10\n",
       "  2 : DoubleTensor - size: 4x4\n",
       "  3 : DoubleTensor - size: 4x4\n",
       "}\n",
       "==================================> RNN t =\t3\t\n",
       "Recurrent type(inp) ==\ttable\t\n",
       "Recurrent inp = \t{\n",
       "  1 : DoubleTensor - size: 4x10x5\n",
       "  2 : DoubleTensor - size: 4x10x6\n",
       "}\n",
       "Recurrent\t\n",
       "inp\t{\n",
       "  1 : DoubleTensor - size: 4x10x5\n",
       "  2 : DoubleTensor - size: 4x10x6\n",
       "}\n",
       "prev_y\t-0.8141 -0.5853\n",
       "-0.8141 -0.5853\n",
       "-0.8141 -0.5853\n",
       "-0.8141 -0.5853\n",
       "[torch.DoubleTensor of size 4x2]\n",
       "\n",
       "prev_h\t{\n",
       "  1 : DoubleTensor - size: 4x10\n",
       "  2 : DoubleTensor - size: 4x4\n",
       "  3 : DoubleTensor - size: 4x4\n",
       "}\n",
       "RNN3 output\t\n",
       "y\t-0.8176 -0.5825\n",
       "-0.8176 -0.5825\n",
       "-0.8176 -0.5825\n",
       "-0.8176 -0.5825\n",
       "[torch.DoubleTensor of size 4x2]\n",
       "\n",
       "h\t{\n",
       "  1 : DoubleTensor - size: 4x10\n",
       "  2 : DoubleTensor - size: 4x4\n",
       "  3 : DoubleTensor - size: 4x4\n",
       "}\n",
       "==================================> RNN t =\t4\t\n",
       "Recurrent type(inp) ==\ttable\t\n",
       "Recurrent inp = \t{\n",
       "  1 : DoubleTensor - size: 4x10x5\n",
       "  2 : DoubleTensor - size: 4x10x6\n",
       "}\n",
       "Recurrent\t\n",
       "inp\t{\n",
       "  1 : DoubleTensor - size: 4x10x5\n",
       "  2 : DoubleTensor - size: 4x10x6\n",
       "}\n",
       "prev_y\t-0.8176 -0.5825\n",
       "-0.8176 -0.5825\n",
       "-0.8176 -0.5825\n",
       "-0.8176 -0.5825\n",
       "[torch.DoubleTensor of size 4x2]\n",
       "\n",
       "prev_h\t{\n",
       "  1 : DoubleTensor - size: 4x10\n",
       "  2 : DoubleTensor - size: 4x4\n",
       "  3 : DoubleTensor - size: 4x4\n",
       "}\n"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "RNN3 output\t\n",
       "y\t-0.8199 -0.5807\n",
       "-0.8199 -0.5807\n",
       "-0.8199 -0.5807\n",
       "-0.8199 -0.5807\n",
       "[torch.DoubleTensor of size 4x2]\n",
       "\n",
       "h\t{\n",
       "  1 : DoubleTensor - size: 4x10\n",
       "  2 : DoubleTensor - size: 4x4\n",
       "  3 : DoubleTensor - size: 4x4\n",
       "}\n",
       "==================================> RNN t =\t5\t\n",
       "Recurrent type(inp) ==\ttable\t\n",
       "Recurrent inp = \t{\n",
       "  1 : DoubleTensor - size: 4x10x5\n",
       "  2 : DoubleTensor - size: 4x10x6\n",
       "}\n",
       "Recurrent\t\n",
       "inp\t{\n",
       "  1 : DoubleTensor - size: 4x10x5\n",
       "  2 : DoubleTensor - size: 4x10x6\n",
       "}\n",
       "prev_y\t-0.8199 -0.5807\n",
       "-0.8199 -0.5807\n",
       "-0.8199 -0.5807\n",
       "-0.8199 -0.5807\n",
       "[torch.DoubleTensor of size 4x2]\n",
       "\n",
       "prev_h\t{\n",
       "  1 : DoubleTensor - size: 4x10\n",
       "  2 : DoubleTensor - size: 4x4\n",
       "  3 : DoubleTensor - size: 4x4\n",
       "}\n"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "RNN3 output\t\n",
       "y\t-0.8215 -0.5794\n",
       "-0.8215 -0.5794\n",
       "-0.8215 -0.5794\n",
       "-0.8215 -0.5794\n",
       "[torch.DoubleTensor of size 4x2]\n",
       "\n",
       "h\t{\n",
       "  1 : DoubleTensor - size: 4x10\n",
       "  2 : DoubleTensor - size: 4x4\n",
       "  3 : DoubleTensor - size: 4x4\n",
       "}\n",
       "==================================> RNN t =\t6\t\n",
       "Recurrent type(inp) ==\ttable\t\n",
       "Recurrent inp = \t{\n",
       "  1 : DoubleTensor - size: 4x10x5\n",
       "  2 : DoubleTensor - size: 4x10x6\n",
       "}\n",
       "Recurrent\t\n",
       "inp\t{\n",
       "  1 : DoubleTensor - size: 4x10x5\n",
       "  2 : DoubleTensor - size: 4x10x6\n",
       "}\n",
       "prev_y\t-0.8215 -0.5794\n",
       "-0.8215 -0.5794\n",
       "-0.8215 -0.5794\n",
       "-0.8215 -0.5794\n",
       "[torch.DoubleTensor of size 4x2]\n",
       "\n",
       "prev_h\t{\n",
       "  1 : DoubleTensor - size: 4x10\n",
       "  2 : DoubleTensor - size: 4x4\n",
       "  3 : DoubleTensor - size: 4x4\n",
       "}\n",
       "RNN3 output\t\n",
       "y\t-0.8227 -0.5785\n",
       "-0.8227 -0.5785\n",
       "-0.8227 -0.5785\n",
       "-0.8227 -0.5785\n",
       "[torch.DoubleTensor of size 4x2]\n",
       "\n",
       "h\t{\n",
       "  1 : DoubleTensor - size: 4x10\n",
       "  2 : DoubleTensor - size: 4x4\n",
       "  3 : DoubleTensor - size: 4x4\n",
       "}\n",
       "==================================> RNN t =\t7\t\n",
       "Recurrent type(inp) ==\ttable\t\n",
       "Recurrent inp = \t{\n",
       "  1 : DoubleTensor - size: 4x10x5\n",
       "  2 : DoubleTensor - size: 4x10x6\n",
       "}\n",
       "Recurrent\t\n",
       "inp\t{\n",
       "  1 : DoubleTensor - size: 4x10x5\n",
       "  2 : DoubleTensor - size: 4x10x6\n",
       "}\n",
       "prev_y\t-0.8227 -0.5785\n",
       "-0.8227 -0.5785\n",
       "-0.8227 -0.5785\n",
       "-0.8227 -0.5785\n",
       "[torch.DoubleTensor of size 4x2]\n",
       "\n",
       "prev_h\t{\n",
       "  1 : DoubleTensor - size: 4x10\n",
       "  2 : DoubleTensor - size: 4x4\n",
       " "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       " 3 : DoubleTensor - size: 4x4\n",
       "}\n",
       "RNN3 output\t\n",
       "y\t-0.8235 -0.5778\n",
       "-0.8235 -0.5778\n",
       "-0.8235 -0.5778\n",
       "-0.8235 -0.5778\n",
       "[torch.DoubleTensor of size 4x2]\n",
       "\n",
       "h\t{\n",
       "  1 : DoubleTensor - size: 4x10\n",
       "  2 : DoubleTensor - size: 4x4\n",
       "  3 : DoubleTensor - size: 4x4\n",
       "}\n",
       "==================================> RNN t =\t8\t\n",
       "Recurrent type(inp) ==\ttable\t\n",
       "Recurrent inp = \t{\n",
       "  1 : DoubleTensor - size: 4x10x5\n",
       "  2 : DoubleTensor - size: 4x10x6\n",
       "}\n",
       "Recurrent\t\n",
       "inp\t{\n",
       "  1 : DoubleTensor - size: 4x10x5\n",
       "  2 : DoubleTensor - size: 4x10x6\n",
       "}\n",
       "prev_y\t-0.8235 -0.5778\n",
       "-0.8235 -0.5778\n",
       "-0.8235 -0.5778\n",
       "-0.8235 -0.5778\n",
       "[torch.DoubleTensor of size 4x2]\n",
       "\n",
       "prev_h\t{\n",
       "  1 : DoubleTensor - size: 4x10\n",
       "  2 : DoubleTensor - size: 4x4\n",
       "  3 : DoubleTensor - size: 4x4\n",
       "}\n"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "RNN3 output\t\n",
       "y\t-0.8241 -0.5773\n",
       "-0.8241 -0.5773\n",
       "-0.8241 -0.5773\n",
       "-0.8241 -0.5773\n",
       "[torch.DoubleTensor of size 4x2]\n",
       "\n",
       "h\t{\n",
       "  1 : DoubleTensor - size: 4x10\n",
       "  2 : DoubleTensor - size: 4x4\n",
       "  3 : DoubleTensor - size: 4x4\n",
       "}\n",
       "==================================> RNN t =\t9\t\n",
       "Recurrent type(inp) ==\ttable\t\n",
       "Recurrent inp = \t{\n",
       "  1 : DoubleTensor - size: 4x10x5\n",
       "  2 : DoubleTensor - size: 4x10x6\n",
       "}\n",
       "Recurrent\t\n",
       "inp\t{\n",
       "  1 : DoubleTensor - size: 4x10x5\n",
       "  2 : DoubleTensor - size: 4x10x6\n",
       "}\n",
       "prev_y\t-0.8241 -0.5773\n",
       "-0.8241 -0.5773\n",
       "-0.8241 -0.5773\n",
       "-0.8241 -0.5773\n",
       "[torch.DoubleTensor of size 4x2]\n",
       "\n",
       "prev_h\t{\n",
       "  1 : DoubleTensor - size: 4x10\n",
       "  2 : DoubleTensor - size: 4x4\n",
       "  3 : DoubleTensor - size: 4x4\n",
       "}\n",
       "RNN3 output\t\n",
       "y\t-0.8246 -0.5770\n",
       "-0.8246 -0.5770\n",
       "-0.8246 -0.5770\n",
       "-0.8246 -0.5770\n",
       "[torch.DoubleTensor of size 4x2]\n",
       "\n",
       "h\t{\n",
       "  1 : DoubleTensor - size: 4x10\n",
       "  2 : DoubleTensor - size: 4x4\n",
       "  3 : DoubleTensor - size: 4x4\n",
       "}\n"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "LSTM\t\n",
       "inp\t 1.6939 -0.1172  1.2906  1.8179\n",
       " 1.6939 -0.1172  1.2906  1.8179\n",
       " 1.6939 -0.1172  1.2906  1.8179\n",
       " 1.6939 -0.1172  1.2906  1.8179\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "prev_h\t-0.2173  0.1906 -0.1768 -0.2238\n",
       "-0.2173  0.1906 -0.1768 -0.2238\n",
       "-0.2173  0.1906 -0.1768 -0.2238\n",
       "-0.2173  0.1906 -0.1768 -0.2238\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "prev_c\t-0.5136  0.3187 -0.5425 -0.6642\n",
       "-0.5136  0.3187 -0.5425 -0.6642\n",
       "-0.5136  0.3187 -0.5425 -0.6642\n",
       "-0.5136  0.3187 -0.5425 -0.6642\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "LSTM\t\n",
       "inp\t 1.6938 -0.1170  1.2906  1.8176\n",
       " 1.6938 -0.1170  1.2906  1.8176\n",
       " 1.6938 -0.1170  1.2906  1.8176\n",
       " 1.6938 -0.1170  1.2906  1.8176\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "prev_h\t-0.2171  0.1783 -0.1736 -0.2119\n",
       "-0.2171  0.1783 -0.1736 -0.2119\n",
       "-0.2171  0.1783 -0.1736 -0.2119\n",
       "-0.2171  0.1783 -0.1736 -0.2119\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "prev_c\t-0.5116  0.2976 -0.5282 -0.6176\n",
       "-0.5116  0.2976 -0.5282 -0.6176\n",
       "-0.5116  0.2976 -0.5282 -0.6176\n",
       "-0.5116  0.2976 -0.5282 -0.6176\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "LSTM\t\n",
       "inp\t 1.6935 -0.1168  1.2906  1.8172\n",
       " 1.6935 -0.1168  1.2906  1.8172\n",
       " 1.6935 -0.1168  1.2906  1.8172\n",
       " 1.6935 -0.1168  1.2906  1.8172\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "prev_h\t-0.2157  0.1629 -0.1683 -0.1979\n",
       "-0.2157  0.1629 -0.1683 -0.1979\n",
       "-0.2157  0.1629 -0.1683 -0.1979\n",
       "-0.2157  0.1629 -0.1683 -0.1979\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "prev_c\t-0.5060  0.2716 -0.5063 -0.5653\n",
       "-0.5060  0.2716 -0.5063 -0.5653\n",
       "-0.5060  0.2716 -0.5063 -0.5653\n",
       "-0.5060  0.2716 -0.5063 -0.5653\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "LSTM\t\n",
       "inp\t 1.6932 -0.1164  1.2907  1.8166\n",
       " 1.6932 -0.1164  1.2907  1.8166\n",
       " 1.6932 -0.1164  1.2907  1.8166\n",
       " 1.6932 -0.1164  1.2907  1.8166\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "prev_h\t-0.2118  0.1436 -0.1600 -0.1811\n",
       "-0.2118  0.1436 -0.1600 -0.1811\n",
       "-0.2118  0.1436 -0.1600 -0.1811\n",
       "-0.2118  0.1436 -0.1600 -0.1811\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "prev_c\t-0.4937  0.2393 -0.4738 -0.5059\n",
       "-0.4937  0.2393 -0.4738 -0.5059\n",
       "-0.4937  0.2393 -0.4738 -0.5059\n",
       "-0.4937  0.2393 -0.4738 -0.5059\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "LSTM\t\n",
       "inp\t 1.6927 -0.1158  1.2908  1.8159\n",
       " 1.6927 -0.1158  1.2908  1.8159\n",
       " 1.6927 -0.1158  1.2908  1.8159\n",
       " 1.6927 -0.1158  1.2908  1.8159\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "prev_h\t-0.2034  0.1193 -0.1469 -0.1602\n",
       "-0.2034  0.1193 -0.1469 -0.1602\n",
       "-0.2034  0.1193 -0.1469 -0.1602\n",
       "-0.2034  0.1193 -0.1469 -0.1602\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "prev_c\t-0.4694  0.1991 -0.4266 -0.4367\n",
       "-0.4694  0.1991 -0.4266 -0.4367\n",
       "-0.4694  0.1991 -0.4266 -0.4367\n",
       "-0.4694  0.1991 -0.4266 -0.4367\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "LSTM\t\n",
       "inp\t 1.6921 -0.1151  1.2909  1.8147\n",
       " 1.6921 -0.1151  1.2909  1.8147\n",
       " 1.6921 -0.1151  1.2909  1.8147\n",
       " 1.6921 -0.1151  1.2909  1.8147\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "prev_h\t-0.1868  0.0889 -0.1268 -0.1333\n",
       "-0.1868  0.0889 -0.1268 -0.1333\n",
       "-0.1868  0.0889 -0.1268 -0.1333\n",
       "-0.1868  0.0889 -0.1268 -0.1333\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "prev_c\t-0.4248  0.1494 -0.3598 -0.3529\n",
       "-0.4248  0.1494 -0.3598 -0.3529\n",
       "-0.4248  0.1494 -0.3598 -0.3529\n",
       "-0.4248  0.1494 -0.3598 -0.3529\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "LSTM\t\n",
       "inp\t 1.6911 -0.1139  1.2911  1.8131\n",
       " 1.6911 -0.1139  1.2911  1.8131\n",
       " 1.6911 -0.1139  1.2911  1.8131\n",
       " 1.6911 -0.1139  1.2911  1.8131\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "prev_h\t-0.1555  0.0521 -0.0965 -0.0962\n",
       "-0.1555  0.0521 -0.0965 -0.0962\n",
       "-0.1555  0.0521 -0.0965 -0.0962\n",
       "-0.1555  0.0521 -0.0965 -0.0962\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "prev_c\t-0.3460  0.0889 -0.2674 -0.2458\n",
       "-0.3460  0.0889 -0.2674 -0.2458\n",
       "-0.3460  0.0889 -0.2674 -0.2458\n",
       "-0.3460  0.0889 -0.2674 -0.2458\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "LSTM\t\n",
       "inp\t 1.6895 -0.1121  1.2913  1.8105\n",
       " 1.6895 -0.1121  1.2913  1.8105\n",
       " 1.6895 -0.1121  1.2913  1.8105\n",
       " 1.6895 -0.1121  1.2913  1.8105\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "prev_h\t0.01 *\n",
       " -9.2555  0.9338 -5.3572 -4.5063\n",
       " -9.2555  0.9338 -5.3572 -4.5063\n",
       " -9.2555  0.9338 -5.3572 -4.5063\n",
       " -9.2555  0.9338 -5.3572 -4.5063\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "prev_c\t-0.2114  0.0170 -0.1436 -0.0974\n",
       "-0.2114  0.0170 -0.1436 -0.0974\n",
       "-0.2114  0.0170 -0.1436 -0.0974\n",
       "-0.2114  0.0170 -0.1436 -0.0974\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "LSTM\t\n",
       "inp\t 1.6365  0.1245  0.9352  1.3752\n",
       " 1.6365  0.1245  0.9352  1.3752\n",
       " 1.6365  0.1245  0.9352  1.3752\n",
       " 1.6365  0.1245  0.9352  1.3752\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "prev_h\t"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       " 0  0  0  0\n",
       " 0  0  0  0\n",
       " 0  0  0  0\n",
       " 0  0  0  0\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n",
       "prev_c\t 0  0  0  0\n",
       " 0  0  0  0\n",
       " 0  0  0  0\n",
       " 0  0  0  0\n",
       "[torch.DoubleTensor of size 4x4]\n",
       "\n"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h = torch.ones(batchSize,L,annotationDepth)\n",
    "m:forward(h)\n",
    "m:backward(h,torch.ones(batchSize,T,outputDepth))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "function customToDot(graph, title, failedNode)\n",
    "   local str = graph:todot(title)\n",
    "   if not failedNode then\n",
    "      return str\n",
    "   end\n",
    "\n",
    "   local failedNodeId = nil\n",
    "   for i, node in ipairs(graph.nodes) do\n",
    "      if node.data == failedNode.data then\n",
    "         failedNodeId = node.id\n",
    "         break\n",
    "      end\n",
    "   end\n",
    "\n",
    "   if failedNodeId ~= nil then\n",
    "      -- The closing '}' is removed.\n",
    "      -- And red fillcolor is specified for the failedNode.\n",
    "      str = string.gsub(str, '}%s*$', '')\n",
    "      str = str .. string.format('n%s[style=filled, fillcolor=red];\\n}',\n",
    "      failedNodeId)\n",
    "   end\n",
    "   return str\n",
    "end\n",
    "\n",
    "function saveSvg(svgPathPrefix, dotStr)\n",
    "   io.stderr:write(string.format(\"saving %s.svg\\n\", svgPathPrefix))\n",
    "   local dotPath = svgPathPrefix .. '.dot'\n",
    "   local dotFile = io.open(dotPath, 'w')\n",
    "   dotFile:write(dotStr)\n",
    "   dotFile:close()\n",
    "\n",
    "   local svgPath = svgPathPrefix .. '.svg'\n",
    "   local cmd = string.format('dot -Tsvg -o %s %s', svgPath, dotPath)\n",
    "   os.execute(cmd)\n",
    "end\n",
    "\n",
    "function outputGraphViz(gmodule, focusNode)\n",
    "   local focusNode = focusNode or gmodule.outnode\n",
    "   local nInputs = gmodule.nInputs or #gmodule.innode.children\n",
    "   local svgPathPrefix = gmodule.name or string.format(\n",
    "   'nngraph_%sin_%sout', nInputs, #gmodule.outnode.children)\n",
    "   local dotStr = customToDot(gmodule.fg, svgPathPrefix, focusNode)\n",
    "   saveSvg(svgPathPrefix, dotStr)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "analyze_graph = m.rnn.data.module.rnn[1].recurrent\n",
    "outputGraphViz(analyze_graph.forwardnodes[41],analyze_graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "nngraph.Node\n",
       "{\n",
       "  data : \n",
       "    {\n",
       "      annotations : \n",
       "        {\n",
       "          _debugLabel : [[C]]:-1\n",
       "          graphAttributes : \n",
       "            {\n",
       "              tooltip : [[C]]:-1\n",
       "            }\n",
       "        }\n",
       "      mapindex : \n",
       "        {\n",
       "          1 : \n",
       "            {\n",
       "              annotations : \n",
       "                {\n",
       "                  _debugLabel : [./Attention.lua]:115\n",
       "                  graphAttributes : table: 0x0fbbf300\n",
       "                }\n",
       "              input : \n",
       "                {\n",
       "                  1 : DoubleTensor - size: 1x4\n",
       "                  2 : DoubleTensor - size: 1x6\n",
       "                }\n",
       "              module : \n",
       "                decoder_mlp\n",
       "                {\n",
       "                  bg : graph.Graph\n",
       "                  forwardnodes : table: 0x0fd2a528\n",
       "                  outnode : nngraph.Node\n",
       "                  fg : graph.Graph\n",
       "                  verbose : false\n",
       "                  backwardnodes : table: 0x0fd2fee8\n",
       "                  output : DoubleTensor - size: 1x2\n",
       "                  innode : nngraph.Node\n",
       "                  name : decoder_mlp\n",
       "                  nInputs : 1\n",
       "                }\n",
       "              forwardNodeId : 1\n",
       "              mapindex : \n",
       "                {\n",
       "                  1 : table: 0x0fd21300\n",
       "                  2 : table: 0x0fd948e0\n",
       "                  table: 0x0fd21300 : 1\n",
       "                  table: 0x0fd948e0 : 2\n",
       "                }\n",
       "            }\n",
       "          2 : \n",
       "            {\n",
       "              annotations : \n",
       "                {\n",
       "                  _debugLabel : [./Attention.lua]:131\n",
       "                  graphAttributes : table: 0x0fbb24f0\n",
       "                }\n",
       "              input : \n",
       "                {\n",
       "                  1 : DoubleTensor - size: 10\n",
       "                  2 : DoubleTensor - size: 1x4\n",
       "                  3 : DoubleTensor - size: 1x4\n",
       "                }\n",
       "              module : \n",
       "                nn.Identity\n",
       "                {\n",
       "                  gradInput : DoubleTensor - empty\n",
       "                  output : table: 0x0f5c07c8\n",
       "                }\n",
       "              forwardNodeId : 3\n",
       "              mapindex : \n",
       "                {\n",
       "                  1 : table: 0x0fd96528\n",
       "                  2 : table: 0x0fd21300\n",
       "                  3 : table: 0x0fd831e8\n",
       "                  table: 0x0fd831e8 : 3\n",
       "                  table: 0x0fd96528 : 1\n",
       "                  table: 0x0fd21300 : 2\n",
       "                }\n",
       "            }\n",
       "          table: 0x0fd241e0 : 1\n",
       "          table: 0x0fd81260 : 2\n",
       "        }\n",
       "      forwardNodeId : 2\n",
       "      input : \n",
       "        {\n",
       "          1 : DoubleTensor - size: 1x2\n",
       "          2 : \n",
       "            {\n",
       "              1 : DoubleTensor - size: 10\n",
       "              2 : DoubleTensor - size: 1x4\n",
       "              3 : DoubleTensor - size: 1x4\n",
       "            }\n",
       "        }\n",
       "    }\n",
       "  visited : false\n",
       "  id : 2\n",
       "  children : table: 0x0fd84fb8\n",
       "  marked : false\n",
       "}\n"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analyze_graph..o.forwardnodes[41]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "  2.3158e+77\n",
       "  2.3158e+77\n",
       "  2.3278e-57\n",
       "  1.4270e-71\n",
       " 1.4627e+165\n",
       " 4.3579e-309\n",
       "[torch.DoubleTensor of size 6]\n",
       "\n",
       "  2.3158e+77   2.3158e+77  1.7055e+256  1.7572e+243  9.9592e-143  7.7511e+228\n",
       " 7.7515e+228  5.5622e+180  2.9791e+228  9.4487e-143  3.9708e+246  2.7762e+184\n",
       "[torch.DoubleTensor of size 2x6]\n",
       "\n"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "V = nn.View(-1)\n",
    "V:setNumInputDims(2)\n",
    "print(V:forward(torch.Tensor(1,6)))\n",
    "print(V:forward(torch.Tensor(2,1,6)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 20\n",
       "  1\n",
       " 10\n",
       "[torch.LongStorage of size 3]\n",
       "\n"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.Reshape(1,10):forward(torch.ones(20,10)):size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "a[1] = nil\n",
    "a[2] = nil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  0 : 1\n",
       "}\n"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "inp = {u,p,2}\n",
    "a,b,c = unpack(inp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2\t\n"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false\t\n"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#torch.Tensor(1) == 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a = {2,3}\n",
    "\n",
    "function recursiveOperation(x,func)\n",
    "    local output\n",
    "    if type(x) == 'table' then\n",
    "        output = {}\n",
    "        for i = 1, #x do\n",
    "            output[i] = recursiveOperation(x[i],func)\n",
    "        end\n",
    "    else\n",
    "        output = func(x)\n",
    "    end\n",
    "    return output\n",
    "end\n",
    "\n",
    "\n",
    "myfunc = function(x) return torch.Tensor(x) end\n",
    "\n",
    "b = recursiveOperation(a,myfunc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  1 : DoubleTensor - size: 2\n",
       "  2 : DoubleTensor - size: 3\n",
       "}\n"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a = nn.Identity()()\n",
    "b = nn.Identity()()\n",
    "\n",
    "c = nn.Identity()({a,b})\n",
    "--d = nn.JoinTable(1,1)(c)\n",
    "\n",
    "e = nn.gModule({a,b},{c})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  1 : DoubleTensor - size: 2\n",
       "  2 : DoubleTensor - size: 3\n",
       "}\n"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "e:forward({torch.ones(2),torch.zeros(3)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "[string \"x = torch.zeros(3,4)...\"]:3: '<name>' expected near '|'",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "[string \"x = torch.zeros(3,4)...\"]:3: '<name>' expected near '|'"
     ]
    }
   ],
   "source": [
    "x = torch.zeros(3,4)\n",
    "dim = {5,3,4}\n",
    "print(x:|resize(unpack(dim)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x = torch.zeros(3,4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 0  0  0  0\n",
       " 0  0  0  0\n",
       " 0  0  0  0\n",
       "[torch.DoubleTensor of size 3x4]\n",
       "\n"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 0  0  0  0\n",
       " 0  0  0  0\n",
       " 0  0  0  0\n",
       "[torch.DoubleTensor of size 3x4]\n",
       "\n"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x:resize(3,4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 0  0  0  0\n",
       " 0  0  0  0\n",
       " 0  0  0  0\n",
       "[torch.DoubleTensor of size 3x4]\n",
       "\n"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x:resize(3,4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 0  0  0  0\n",
       " 0  0  0  0\n",
       " 0  0  0  0\n",
       "[torch.DoubleTensor of size 3x4]\n",
       "\n"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x:resize(3,4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  1 : DoubleTensor - size: 2\n",
       "  2 : DoubleTensor - size: 3\n",
       "}\n"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = {}\n",
    "a = torch.zeros(2)\n",
    "b[1] = a\n",
    "a = torch.ones(3)\n",
    "b[2] = a\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 0\n",
       " 0\n",
       "[torch.DoubleTensor of size 2]\n",
       "\n"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 1\n",
       " 1\n",
       " 1\n",
       "[torch.DoubleTensor of size 3]\n",
       "\n"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "function resizeGradInput(input,gradInput)\n",
    "    if type(input) == 'table' then\n",
    "        if type(gradInput) ~= 'table' then\n",
    "            gradInput = {}\n",
    "        end\n",
    "        for i = 1, #input do\n",
    "            gradInput[i] = resizeGradInput(input[i],gradInput[i])\n",
    "        end\n",
    "    else\n",
    "        gradInput = gradInput or input.new()\n",
    "        gradInput:resizeAs(input):zero()\n",
    "    end\n",
    "    return gradInput\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 0  0  0  0\n",
       " 0  0  0  0\n",
       " 0  0  0  0\n",
       "[torch.DoubleTensor of size 3x4]\n",
       "\n"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input = torch.Tensor(3,4)\n",
    "gradInput = torch.Tensor(3,4)\n",
    "print(resizeGradInput(input,gradInput))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 0  0  0  0\n",
       " 0  0  0  0\n",
       " 0  0  0  0\n",
       "[torch.DoubleTensor of size 3x4]\n",
       "\n"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input = torch.Tensor(3,4)\n",
    "gradInput = nil\n",
    "print(resizeGradInput(input,gradInput))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  1 : DoubleTensor - size: 3x4\n",
       "}\n"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input = {torch.Tensor(3,4)}\n",
    "gradInput = torch.Tensor(3,4)\n",
    "print(resizeGradInput(input,gradInput))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  1 : DoubleTensor - size: 3x4\n",
       "}\n"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input = {torch.Tensor(3,4)}\n",
    "gradInput = nil\n",
    "print(resizeGradInput(input,gradInput))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  1 : DoubleTensor - size: 3x4\n",
       "  2 : DoubleTensor - size: 2x3\n",
       "}\n"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input = {torch.Tensor(3,4),torch.Tensor(2,3)}\n",
    "gradInput = nil\n",
    "print(resizeGradInput(input,gradInput))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  1 : DoubleTensor - size: 3x4\n",
       "  2 : DoubleTensor - size: 2x3\n",
       "}\n"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input = {torch.Tensor(3,4),torch.Tensor(2,3)}\n",
    "gradInput = {torch.Tensor(3,4),torch.Tensor(2,3)}\n",
    "print(resizeGradInput(input,gradInput))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  1 : DoubleTensor - size: 3x4\n",
       "  2 : DoubleTensor - size: 2x3\n",
       "}\n",
       " 0  0  0  0\n",
       " 0  0  0  0\n",
       " 0  0  0  0\n",
       "[torch.DoubleTensor of size 3x4]\n",
       "\n",
       " 0  0  0\n",
       " 0  0  0\n",
       "[torch.DoubleTensor of size 2x3]\n",
       "\n"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input = {torch.Tensor(3,4),torch.Tensor(2,3)}\n",
    "gradInput = {torch.ones(3,3,4),torch.zeros(3,2,3)}\n",
    "print(resizeGradInput(input,gradInput))\n",
    "print(resizeGradInput(input,gradInput)[1])\n",
    "print(resizeGradInput(input,gradInput)[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  1 : DoubleTensor - size: 3x4\n",
       "  2 : \n",
       "    {\n",
       "      1 : DoubleTensor - size: 1x2\n",
       "      2 : DoubleTensor - size: 2x3\n",
       "    }\n",
       "}\n",
       " 0  0  0  0\n",
       " 0  0  0  0\n",
       " 0  0  0  0\n",
       "[torch.DoubleTensor of size 3x4]\n",
       "\n",
       "{\n",
       "  1 : DoubleTensor - size: 1x2\n",
       "  2 : DoubleTensor - size: 2x3\n",
       "}\n"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input = {torch.Tensor(3,4),{torch.Tensor(1,2),torch.Tensor(2,3)}}\n",
    "gradInput = {torch.ones(3,3,4),torch.zeros(3,2,3)}\n",
    "print(resizeGradInput(input,gradInput))\n",
    "print(resizeGradInput(input,gradInput)[1])\n",
    "print(resizeGradInput(input,gradInput)[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "iTorch",
   "language": "lua",
   "name": "itorch"
  },
  "language_info": {
   "name": "lua",
   "version": "20100"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}