pad_and_pack.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "pad 和 pack 是如何实现的？先来段程序观察一下数据的情况"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[20, 19, 18, 17, 16, 15, 14, 13, 12, 11]\n",
      "torch.Size([155, 30])\n",
      "[10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1]\n",
      "torch.Size([155, 50]) <class 'torch.nn.utils.rnn.PackedSequence'>\n",
      "torch.Size([1, 10, 50]) <class 'torch.autograd.variable.Variable'>\n",
      "torch.Size([1, 10, 50]) <class 'torch.autograd.variable.Variable'>\n",
      "torch.Size([20, 10, 50])\n",
      "<class 'torch.autograd.variable.Variable'>\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "from torch.nn.utils.rnn import pad_packed_sequence, pack_padded_sequence\n",
    "\n",
    "x = Variable(torch.randn(10, 20, 30)).cuda()\n",
    "lens = range(11, 21)[::-1]\n",
    "print(lens)\n",
    "x = pack_padded_sequence(x, lens, batch_first=True)\n",
    "print x.data.size()\n",
    "print x.batch_sizes\n",
    "\n",
    "lstm = nn.LSTM(30, 50, batch_first=True).cuda()\n",
    "h0 = Variable(torch.zeros(1, 10, 50)).cuda()\n",
    "c0 = Variable(torch.zeros(1, 10, 50)).cuda()\n",
    "\n",
    "packed_h, (packed_h_t, packed_c_t) = lstm(x, (h0, c0))\n",
    "print packed_h.data.size(), type(packed_h)\n",
    "print packed_h_t.size(), type(packed_h_t)\n",
    "print packed_c_t.size(), type(packed_c_t)\n",
    "h, _ = pad_packed_sequence(packed_h) \n",
    "print h.size() # Size 20 x 10 x 50 instead of 10 x 20 x 50，\n",
    "# 也就是说即使输入的packedsequence是用batch first的形式构造的，输出的packedsequence依旧是step first\n",
    "print type(h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PackedSequence(data=Variable containing:\n",
      " 0.2981 -0.3993  1.3740  ...   1.0439  0.5730  0.2853\n",
      " 0.5220  0.8148 -0.3350  ...   0.4287  3.6346 -0.9161\n",
      " 0.6322  0.8707 -1.6416  ...   0.1630 -0.9379 -0.8117\n",
      "          ...             ⋱             ...          \n",
      " 0.1682 -0.3832  0.7674  ...   0.6986  0.9199  0.9385\n",
      " 0.4829 -0.4175 -1.5790  ...  -0.5006 -2.2780 -1.6367\n",
      " 2.1415  1.8825  0.5418  ...   1.3033 -1.0589  1.3188\n",
      "[torch.cuda.FloatTensor of size 155x30 (GPU 0)]\n",
      ", batch_sizes=[10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1])\n"
     ]
    }
   ],
   "source": [
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(Variable containing:\n",
      "(0 ,.,.) = \n",
      "  0.2981 -0.3993  1.3740  ...   1.0439  0.5730  0.2853\n",
      "  0.5220  0.8148 -0.3350  ...   0.4287  3.6346 -0.9161\n",
      "  0.6322  0.8707 -1.6416  ...   0.1630 -0.9379 -0.8117\n",
      "           ...             ⋱             ...          \n",
      "  0.4115 -1.4341  1.0239  ...   1.5278 -0.8339 -0.3697\n",
      "  0.4981  1.2309  1.2682  ...   0.3588  0.0668  0.5734\n",
      "  0.0620 -0.4905  0.7230  ...   1.1195 -1.1458 -0.4309\n",
      "\n",
      "(1 ,.,.) = \n",
      " -0.4845  0.8116  1.1948  ...   1.5190 -0.2912 -0.2472\n",
      " -0.4461  2.9652  0.6847  ...  -1.5527  0.5092  0.6569\n",
      " -1.3457  1.3938  0.7274  ...   1.3539 -0.6283  0.5273\n",
      "           ...             ⋱             ...          \n",
      " -1.6433  0.6367  0.1309  ...  -0.7696 -1.0221 -1.7932\n",
      " -0.6540 -0.7953 -2.2092  ...  -0.2899 -1.3269 -1.2289\n",
      "  0.4172  1.0223 -1.2531  ...   1.1317  0.9870 -0.8640\n",
      "\n",
      "(2 ,.,.) = \n",
      " -0.9000  0.2971  0.2086  ...  -1.0964 -1.9530  0.5754\n",
      " -2.0922  1.2616  0.3765  ...   0.1584  1.0004 -0.9046\n",
      "  0.3324  0.0206  0.5188  ...   0.0596  1.8765 -1.8499\n",
      "           ...             ⋱             ...          \n",
      " -0.1248  0.9088  0.9564  ...  -0.3210 -0.0383  0.4644\n",
      " -0.0511 -0.7683  2.2531  ...  -0.9666 -0.8369  0.3648\n",
      "  0.8526 -0.4870 -0.2440  ...   0.0259 -1.0961  0.2565\n",
      "...\n",
      "\n",
      "(17,.,.) = \n",
      "  1.5964 -0.7572  0.2053  ...  -1.1004  0.3298 -1.3271\n",
      " -0.4084  1.0633  0.0553  ...   0.9313  2.0408 -0.9041\n",
      " -0.2775  0.8064  2.5784  ...   0.4241  1.1517  0.6563\n",
      "           ...             ⋱             ...          \n",
      "  0.0000  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
      "  0.0000  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
      "  0.0000  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
      "\n",
      "(18,.,.) = \n",
      "  0.1682 -0.3832  0.7674  ...   0.6986  0.9199  0.9385\n",
      "  0.4829 -0.4175 -1.5790  ...  -0.5006 -2.2780 -1.6367\n",
      "  0.0000  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
      "           ...             ⋱             ...          \n",
      "  0.0000  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
      "  0.0000  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
      "  0.0000  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
      "\n",
      "(19,.,.) = \n",
      "  2.1415  1.8825  0.5418  ...   1.3033 -1.0589  1.3188\n",
      "  0.0000  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
      "  0.0000  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
      "           ...             ⋱             ...          \n",
      "  0.0000  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
      "  0.0000  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
      "  0.0000  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
      "[torch.cuda.FloatTensor of size 20x10x30 (GPU 0)]\n",
      ", [20, 19, 18, 17, 16, 15, 14, 13, 12, 11])\n"
     ]
    }
   ],
   "source": [
    "t = pad_packed_sequence(x)\n",
    "print(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "从上面的程序上看，pack_padded_sequence会把sequence的steps和batch size根据lengths合并，从(steps, batch size, [other dim]) 变成 (sum(lengths), [other dim])，看看源码怎么写的："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n",
      "[2, 1]\n",
      "3\n"
     ]
    }
   ],
   "source": [
    "a = [1,2,3]\n",
    "b = reversed(a)\n",
    "print(next(b))\n",
    "print(list(b))\n",
    "print(a[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pack_padded_sequence(input, lengths, batch_first=False):\n",
    "    \"\"\"Packs a Variable containing padded sequences of variable length.\n",
    "\n",
    "    Input can be of size ``TxBx*`` where T is the length of the longest sequence\n",
    "    (equal to ``lengths[0]``), B is the batch size, and * is any number of\n",
    "    dimensions (including 0). If ``batch_first`` is True ``BxTx*`` inputs are expected.\n",
    "\n",
    "    The sequences should be sorted by length in a decreasing order, i.e.\n",
    "    ``input[:,0]`` should be the longest sequence, and ``input[:,B-1]`` the\n",
    "    shortest one.\n",
    "\n",
    "    Note:\n",
    "        This function accept any input that has at least two dimensions. You\n",
    "        can apply it to pack the labels, and use the output of the RNN with\n",
    "        them to compute the loss directly. A Variable can be retrieved from\n",
    "        a :class:`PackedSequence` object by accessing its ``.data`` attribute.\n",
    "\n",
    "    Arguments:\n",
    "        input (Variable): padded batch of variable length sequences.\n",
    "        lengths (list[int]): list of sequences lengths of each batch element.\n",
    "        batch_first (bool, optional): if True, the input is expected in BxTx*\n",
    "            format.\n",
    "\n",
    "    Returns:\n",
    "        a :class:`PackedSequence` object\n",
    "    \"\"\"\n",
    "    if batch_first:\n",
    "        input = input.transpose(0, 1)\n",
    "\n",
    "    steps = []\n",
    "    batch_sizes = []\n",
    "    lengths_iter = reversed(lengths)\n",
    "    # current_length 的初始值是序列中的最小长度\n",
    "    current_length = next(lengths_iter)\n",
    "    batch_size = input.size(1)\n",
    "    if len(lengths) != batch_size:\n",
    "        raise ValueError(\"lengths array has incorrect size\")\n",
    "\n",
    "    for step, step_value in enumerate(input, 1):\n",
    "        steps.append(step_value[:batch_size])\n",
    "        batch_sizes.append(batch_size)\n",
    "\n",
    "        # 当step等于current_length时，意味着有些数据已经读完了\n",
    "        # while循环的内部用来判定有多少个数据已经读完\n",
    "        # 并且相应地调整下一个step中要处理的batch 的大小\n",
    "        while step == current_length:\n",
    "            try:\n",
    "                new_length = next(lengths_iter)\n",
    "            except StopIteration:\n",
    "                current_length = None\n",
    "                break\n",
    "\n",
    "            if current_length > new_length:  # remember that new_length is the preceding length in the array\n",
    "                raise ValueError(\"lengths array has to be sorted in decreasing order\")\n",
    "            batch_size -= 1\n",
    "            current_length = new_length\n",
    "        if current_length is None:\n",
    "            break\n",
    "    return PackedSequence(torch.cat(steps), batch_sizes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pad_packed_sequence(sequence, batch_first=False):\n",
    "    \"\"\"Pads a packed batch of variable length sequences.\n",
    "\n",
    "    It is an inverse operation to :func:`pack_padded_sequence`.\n",
    "\n",
    "    The returned Variable's data will be of size TxBx*, where T is the length\n",
    "    of the longest sequence and B is the batch size. If ``batch_size`` is True,\n",
    "    the data will be transposed into BxTx* format.\n",
    "\n",
    "    Batch elements will be ordered decreasingly by their length.\n",
    "\n",
    "    Arguments:\n",
    "        sequence (PackedSequence): batch to pad\n",
    "        batch_first (bool, optional): if True, the output will be in BxTx* format.\n",
    "\n",
    "    Returns:\n",
    "        Tuple of Variable containing the padded sequence, and a list of lengths\n",
    "        of each sequence in the batch.\n",
    "    \"\"\"\n",
    "    var_data, batch_sizes = sequence\n",
    "    max_batch_size = batch_sizes[0]\n",
    "    output = var_data.data.new(len(batch_sizes), max_batch_size, *var_data.size()[1:]).zero_()\n",
    "    output = Variable(output)\n",
    "\n",
    "    lengths = []\n",
    "    data_offset = 0\n",
    "    prev_batch_size = batch_sizes[0]\n",
    "    for i, batch_size in enumerate(batch_sizes):\n",
    "        output[i, :batch_size] = var_data[data_offset:data_offset + batch_size]\n",
    "        data_offset += batch_size\n",
    "\n",
    "        dec = prev_batch_size - batch_size\n",
    "        if dec > 0:\n",
    "            lengths.extend((i,) * dec)\n",
    "        prev_batch_size = batch_size\n",
    "    lengths.extend((i + 1,) * batch_size)\n",
    "    lengths.reverse()\n",
    "\n",
    "    if batch_first:\n",
    "        output = output.transpose(0, 1)\n",
    "    return output, lengths"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "pack_padded_sequence的处理是按照time step展开的，pad_packed_sequence的处理是按照batch size（由pack_padded_sequence得到）展开的。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "最后还有一个问题， 就是RNN是如何处理packedsequence作为输入的情况呢？追踪一下源码，可以发现是用的VariableRecurrent，源码如下："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def VariableRecurrent(batch_sizes, inner):\n",
    "    def forward(input, hidden, weight):\n",
    "        output = []\n",
    "        input_offset = 0\n",
    "        last_batch_size = batch_sizes[0]\n",
    "        hiddens = []\n",
    "        flat_hidden = not isinstance(hidden, tuple)\n",
    "        if flat_hidden:\n",
    "            hidden = (hidden,)\n",
    "        for batch_size in batch_sizes:\n",
    "            step_input = input[input_offset:input_offset + batch_size]\n",
    "            input_offset += batch_size\n",
    "\n",
    "            dec = last_batch_size - batch_size\n",
    "            if dec > 0:\n",
    "                hiddens.append(tuple(h[-dec:] for h in hidden))\n",
    "                hidden = tuple(h[:-dec] for h in hidden)\n",
    "            last_batch_size = batch_size\n",
    "\n",
    "            if flat_hidden:\n",
    "                hidden = (inner(step_input, hidden[0], *weight),)\n",
    "            else:\n",
    "                hidden = inner(step_input, hidden, *weight)\n",
    "\n",
    "            output.append(hidden[0])\n",
    "        hiddens.append(hidden)\n",
    "        hiddens.reverse()\n",
    "\n",
    "        hidden = tuple(torch.cat(h, 0) for h in zip(*hiddens))\n",
    "        assert hidden[0].size(0) == batch_sizes[0]\n",
    "        if flat_hidden:\n",
    "            hidden = hidden[0]\n",
    "        output = torch.cat(output, 0)\n",
    "\n",
    "        return hidden, output\n",
    "\n",
    "    return forward"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "可以看出也是按照step把输入展开，即把packed sequence 恢复成mini batch，只是batch中的不同sequence的长度不同，也就是说不同的step中的输入是不同的，这个时候就需要对hidden进行裁剪和保存。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}