misc/lstm-mnist-py.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline\n",
    "figsize(10,5)\n",
    "matplotlib.rcParams[\"image.interpolation\"] = \"none\"\n",
    "matplotlib.rcParams[\"image.cmap\"] = \"afmhot\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import clstm\n",
    "import h5py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LSTM + CTC Training on MNIST"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's start by getting the MNIST dataset. This version of MNIST in HDF5 represents images in a sequence format suitable for training with `clstm` command line models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100 49.5M  100 49.5M    0     0  34.2M      0  0:00:01  0:00:01 --:--:-- 34.2M\n"
     ]
    }
   ],
   "source": [
    "!test -f mnist_seq.h5 || curl http://www.tmbdev.net/ocrdata-hdf5/mnist_seq.h5 > mnist_seq.h5 || rm -f mnist_seq.h5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In HDF5 data files for CLSTM, row `t` represents the input vector at time step `t`. For MNIST, we scan through the original image left-to-right over time.\n",
    "\n",
    "Image storage in HDF5 would have to be a rank 3 doubly ragged array, but HDF5 supports only rank 2 arrays. We therefore store image dimensions in a separate array."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000,)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATMAAAEyCAYAAACWBDf7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADoFJREFUeJzt3X+MXXWZx/H3bUuLSkF0s6TGLmxKoW42KgIpIlI2+ANk\nC02jGNdfbVDChkRcYBtTK43gZv/AVpddwF0FK43SbEmpGkOt0dk2UCxL2khcqLR2EcEuuE2gtolA\nO9c/zmlmGGY6z5x7zty5z7xfyYRzex/Oec6cmc+ce+73ew9IkiRJkiRJkiRJkiSpIa0G191ucN2S\nJq8twEVD/3FKxZVNAb4BbAP6gDmV25KkuI3AguGeqBpmi4DpwPnAF4BVFdcjSbWoGmbvATaVy9uB\nc+ppR5KqqRpmJwIHBj0+0sG6JKljVQPoADBzyHr6O29Hko5p3khPVA2zh4APlcvnAY9VXI8kjcWu\nkZ6YVnGF9wPvpwg1gKUV1yNJtagaZm3g7+tsRJI64UV7SSkYZpJSMMwkpWCYSUrBMJOUgmEmKQXD\nTFIKhpmkFAwzSSkYZpJSMMwkpWCYSUrBMJOUgmEmKQXDTFIKhpmkFAwzSSkYZpJSMMwkpWCYSUrB\nMJOUgmEmKQXDTFIKhpmkFAwzSSkYZpJSMMwkpWCYSUrBMJOUgmEmKQXDTFIK07rdgKTx8blWrO5r\nq98Yqpv6Dy900E39PDOTlIJhJikFw0xSCoaZpBQMM0kpGGaSUjDMJKVgmElKwTCTlEJwTHAl7QbX\nLanUCv4WH3n8Y7HCM9eEyqZMmRFbX702AosYJrs6mc60A3ixXN4LXNXBuiSpI1XD7Pjyv39TVyOS\n1Imq18zeAbwe+DHwU2B+bR1JUgVVz8wOAbcCdwFzgQeAM4D+mvqSpDGpGmZPAnvK5d3AfmAW8Gwd\nTUnSCOaN9ETVl5lLgVXl8luAE4F9FdclSVG7Rnqi6pnZXcC3ga3l46X4ElNSF1UNs8PAJ+tsRJI6\n4QwASSl4DwCpx50enMjTmndtqK7dbnJiUHM8M5OUgmEmKQXDTFIKhpmkFAwzSSkYZpJSMMwkpWCY\nSUrBMJOUgjMAxtGSYN2Dwbo9o5eoh0U/2/9Xv/xIcI3Rz1Dtzdt3eGYmKQXDTFIKhpmkFAwzSSkY\nZpJSMMwkpWCYSUrBMJOUgoNma/De4OjGu4/sjq2w/atQ2XfmXBaqW/pUbLOaWN4e/Tjsv1pS85Y3\n1Ly+8eGZmaQUDDNJKRhmklIwzCSlYJhJSsEwk5SCYSYpBcNMUgqGmaQUnAFQgxOCda3WqcHCvwiV\nffrOd4fqvnTpw6G6Z0JV6lTw07D5cPi38/3BuuDHYX/9muiGJxTPzCSlYJhJSsEwk5SCYSYpBcNM\nUgqGmaQUDDNJKRhmklIwzCSl4AyAY4iO1L7hgpo3HByo/cS3YyP7f9dBK6rfO4P3jFjx0n823MkI\nXvpjd7bboeiZ2Xygr1w+HXgQ2ArcQfx3XpIaEwmzZcA3gRnl49XAcuBCiiC7opnWJCkuEmZ7gMUM\nnIG9i+KsDOAB4H0N9CVJYxIJsw3A4UGPB7+sPAicVGtHklRBlXcz+wctzwReqKkXSRrNvJGeqBJm\nO4EF5fKlDLzklKSm7RrpibEMzTg6YOAGijcEpgOPA/dV70uS6hENs6eA88vl3cBFTTQjSVU5A0BS\nCk0OeA2OY5+4ggO16f/hubHCy7ZVb2Y47f7Ra4D2j94Tq3v66VDdrGufD9XFqvKI/jL95jOxytnf\nfLl6M8Not/8/VDdn6qxQ3f925zd8I7CIYb7dnplJSsEwk5SCYSYpBcNMUgqGmaQUDDNJKRhmklIw\nzCSlYJhJSsF7ABzD9Gjhaac12MUxtGJ/i1p/G5t5EB3Bvu+zN4fqLpn+lVDdT4LbnegWBqeMzP6P\nzTVvOTgU/6tzQ2VdGtnfMc/MJKVgmElKwTCTlIJhJikFw0xSCoaZpBQMM0kpGGaSUjDMJKXgPQCO\nYW5wRPeT/S8F11jzt7u9JlT2o/OuDtVdtu1Lse1OXREqa/fvC9WtnXtqqO7Te0NltYveC+LIvW+L\nre+jj3XQzWu127eG6mZNXR6qe25i/+Z6DwBJuRlmklIwzCSlYJhJSsEwk5SCYSYpBcNMUgqGmaQU\nDDNJKTgD4BjiMwBernW77fZtobpZU28M1T3fjh2K44I/Df88M/Y38PoXt4Tq2v1vDdX93dS/DNWt\nC1XFfSL4c7C2/5HgGt8ZKwset+/MmRGqW9KrH+7/as4AkJSbYSYpBcNMUgqGmaQUDDNJKRhmklIw\nzCSlYJhJSsEwk5TCtG43MLkER2DvfypU9lxwhHjUy8HV/eMf+kN1129ZGaprLdgUqvvugRtCdT8/\ncVWo7jfBGQ/33BL9NQmO7A9q86+huutzjOzvWPTMbD7QVy6fBTxTPu4DrmygL0kak8ifnGXAJ4CD\n5eOzgdXllyRNCJEzsz3AYgYmdp4NXAZsAb4FnNBMa5IUFwmzDcDhQY+3AzcCC4C9QOzCiCQ1qMq7\nmfcDO8vljRTX0CRpPMwb6YkqYbYJOLdcvhh4tEpHklTBrpGeGMvQjKPv/14D3A68AuwDrq7elyTV\nIxpmTwHnl8u/AC5opBtJqsgZAJJSmLT3AIjs+MsrYt+eabfUfQ+AP4TqPjrlTaG69Z00M4zoD80T\nH4tVnvm94PevHZt5sPmC2GfiX/JwbLP9+6+LFZ781VhdcObG2rmx/fjUryf0r1rdvAeApNwMM0kp\nGGaSUjDMJKVgmElKwTCTlIJhJikFw0xSCoaZpBQm7z0AAoPTp/3ZG2ve6Cuhqlbr4OhFwLofnhOq\nW7+w3g82iY43/+t1scrn/zw20v3kr78UqvvAg38M1R0hOPOgdXysLqjNv4fqrppcI/s75pmZpBQM\nM0kpGGaSUjDMJKVgmElKwTCTlIJhJikFw0xSCoaZpBQm7wyAiMsvj9W1Yx8m/5P3LgjVfXBbbLMT\n/DYLHA6294HbYp/t/98XxmY8sDg246HF62Lri2rHZh7cfernQnWx+SI6yjMzSSkYZpJSMMwkpWCY\nSUrBMJOUgmEmKQXDTFIKhpmkFAwzSSlM2hkA7cDo9Pmn3xNa1/b/iX2W/P89FxsSH+ktk0eD+/vu\njzwWqnv44IrYCl8XrOO+UFV7/45Q3Wd/O8kO8DjxzExSCoaZpBQMM0kpGGaSUjDMJKVgmElKwTCT\nlIJhJikFw0xSCq0G1z2hhzm3Ant+5JGFsXWdHRshvvaMGaG6T+0JlWkEkWMLsDL44/9Pwe0eDv7I\nT7YZHjXbCCximOwabTrTccDdwKnADOArwBPAGqAf+CVwLRM8uCTlN9rLzI8DvwcuBC4BbgdWAcvL\nf2sBVzTZoCRFjBZm64GbBtW+ArwL2Fr+2wPA+5ppTZLiRguzQ8BBYCZFsK0Y8v8cBE5qpjVJiou8\nmzkb+BlwD3AvxbWyo2YCLzTQlyQNZ95IT4wWZqcAm4FlFBf9AXYCR2/NfSkDLzklqWm7RnpitHcz\nl1O8jLyJgWtn1wG3AdOBx4l+cp0kNWi0MLuu/BrqovpbkaTqnAEgKYVJew+AyOSH1jnfD62p3X42\nVLfDkf3jIjrC/svREfsd9KLx45mZpBQMM0kpGGaSUjDMJKVgmElKwTCTlIJhJikFw0xSCoaZpBQm\n8QyAiBeDdWtDVXurN6IGOLI/F8/MJKVgmElKwTCTlIJhJikFw0xSCoaZpBQMM0kpGGaSUjDMJKUw\neWcABD4ovr3yTaFV3XxLbJM/iJVJqsAzM0kpGGaSUjDMJKVgmElKwTCTlIJhJikFw0xSCoaZpBQM\nM0kpTNoZAJHPf596s58SL/UKz8wkpWCYSUrBMJOUgmEmKQXDTFIKhpmkFAwzSSkYZpJSMMwkpTBp\nZwBEOP5f6h2jnZkdB6wFtgLbgYXAWcCzQF/5dWWTDUpSxGhnZh8Hfg98EjgZ+AXwZWAVsLrZ1iQp\nbrQwWw/cVy5PAV4BzgbOBK4AdgOfBw421aAkRYz2MvMQRVDNpAi2LwKPADcCC4C9wMomG5SkiMgb\nALOBDcDtwDrgJODF8rmNwG3NtCZJrzFvpCdGOzM7BdgMLAPWlP+2CTi3XL4YeLTD5iQpatdIT4x2\nZrac4kzspvILimtkX6O4frYPuLqGBiWpI60G1+0wLUl12wgsYpjscgaApBQMM0kpGGaSUjDMJKVg\nmElKwTCTlIJhJikFw0xSCoaZpBQMM0kpGGaSUjDMJKVgmElKwTCTlIJhJikFw0xSCoaZpBQMM0kp\nGGaSUjDMJKUQuW9mJ3YMeTyL4o5OvSzDPoD7MdG4HzF7R3qiybsz/RfFXc8lqU5bgIu63YQkSZIk\nSR2aAnwD2Ab0AXO6205HdlDsQx9wV5d7qWI+Re8ApwMPAluBO2j2+mmdBu/DWcAzDByTK7vV1Bgd\nB6yl+N5vBxbSm8djuP04C3iW3jsmIYuBu8vl+RS3V+9Fx/Pad2d7yTLgMYo/KgA/AC4sl++kuOX9\nRDd0Hz4DXN+9dipbAqwul08Gnga+T+8djyW8dj+uojePScgqXp3Oz3SrkQ7NB3YBPwZ+Wj7uJYsp\n/vo/XD4efBwuB/5t3Dsau6H7cCfFsdgCfAs4oUt9jdUbGOj1zcCvgd8Oer5Xjsdw+3EHXTom4zFo\n9kTgwKDHR8Zpu3U7BNwKfBC4BvguvbUfG4DDgx4PfhlzEDhpfNupZOg+bAdupBgCtBdY2Y2mKjhE\n8T2fCawHVvDqn6VeOR5D9+OLwCN06ZiMxy/jAYqdHbzN/nHYbt2epAgwgN3AfooBgr1q8DGYCbzQ\nrUY6cD+ws1zeSHG9plfMBn4G3APcS+8ej8H7sY4uHpPxCLOHgA+Vy+dRXPPoRUspXjIDvIXijLOX\nR2zvZGBQ86UUF3F7zSbg3HL5YuDRLvYyFqcAmymuAa4p/60Xj8dw+9GrxySkRXFt46Hy64zutlPZ\nNAbeudlKEcy95jQGLp7PpZilsY3i2kYvvHsGr96Hd1C8A9gHfI/euWb2L8DvGHjHrw94O713PIbb\nj/n05jGRJEmSJEmSJEmSJEmSJEmSJr4/AdiuNDG6MkYWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7efc8d7fe210>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "h5 = h5py.File(\"mnist_seq.h5\",\"r\")\n",
    "imshow(h5[\"images\"][0].reshape(*h5[\"images_dims\"][0]))\n",
    "print h5[\"images\"].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's use a bidirectional LSTM and a fiarly high learning rate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".stacked: 0.010000 0.900000 28 11\n",
      ".stacked.parallel: 0.010000 0.900000 28 20\n",
      ".stacked.parallel.lstm: 0.010000 0.900000 28 10\n",
      ".stacked.parallel.reversed: 0.010000 0.900000 28 10\n",
      ".stacked.parallel.reversed.lstm: 0.010000 0.900000 28 10\n",
      ".stacked.softmax: 0.010000 0.900000 20 11\n",
      "\n"
     ]
    }
   ],
   "source": [
    "net = clstm.make_net_init(\"bidi\",\"ninput=28:nhidden=10:noutput=11\")\n",
    "net.setLearningRate(1e-2,0.9)\n",
    "print clstm.network_info(net)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The class labels in the dataset are such that digit `0` has been assigned class `10`, since class 0 is reserved for epsilon states in CTC alignment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\\x00', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0']\n"
     ]
    }
   ],
   "source": [
    "print [chr(c) for c in h5[\"codec\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7efc76630850>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATMAAAEyCAYAAACWBDf7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADilJREFUeJzt3W+MVfWdx/H3KNLGZRyJD5BG3dVSJBol1CXT4AZY7bJS\nwyoTYzRdkhJN043EbnaVbGijWbP+SRA24kohW1lT0qVGoyyaaNEyOqk006BI07BYXdK4svOgQUEH\ngwFm9sG5dMbpTO93zr137twv71cy8dw5H8/8Dmf4cO75cw9IkiRJkiRJkiRJkiSpQdoauOzBBi5b\n0pnrdWDxyG+eVXJhZwGbgN1AN/Dl0sOSpLjtwKLRZpQts5uBqcAC4J+AdSWXI0l1UbbMrgVerkz3\nAn9en+FIUjlly+w84ONhr0/VsCxJqlnZAvoYaB+xnIHahyNJf9ScsWaULbM3gG9Upr8G/KrkciRp\nPA6MNWNKyQU+D/wVRakBrCy5HEmqi7JlNgj8XT0HIkm18KC9pBQsM0kpWGaSUrDMJKVgmUlKwTKT\nlIJlJikFy0xSCpaZpBQsM0kpWGaSUrDMJKVgmUlKwTKTlIJlJikFy0xSCpaZpBQsM0kpWGaSUrDM\nJKVgmUlKwTKTlIJlJikFy0xSCpaZpBQsM0kpWGaSUrDMJKVgmUlKwTKTlIJlJikFy0xSCpaZpBQs\nM0kpWGaSUrDMJKVgmUlKwTKTlIJlJikFy0xSCpaZpBSmNHsAaj1nn312KNfR0dHgkYxu1apVody5\n554bys2ePbuuP/fRRx8N5W677bZQ7vjx46HcI488Eso98MADodxkU0uZvQUcrUwfBO6ofTiSVE7Z\nMvti5b9/Wa+BSFItyh4zmwucC/wU+BnQWbcRSVIJZffMjgFrgSeBrwAvAbOBgTqNS5LGpWyZ/QZ4\nrzL9LnAYmAkcqsegJGkMc8aaUfZt5kpgXWX6S8B5QF/JZUlS1IGxZpTdM3sS+A+gp/J6Jb7FlNRE\nZcvsJLCingORpFp4B4CkFLwDYBK65JJLQrmpU6eGcgsWLAjlrr322lBu+vTpoVxXV1coV29tbW2h\n3ODgYCh36FDsvNaGDRtCueXLl4dy/f39ody+fftCuZ6enuqhFuaemaQULDNJKVhmklKwzCSlYJlJ\nSsEyk5SCZSYpBctMUgqWmaQUYpdKlxO7vPoMMm/evFDu1VdfDeXq/Rn70Svnszh16lQod8cdsU+E\nj16xH/1z7uuLfRDNhx9+GMq98847odwktx24mVG6yz0zSSlYZpJSsMwkpWCZSUrBMpOUgmUmKQXL\nTFIKlpmkFPzY7An0/vvvh3LRiyDPP//8WoYzafT29oZyR48eDeUWL14cyn322Weh3NatW0M5NZd7\nZpJSsMwkpWCZSUrBMpOUgmUmKQXLTFIKlpmkFCwzSSlYZpJS8A6ACXT48OFQbvXq1aHcjTfeGMq9\n/fbbodxjjz0WykXt3bs3lFuyZEkod+zYsVDuyiuvDOXuvvvuUE6twT0zSSlYZpJSsMwkpWCZSUrB\nMpOUgmUmKQXLTFIKlpmkFCwzSSm0NXDZgw1ctoCOjo5Q7pNPPgnlNm/eHMqtXLkylFuxYkUot23b\ntlCu3traYr/+g4P+Kk8i24GbGaW7ontmnUB3ZXoW8HOgB9g42kIlaaJFymw18O/AFyqv1wNrgIUU\nRXZTY4YmSXGRMnsP6GJoD+yrFHtlAC8BX2/AuCRpXCJl9hxwctjr4W8r+4HYgRtJaqAyZzMHhk23\nA0fqNBZJqmbOWDPKlNleYFFleilDbzklqdEOjDVjPB/OePr89D9SnBCYCuwHni0/Lkmqj2iZ/RZY\nUJl+F1jciMFIUlneASApBZ8B0MKOHj0aykWvdK/38u68885Q7umnnw7lBgYGqofGwSv7c3HPTFIK\nlpmkFCwzSSlYZpJSsMwkpWCZSUrBMpOUgmUmKQXLTFIKPgNAvzdt2rRQbseOHaHcwoULQ7mlS5eG\ncq+88koop9RqfgaAJE1qlpmkFCwzSSlYZpJSsMwkpWCZSUrBMpOUgmUmKQXLTFIK3gGgcZs1a1Yo\n9+abb4ZyH330USjX3d0dyu3ZsyeU27hxYyjnswImFe8AkJSbZSYpBctMUgqWmaQULDNJKVhmklKw\nzCSlYJlJSsEyk5SCdwCoYbq6ukK5LVu2hHLt7e2hXPSK/TVr1oRyW7duDeX6+vpCOdXEOwAk5WaZ\nSUrBMpOUgmUmKQXLTFIKlpmkFCwzSSlYZpJSsMwkpeAdAGq6q666KpRbv359KHfdddeFctE7BTZv\n3hzKPfTQQ6HcoUOHQjmNquY7ADqB00+TmAd8UHndDdxahwFKUk2mBDKrgb8F+iuvrwHWV74kaVKI\n7Jm9B3QxtFt3DXAj8DrwQ2BaY4YmSXGRMnsOODnsdS9wD7AIOAjc34BxSdK4lDmb+TywtzK9neIY\nmiRNhDljzShTZi8D8yvT1wOxx0dLUu0OjDUjcgLgtNPnsb8DPAGcAPqAb5cflyTVR7TMfgssqEzv\nA/6iIaORpJK8A0BSCt4BoJYxffr0UG7ZsmWhXPTZA1G7du0K5ZYsWVLXn3uG8RkAknKzzCSlYJlJ\nSsEyk5SCZSYpBctMUgqWmaQULDNJKVhmklLwDgCl09YW+7U+fvx4KDdlSuwW5hMnToRyN9xwQyj3\n2muvhXJnGO8AkJSbZSYpBctMUgqWmaQULDNJKVhmklKwzCSlYJlJSsEyk5TCeB41JzXE1VdfHcrd\ncsstodz8+fOrh4hf2R+1f//+UK6np6euP1cF98wkpWCZSUrBMpOUgmUmKQXLTFIKlpmkFCwzSSlY\nZpJSsMwkpeAdABq3yy+/PJRbtWpVKLd8+fJQbubMmaFcvZ06dSqU6+vrC+UGBgZqGY7G4J6ZpBQs\nM0kpWGaSUrDMJKVgmUlKwTKTlIJlJikFy0xSCpaZpBS8A+AMcOGFF4Zyt99+eyh31113hXKXXnpp\nKNcse/bsCeUefPDBUO6FF16oZTiqUbUyOwfYAvwp8AXgX4D/Bp4CBoBfA3cBg40boiRVV+1t5jeB\n3wELgRuAJ4B1wJrK99qAmxo5QEmKqFZmzwD3DcueAL4KnH5W1kvA1xszNEmKq1Zmx4B+oJ2i2L4/\n4v/pBzoaMzRJiouczbwY2AX8CNhGcazstHbgSAPGJUmjmTPWjGplNgPYCaymOOgPsBdYVJleytBb\nTklqtANjzah2NnMNxdvI+xg6dvZdYAMwFdgPPFuHAUpSTaqV2XcrXyMtrv9QJKk87wCQlIJ3AExC\nM2bMCOWuuOKKUO7xxx8P5ebMGfPY6ue0tbWFcvXW29sbyq1duzaU27FjRygXfQaAmss9M0kpWGaS\nUrDMJKVgmUlKwTKTlIJlJikFy0xSCpaZpBQsM0kpeAdAHVxwwQWh3KZNm0K5uXPnhnKXXXZZKNes\nK/Z3794dyq1bty6U27lzZyj36aefhnLKxT0zSSlYZpJSsMwkpWCZSUrBMpOUgmUmKQXLTFIKlpmk\nFCwzSSmcsXcAdHZ2Vs3ce++9oWXNnz8/lLvoootCuWaJXjm/YcOGUO7hhx8O5fr7+0M56Y9xz0xS\nCpaZpBQsM0kpWGaSUrDMJKVgmUlKwTKTlIJlJikFy0xSCunuAIh+3v3y5cvrkmmE/fv3h3Ivvvhi\nKHfy5MlQbv369aHckSNHQrnBwcFQTqoH98wkpWCZSUrBMpOUgmUmKQXLTFIKlpmkFCwzSSlYZpJS\nsMwkpRC7XL6cSX35d/ROgTOJV+yrBWwHbmaU7qq2Z3YOsBXoAXqBZcA84BDQXfm6tZ4jlaQyqt2b\n+U3gd8AKYDqwD/hnYB0Qu5FPkiZAtTJ7Bni2Mn0WcAK4BrgcuAl4F/h7wGeFSWqqam8zj1EUVTtF\nsX0P+CVwD7AIOAjc38gBSlJE5COALgaeA54AfgJ0AEcr87YDsSfCSlLt5ow1o9qe2QxgJ7AaeKry\nvZeB04/wvh7YU+PgJCnqwFgzqu2ZraHYE7uv8gXFMbJ/pTh+1gd8uw4DlKSaeJ2Zfs/rzNQCSl9n\nJkktId0zAKLcC5Fycc9MUgqWmaQULDNJKVhmklKwzCSlYJlJSsEyk5SCZSYpBctMUgqWmaQULDNJ\nKVhmklKwzCSlYJlJSsEyk5SCZSYpBctMUgqWmaQUGv2x2W+NeD2T4olOrSzDOoDrMdm4HjEHx5rR\nyEcUvUbx1HNJqqfXgcXNHoQkSZIkSTU6C9gE7Aa6gS83dzg1eYtiHbqBJ5s8ljI6KcYOMAv4OdAD\nbKSxx0/rafg6zAM+YGib3NqsQY3TOcBWij/7XmAZrbk9RluPecAhWm+bhHQBWyrTnRSPV29FX+QP\nz862ktXAryj+UQHYASysTP+A4pH3k93IdbgT+IfmDae0bwHrK9PTgfeB/6L1tse3+MP1uIPW3CYh\n6/h8O3/QrIHUqBM4APwU+FnldSvpovjX/xeV18O3w98A/zbhIxq/kevwA4pt8TrwQ2Bak8Y1Xn/C\n0FgvAP4H+N9h81tle4y2Hhtp0jaZiItmzwM+Hvb61AT93Ho7BqwF/hr4DvBjWms9ngNODns9/G1M\nP9AxscMpZeQ69AL3UFwCdBC4vxmDKuEYxZ95O/AM8H0+/7vUKttj5Hp8D/glTdomE/GX8WOKlR3+\nMwcm4OfW228oCgzgXeAwxQWCrWr4NmgHjjRrIDV4Hthbmd5OcbymVVwM7AJ+BGyjdbfH8PX4CU3c\nJhNRZm8A36hMf43imEcrWknxlhngSxR7nK18xfZehi5qXkpxELfVvAzMr0xfD+xp4ljGYwawk+IY\n4FOV77Xi9hhtPVp1m4S0URzbeKPyNbu5wyltCkNnbnooirnV/BlDB8+/QnGXxm6KYxutcPYMPr8O\ncynOAHYD/0nrHDN7DPg/hs74dQNX03rbY7T16KQ1t4kkSZIkSZIkSZIkSZIkSZI0+f0/NrpH7Njj\nRJ8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7efc766e1350>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "index = 0\n",
    "xs = array(h5[\"images\"][index].reshape(28,28,1),'f')\n",
    "cls = h5[\"transcripts\"][index][0]\n",
    "print cls\n",
    "imshow(xs.reshape(28,28).T,cmap=cm.gray)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Forward propagation is quite simple: we take the input data and put it into the input sequence of the network, call the `forward` method, and take the result out of the output sequence.\n",
    "\n",
    "Note that all sequences (including `xs`) in clstm are of rank 3, with indexes giving the time step, the feature dimension, and the batch index, in order.\n",
    "\n",
    "The output from the network is a vector of posterior probabilities at each time step."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7efc76531c10>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkkAAAD2CAYAAADCgxO9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEVJJREFUeJzt3WuQnXV9B/Bvks0FciXEBGJiYlSMjPeqRLSwLanWDlTk\nRV/0CtZOrW/stJbiZTRtR9up1ZnOVLFTa2m16ohFRLGWiYZEgoIoEDrchOUihFtACLm7ZPviLGaN\n/23CyW9zDsnnM3PGZ9eHb36zzz7nfPc55/xPAgAAAAAAAAAAAAAAAAAAAPSBSRMRetrzMrLh3olI\nBgAotz7J4P7fnJCSlGRk7wcPvNOaK5M1gwfeb3j4UMf5ebv31GVt31aXlSSLVs6rDTz3vIPabc3f\nfjdr3vv6A+84a9YhDrSfKQtq8zZ+rTTu0S+uLc3bu/fg9vv7a5PzX3fg/ebPP7R59jflpBWleXde\nMVSat6J2vDz44MHt9w/XJe95zYH321N433LZZXVZSXLXY7V5U2rjsmLhwe339W3JmQdxN3TmmYc2\nz/6q7/qOPbY2b1vxY9HkyQe33999L7lg1YH3mzHj0ObZ38BAadzaaR/J6jQ60UH+GAAAji5KEgBA\nQ09L0uDyXv7rjDX4xiW9HoEx3vDcXk/AWKcu7vUEPO2kab2egLGO9IcOJYkkyeAvL+31CIyhJPUX\nJal/KEn9RUlq/zefSnJ1knVJXlA6EQBAH+imJJ2dZFqSU5NckORjpRMBAPSBbkrSG5J8c3T7miQH\n8cZYAIBnl25K0pwkW8d8/VSXOQAAfaub5Zi2Jpk95uvJSX5hibw1V+7bHlzuRdoAQH9Yf0/nliSP\n7sy4y9R2U5I2JjkrycVJViXZ1NrpYFbSBgA43E5f1rmNGrrwB+2i1E1J+kqSX0unLCXJwX3uBQDA\ns0g3JWkkyZ9UDwIA0E+84BoAoEFJAgBoUJIAABqUJACABiUJAKBh0gTljtx9bl3YtOJPfZ49py5r\n1pJ5dWFJsmRpbd6tt5TGPbJ5uDTvqo21v4InnzxSmnfCotK43H57bd7QUG3eAw/U5j3/+bV5d91V\nm7fgObV5Uwr/7Fy1qi4rSZafMe56eX1h0rZtpXk/vf/h0rzh2ru+DHTz3vL/x9RZtQ+UOx/fU5pX\nfHizY0dp3NrlF2V1Gp3IlSQAgAYlCQCgQUkCAGhQkgAAGpQkAIAGJQkAoEFJAgBoUJIAABqUJACA\nBiUJAKBBSQIAaFCSAAAalCQAgAYlCQCgQUkCAGhQkgAAGpQkAIAGJQkAoEFJAgBoGJio4GWnLasL\nG5hSl5Ukx86sy5o/vy4rSXZsr80rnm/ejodr8+aOlOZtWF8al8HB2rzHHqvNu/TW2ryVtXFZtao2\n762X/1Ft4M21P8Cn1n+nLGvK/LllWUmSpUtr82ZMr80bGiqNm7p7d23eCYtK87LgObV5xx5TGnfM\n1q19nZddu2rzLrq7+W1XkgAAGpQkAIAGJQkAoEFJAgBoUJIAABqUJACAhm5K0tQkn02yIck1Sc4q\nnQgAoA90s07S7yR5JMnvJTkuyQ1JvlY5FABAr3VTki5O8uXR7clJhuvGAQDoD92UpKeXhJ6dTmF6\nf2unNZc+/rPtwZUzMrhyRhf/FABArStv2Zkrb+2s2r1l294V4+3X7ceSLE1ySZJPJPlia4c1Z8/r\nMhoAYOIMvuSYDL7kZx/lMvSJbz3ZLErdlKRFSa5I8q4k67obDwCgv3Xz7rb3JZmb5IPplKR1STyX\nBgAcUbq5kvTu0RsAwBHLYpIAAA1KEgBAg5IEANCgJAEANChJAAANkyYod2Tkyy+boOgCA1PqsmYc\nc+B9nomVK2vzlq2qzRs5uTYvry3OqzatOO+HpWkjO79UmjfpmGWleblxbW3eTTfW5q18SW3e3Ll1\nWZdfXpeV5LZrnyjNqzbQ7dLG41iypDZv+uL5tYFz5tTmbd1am1d9QKofKysfx5O1k35/aHUanciV\nJACABiUJAKBBSQIAaFCSAAAalCQAgAYlCQCgQUkCAGhQkgAAGpQkAIAGJQkAoEFJAgBoUJIAABqU\nJACABiUJAKBBSQIAaFCSAAAalCQAgAYlCQCgQUkCAGgYmLDkaVMLs6bXZSXJrJl1WSecWJeVJPPm\n1eb94Ou1eVsuqs27+ZbSuEfufKI0b0bxr97QUG3ef311pDRv/pxJpXkvfWntfKvfUnyXNbn478SF\ni+qy3v72uqwkL/7TpaV5ybG1cU/cWpv36KO1eZvvL87bXJu3pPj47t1bm7dnd23e8HBt3jhcSQIA\naFCSAAAalCQAgAYlCQCgQUkCAGg4lJK0MMmPk5xUNAsAQN/otiRNTfLPSbYXzgIA0De6LUkfTXJh\nkgcKZwEA6BvdlKRzkzyS5IrRr2tXnwMA6APdLF97XpKRJKuTvDLJvyd5a5KHxu605vP7VhMdfNns\nDL5sdvdTAgAUufKmJ3Pl/25Lkmx58qkV4+3XTUk6fcz2uiR/nP0KUpKs+e3FXUQDAEys/S7eDH3i\nG1uaRckSAAAADYf6aZG/UjIFAECfcSUJAKBBSQIAaFCSAAAalCQAgAYlCQCgQUkCAGiYqI8UGdl0\nTl3YwKEuVLCfGTPqsuYdV5eVJMctm1sbuKP2M4jvuWO4NG/TptK4HDe/Nm/7ttq8mbNq895weu3J\nsXF97fE9vvh4fOHi2rxpxX8m/tJr6rJOPrkuK0met7T27n77ztK4zHpR8QLEe3aXxu28d0tp3p1D\npXGZPq02b/ee2ryRvbV5e2vz1r7y0qxOoxO5kgQA0KAkAQA0KEkAAA1KEgBAg5IEANCgJAEANChJ\nAAANShIAQIOSBADQoCQBADQoSQAADUoSAECDkgQA0KAkAQA0KEkAAA1KEgBAg5IEANCgJAEANChJ\nAAANkyYod2T4A3Vhe/bUZSXJrl11WY89VpeVJDfcUJu3eXNt3uzZtXlve1tt3tyTn1sbODBQm/fm\nN9fmTZtWm3f7bbV5G68ujRuZMb00b9KiE0vzcuKiuqy58+qykuSmTbV5mx+ozTv19bV5W7fW5j34\nUG3evfeUxt01VBqXOXNq8449tjav+K557bSPZHUanciVJACABiUJAKBBSQIAaFCSAAAalCQAgIZu\nS9J7k1yd5PtJ/qBuHACA/tBNSRpM8vokp45uryicBwCgL3Sz0sCbktyU5NIkc5L8RelEAAB9oJuS\n9JwkS5Ocmc5VpMuSrKwcCgCg17opSVuS3JJkOMntSXYlWTD6/Z/5q/X7tk9flgwu73ZEAIA66+/p\n3JLk0Z3jv2yom5J0VZJ3J/l4ksVJZiZ5dP+dPnR6F8kAABPs9GWd26ihC3/QLkrdlKTLk5yW5Np0\nXvj9riQjXeQAAPStbj8i7i9LpwAA6DMWkwQAaFCSAAAalCQAgAYlCQCgQUkCAGhQkgAAGiZNUO7I\n33S7uEDDU8N1WUmyuzBre2FWkuwoznu4OK/ancV5e4p/oxfWxmVucd7S4ryfFuedNKM4sNh9u2rz\n7i7M+n5hVpLML857XnFe8aHI1uK8E4rzTirOO774vm+g8DF8IkyuvcSz9oJdWZ1GJ3IlCQCgQUkC\nAGhQkgAAGpQkAIAGJQkAoEFJAgBoUJIAABqUJACABiUJAKBBSQIAaFCSAAAalCQAgAYlCQCgQUkC\nAGhQkgAAGpQkAIAGJQkAoEFJAgBoUJIAABoGJir4jF+ty9q2rS4rSR56uC7rtjvqspLkktq43F+c\nt6o473Nn1+bNm1ebt3Bhbd60abV5U+bMrA1curQ0btJ9tb+BI5Nr/677+heeKM1bd11d1ta6qCTJ\n8uK8PcV5y4vzdhXnzSrOO/mk2rzFi2vzqu+rqvOK7wpywWXj/Du1/wwAwJFBSQIAaFCSAAAalCQA\ngAYlCQCgoZuSNDnJZ5JclWRDkheXTgQA0Ae6KUlvSjIzyRuT/HWSD5dOBADQB7opSTuTzE0yafR/\nq5fLAADouW4Wk9yYZEaSW5Mcn+Ss0okAAPpANyXp/HSK0vuTLEny7SQvzX5XlD49ZiXqV8/v3AAA\neu26LZ1bkjy+JyvG26+bkjQz+1bM/0mSqUmm7L/TO17YRTIAwAR7zYLObdTQl+5uF6VuStJHk/xb\nku+kU5Dem87rlAAAjhjdlKTHk7ytehAAgH5iMUkAgAYlCQCgQUkCAGhQkgAAGpQkAIAGJQkAoKGb\nJQAOysKFdVkLFhx4n2fixMV1WS8qXjTzvMKfW5I8/xVzawMHfmHd0EOy54HHSvOmL5hdmrftwSdL\n8wYWFi89v2LchWK7M29eadyT37+1NG/z5tK4nPXOpaV5Z06dWpY1csdQWVaS3HxLaVxmzarNe2q4\nNm9P8aeK7thRm1dt797avIHidjBjRm1e9XzjcSUJAKBBSQIAaFCSAAAalCQAgAYlCQCgQUkCAGhQ\nkgAAGpQkAIAGJQkAoEFJAgBoUJIAABqUJACABiUJAKBBSQIAaFCSAAAalCQAgAYlCQCgQUkCAGhQ\nkgAAGiZNUO7IN15XFza5uMpV5lXP9vjjtXk7dtTmbdlSm3fH1tq8BcW/0feN1OadUDzfhz9Um3fb\nbbV5Lz5jaWneTzb9uDRv167SuJz4yoV1Ye98V11Wklx/fW1e8Z3B7nUbS/OmzxoozcvwcGncnXeU\nxmVoqDav+rFt796+zlv769/L6jQ6kStJAAANShIAQIOSBADQoCQBADQoSQAADQdTkk5Jsm50+4VJ\nrkqyIcknM3HvjgMA6KkDlaTzk/xLkumjX388yfuSnJZOQXrrxI0GANA7BypJdyQ5J/uuGL06natI\nSfLfSVZP0FwAAD11oJJ0SZKxK2iNfXptW5K55RMBAPSBZ7ok6dg1LmcnGXd96M/dt2/75XM6NwCA\nXrvxiWTT6Cc+bB3OivH2e6Yl6fokpydZn+QtSb413o6/u+QZJgMAHAavmNu5jRr62kPtonSwJenp\nT7D683ReyD0tyc1JvnwIMwIA9K2DKUl3Jzl1dPtHSQYnahgAgH5hMUkAgAYlCQCgoacl6elXltN7\nt+zs9QSMdc/Igffh8Ll6c68n4Gnr7+n1BIx1w7jvcT8yKEkkUZL6jceB/qIk9Y8N9/Z6Asa68Yle\nTzCxPN0GANDwTNdJeiZ+eKAdduzNiUkemMAZOEi7RhyLfrInjkc/2fZTx6NfPLnHsegnO546Io7H\njzLOx6xNan2zwJXpLDoJANDv1scSRwAAAAAAAPBsNznJp5JcnWRdkhf0dhzSeZH9utHbv/Z4lqPV\nKen8/JPkhUmuSrIhySczca8dZHxjj8erktyXfefIb/VqqKPQ1CSfTedcuCbJWXF+9FLreLwqyf1x\nfpQ5J8lnRrdPSXJpD2chmZGDeCciE+r8JJvS+cMhSS5Lctro9oVJzu7FUEex/Y/HO5L8We/GOaqd\nm+Tjo9vHJbk3yVfj/OiVc/OLx+MP4/wo9bH8fNO8r1eDkKRTVG9N8j9JvjX6NYfXOen8dfzd0a/H\nnhO/meSfDvtER7f9j8eF6Zwb65N8OsmsHs11NJqZfT/v45PcmeTHY/5/58fh1Toen8wRfH70YjHJ\nOUnGrrX9VI/moGN7ko8meXOSdyb5zzgeh9slSYbHfD326YNtSeYe3nGOevsfj2uSvCedZU2Gknyo\nF0Mdpbancw7MTnJxkg/k5++fnB+H1/7H4/1Jrs0RfH704sFwazo/4LEz7O3BHHTcnk4xSjoLaj2a\nzkKG9M7Y82F2kiP805H63leSXD+6fWk6r8Hg8Fma5NtJ/iPJF+L86LWxx+OLOcLPj16UpI1JfmN0\ne1U6z/3TO+el8xRokixO50rfs3311Ge767NvMda3pPMiSXrnm0leO7p9RpLrejjL0WZRkivSeZ3Y\nRaPfc370Tut4OD+KTUrnOf6No7eTejvOUW8g+96tsCGd4srhtzz7Xij8onRWrb86nef4vXvn8Fue\nfcfjFem8m2pdks/nCHvNRZ/7xySbs++dU+uSvDzOj15pHY9T4vwAAAAAAAAAAAAAAAAAAAAAAAAA\nAKAf/B+XXLccWteyXgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7efc765d52d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "net.inputs.aset(xs)\n",
    "net.forward()\n",
    "pred = net.outputs.array()\n",
    "imshow(pred.reshape(28,11).T, interpolation='none')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now construct a \"target\" array and perform CTC alignment with the output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7efc743f2150>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkkAAAD2CAYAAADCgxO9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADAFJREFUeJzt3W+MZfVdx/HPDAvFLrtLWwqWiJKlFh8JNRraWtkxItpa\n2soDnxgtRBO1D/2D0vpnYmJ8QNrExECNWi2N2qRNixjTSrC7i0BDNWI00Vp1g5TSWhZcN2xocLvj\ng3PNXIbvZO6e+7tzzsy8Xslm79179rff7Jkz+9577/wmAQAAAAAAAAAAAAAAAAAAAEZgaRGLHjly\nZO348eOLWBoAoLXjSVY2/uJCIinJ2tra2pYHra6uZnV1dYblnph3ngWu92TDtZLky22X+59nZjps\n9bc/l9U737z1gV/72pwDbfDcs43Xe67teqdOtV3v+TMzHbZ6/+msvvPg1gd+/YU5B9rgxf9tu97Z\nxuu1tnzBTIetPvBiVm++aOsD9+2bc6DptWabbWav3N92vYMzfHyej0svnemw1XufzOpPfuvWB15+\n+ZwDbVzvtW3Xe91VbdfLtzReb4a/4ySrqx/J6up7Zjjy8HzjvMzVLRd7cGlp6aYUTbTc8k8BANgt\nRBIAQGHQSFpZWRnyj2fKyltbP1XLPFaufcXQIzBl5ZrGL33R28p1h4YegSkrK9cNPcJCiSSSJCvf\n1/r1ceYhksZFJI2HSBqXlZXrhx5hofpE0nKSDyV5NMnRJNc0nQgAYAT6RNK7k1yU5C1JfiXJB5pO\nBAAwAn0i6XuTfGZy+7Ek391uHACAcegTSQeTnJ66/42e6wAAjFafnc9OJzkwdX85ybmNB01vErmy\nsuJN2gDAKBw7dizHjh1Lkpw8eXLTnS777Lh9a5Jbktye5E1Jfi3Jj2w4ZqYdt2f3RMO1Wq+3O3bc\nnpkdt+cz447bM7Pj9nxm3HF7Znbc7m/GHbdnZsftOc224/bsduaO232u6E8l+cEkj0zu3z7HYAAA\no9QnktaS/FzrQQAAxsQbrgEACiIJAKAgkgAACiIJAKAgkgAACg039Xip5WX9NQ4t96sCdq7Ge4gx\npz7bFLLdlAwAQEEkAQAURBIAQEEkAQAURBIAQEEkAQAURBIAQEEkAQAURBIAQEEkAQAURBIAQEEk\nAQAURBIAQEEkAQAURBIAQEEkAQAURBIAQEEkAQAURBIAQGHfohZeW1tb1NIAsMP5N3In8EwSAEBB\nJAEAFEQSAEBBJAEAFEQSAEBBJAEAFPpE0oVJPprkoSSPJbml6UQAACPQZ5+kH0/yTJKfSPKqJP+Q\n5C9aDgUAMLQ+kfTxJJ+Y3F5OcrbdOAAA49Anks5Mfj6QLpje324cAIBtdXizB/q+cfuqJJ9Ncm+S\nj/VcAwBgaCc2e6DPM0lXJHkgyXuTHO07EQDAmPV5Jul9SQ4l+fV0kXQ0ycUthwIAGNrSgtb17Y0B\ngJ3gwSQ3pWgim0kCABREEgBAQSQBABREEgBAQSQBABREEgBAQSQBABREEgBAQSQBABREEgBAQSQB\nABREEgBAQSQBABREEgBAQSQBABREEgBAQSQBABREEgBAQSQBABREEgBAQSQBABREEgBAQSQBABRE\nEgBAQSQBABREEgBAQSQBABREEgBAQSQBABREEgBAQSQBABTmiaTLk3wpyRsazQIAMBp9I+nCJL+X\n5EzDWQAARqNvJN2V5J4kX2k4CwDAaPSJpNuSPJPkgcn9pWbTAACMRJ/AOZ5kbfLj+iT/muRdSf5r\n6pi1+UcDAFi4E0kOp2iifT0WOzJ1+2iSn8lLAwkAYKf4/0h6GVsAAAAU+jyTNO37m0wBADAynkkC\nACiIJACAgkgCACiIJACAgkgCACiIJACAwrxbAGzq1Q3XurrhWknybSNdK0mufGXb9a745rbrXXZZ\n2/Ve3fIDZQHrXXpp2/UuuaTtet/U+OMl+xp/SrhgYZ9i2jh3brTrrb14ttlaSXLmhabL5czzbdd7\n7rm265082Xa9r3617XpPP912vadOtV3vP9sulycar3ei8Xqbffh5JgkAoCCSAAAKIgkAoCCSAAAK\nIgkAoCCSAAAKIgkAoCCSAAAKIgkAoCCSAAAKIgkAoCCSAAAKIgkAoCCSAAAKIgkAoCCSAAAKIgkA\noCCSAAAKIgkAoLC0oHXXFrTurreoE8Iu5QNm9/JZlPPgw2UuDya5KcVnVM8kAQAURBIAQEEkAQAU\nRBIAQEEkAQAU+kbSnUkeTfK3Sd7TbhwAgHHoE0krSd6c5C2T24cbzgMAMAr7evyem5P8U5L7khxM\n8ktNJwIAGIE+kfTaJFcleUe6Z5HuT/IdLYcCABhan0g6meRfkpxN8sUkX09y2eTXAQB2kk3fNtTn\nPUkPJ/nhye0rk+xP8myPdQAAhnZiswf6PJP0l0luTPL5dJH13vi2MQDALtMnkpLkl5tOAQAwMjaT\nBAAoiCQAgIJIAgAoiCQAgIJIAgAoiCQAgELfLQBYEBtOcV58wAAsjGeSAAAKIgkAoCCSAAAKIgkA\noCCSAAAKIgkAoCCSAAAKIgkAoCCSAAAKIgkAoCCSAAAKIgkAoCCSAAAKIgkAoCCSAAAKIgkAoCCS\nAAAKIgkAoCCSAAAKIgkAoCCSAAAKIgkAoCCSAAAKIgkAoCCSAAAKfSJpOcmHkzyc5KEk1zadCABg\nBPpE0s1J9id5a5LfTPJbTScCABiBPpH0QpJDSZYmP7/YdCIAgBHY1+P3PJLk4iRfSPKaJLc0nQgA\nYAT6PJN0R7pQujbJ9Uk+kuSilkMBAGyTw5s90CeS9ic5Pbn930kuTHJBj3UAAIZ2YrMH+rzcdleS\nP0ryN+kC6c5071MCANg1+kTSqSQ/2noQAIAxsZkkAEBBJAEAFEQSAEBBJAEAFEQSAEBBJAEAFEQS\nAEBBJAEAFEQSAEBBJAEAFEQSAEBBJAEAFEQSAEBBJAEAFEQSAEBBJAEAFEQSAEBBJAEAFEQSAEBB\nJAEAFEQSAEBBJAEAFEQSAEBBJAEAFEQSAEBBJAEAFEQSAEBBJAEAFEQSAEBBJAEAFEQSAEBBJAEA\nFGaJpBuSHJ3cfn2Sh5M8lOTuJEsLmgsAYFBbRdIdSX4/ySsm9z+Y5H1JbkwXSO9a3GgAAMPZKpL+\nPcmtWX/G6LvSPYuUJJ9OctOC5gIAGNRWkfTJJGen7k+/vPZ8kkPNJwIAGIHzfeP2uanbB5KcajgL\nAMB2O7zZA+cbSY8nOTK5/basv/QGALATndjsgX0zLrA2+fkX0r2R+6Ik/5zkE/PNBQAwTov6Ev61\nrQ8BABjcg+m+EO1lTWQzSQCAgkgCACiIJACAgkgCACiIJACAwqxbAPTx9zMc87okX1ngDMzOuRgX\n52NcnI/xcC7GZTecj3/LJt9mbVFbABzL+qaTAABjdjzJytBDAAAAAAAAAHNZTvKhJI8mOZrkmmHH\nId2b7I9OfvzhwLPsVTek+/tPktcneTjdN5C+O4t77yCbmz4fb0zyVNavkR8baqg96MIkH013LTyW\n5Ja4PoZUnY83JvlyXB/N3Jrkw5PbNyS5b8BZSC7ObF+JyOLckeQf0/3HIUnuT3Lj5PY9Sd49xFB7\n2Mbz8dNJfn64cfa025J8cHL7VUmeTPLncX0M5ba8/Hz8VFwfTX0gLy3Np4YahCRdqH4hyV8l+evJ\nfbbXren+d/y5yf3pa+KdSX532yfa2zaej3vSXRvHk/xBkksGmmsv2p/1v+/XJPmPJF+aetz1sb2q\n83F3dvH1McRmkgeTnJ66/42B5qBzJsldSX4oyc8m+ZM4H9vtk0nOTt2ffvng+SSHtnecPW/j+Xgs\nyS+m29bkRJLfGGKoPepMumvgQJKPJ/nVvPTzk+tje208H+9P8vns4utjiH8MT6f7C56e4dwAc9D5\nYrowSroNtZ5NtzkYw5m+Hg4kOTXUICRJPpXk8cnt+9K9B4Ptc1WSzya5N8mfxfUxtOnz8bHs8utj\niEh6JMnbJ7fflO61f4Zze7qXQJPkynTP9O303VN3usezvhnr29K9SZLhfCbJ90xu/0CSvxtwlr3m\niiQPpHuf2B9Pfs31MZzqfLg+GltK9xr/I5Mfbxh2nD1vX9a/WuGhdOHK9rs6628U/vZ0u9Y/mu41\nfl+9s/2uzvr5uC7dV1MdTfKn2WXvuRi530nydNa/cupoku+M62Mo1fm4Ia4PAAAAAAAAAAAAAAAA\nAAAAAAAAAADG4P8AXriHQTwyIkoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7efc5685cf10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "target = zeros((3,11),'f')\n",
    "target[0,0] = 1\n",
    "target[2,0] = 1\n",
    "target[1,cls] = 1\n",
    "seq = clstm.Sequence()\n",
    "seq.aset(target.reshape(3,11,1))\n",
    "aligned = clstm.Sequence()\n",
    "clstm.seq_ctc_align(aligned,net.outputs,seq)\n",
    "aligned = aligned.array()\n",
    "imshow(aligned.reshape(28,11).T, interpolation='none')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we take the aligned output, subtract the actual output, set that as the output deltas, and the propagate the error backwards and update."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "deltas = aligned - net.outputs.array()\n",
    "net.d_outputs.aset(deltas)\n",
    "net.backward()\n",
    "net.update()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If we repeat these steps over and over again, we eventually end up with a trained network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for i in range(60000):\n",
    "    index = int(rand()*60000)\n",
    "    xs = array(h5[\"images\"][index].reshape(28,28,1),'f')\n",
    "    cls = h5[\"transcripts\"][index][0]\n",
    "    net.inputs.aset(xs)\n",
    "    net.forward()\n",
    "    pred = net.outputs.array()\n",
    "    target = zeros((3,11),'f')\n",
    "    target[0,0] = 1\n",
    "    target[2,0] = 1\n",
    "    target[1,cls] = 1\n",
    "    seq = clstm.Sequence()\n",
    "    seq.aset(target.reshape(3,11,1))\n",
    "    aligned = clstm.Sequence()\n",
    "    clstm.seq_ctc_align(aligned,net.outputs,seq)\n",
    "    aligned = aligned.array()\n",
    "    deltas = aligned - net.outputs.array()\n",
    "    net.d_outputs.aset(deltas)\n",
    "    net.backward()\n",
    "    net.update()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f099fe9bf50>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATQAAAICCAYAAABBSjGyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGOVJREFUeJzt3X+wXGV9x/HPJxeohdAigw0hRgMOOFpxCFoGRUpk0AZ/\n8KO1UKbOpLRl6BSBOlMrMFOh/UelA6XWlhlKYIIilAFB7KgQGBAcKhBMSIAgYJMxQZIgPyqxOobk\n2z/2RK7Xe+8+d/fs2bPf+37N3GH33Cfn+Z49y+ees+fZ5zgiBAAZzBl2AQBQFwINQBoEGoA0CDQA\naRBoANIg0ACksUeTndleKukKSWOSro6Iz0/4PWNIgFkmIlzXutzUODTbY5K+L+kESc9KeljSGRGx\nflwbAg2YZeoMtCZPOY+S9ExEbIyIHZJulHRyg/0DSK7JQFsgadO455urZQBQiyYDjdNJAAPVZKA9\nK2nhuOcL1TlKA4BaNBloqyQdanuR7b0knS7p9gb7B5BcY8M2IuJV25+QdIc6wzaWj7/CCQD9amzY\nRgmGbQCzz6gO2wCAgSLQAKRBoAFIg0ADkAaBBiANAg1AGgQagDQINABpEGgA0mh0xlpgKu9398Hi\nd++8u2ubkm/TjY2dU1QTRg9HaADSINAApEGgAUiDQAOQBoEGIA0CDUAaBBqANAg0AGkwsBYDd0LB\noNk7d97ZtY19XEFve3Zt8QcFa7mjoA3ahyM0AGkQaADSINAApEGgAUiDQAOQBoEGIA0CDUAaBBqA\nNBhYi54VjJeVJN35D93/btrH91lNtR69p2ubj72x+3ru2FxDMWgcR2gA0iDQAKRBoAFIg0ADkAaB\nBiANAg1AGgQagDQINABpNDqw1vZGST+RtFPSjog4qsn+Ua8lKhtZ67+/e8CVAB1Nf1MgJC2JiBcb\n7hfALDCMU87CL8wAwMw0HWgh6S7bq2yf1XDfAJJr+pTzmIh4zvYbJK20/WRE3N9wDQCSavQILSKe\nq/77vKRbJXFRAEBtGgs023vb3rd6vI+kD0pa11T/APJr8pRznqRb3ZlEaw9J10dE97vLAkChxgIt\nIjZIOqKp/gDMPsxYi0m5YDrau3deWri2Y/sr5peie4s7f79rm08xG21afPUJQBoEGoA0CDQAaRBo\nANIg0ACkQaABSINAA5AGgQYgDQbWomf2/Eb7i7iqa5s3nPhA1zYv11EMWokjNABpEGgA0iDQAKRB\noAFIg0ADkAaBBiANAg1AGgQagDQYWDsLldzp+V/2L1nTR/usZJzoPhvtzYef07XNC91Xg8Q4QgOQ\nBoEGIA0CDUAaBBqANAg0AGkQaADSINAApEGgAUiDgbWzkbsPrT33+X8sWNHc/muphNZ3bXPr44ya\nxfQ4QgOQBoEGIA0CDUAaBBqANAg0AGkQaADSINAApEGgAUiDgbXJlMxG+4UDSlb0iX5LeU3RbLTv\n7NrmhjpqQWq1H6HZvsb2Vtvrxi3b3/ZK20/ZvtP2fnX3CwCDOOW8VtLSCcsukLQyIg6TdHf1HABq\nVXugRcT9kl6asPgkSSuqxysknVJ3vwDQ1EWBeRGxtXq8VdK8hvoFMIs0fpUzIkIS0yYAqF1TgbbV\n9oGSZHu+pG0N9QtgFmkq0G6XtKx6vEzSbQ31C2AWGcSwjRskPSDprbY32T5T0uckfcD2U5KOr54D\nQK0cBYMem2K7PcWMKBfMRrtrZ8FstL6ohmo6Ir7Qtc2b5nyya5vNBX39WUGbQxocTv7Kq93b/NPg\ny2i1iCgZD16Erz4BSINAA5AGgQYgDQINQBoEGoA0CDQAaRBoANIg0ACkwYy1yXSf91WSP1RTb4Xj\noF/4n65N9iwYWrnrjmO6Nzrh2q5NrEO6r6cmEa90bXPYm1/ftc1ZJaOKwREagDwINABpEGgA0iDQ\nAKRBoAFIg0ADkAaBBiANAg1AGgQagDT4psAIKZlee3XJ9No6ov9iZsBfvaVrmx/s/GL39fivSnor\naNMca9+ubd79roIV8U2BIhyhAUiDQAOQBoEGIA0CDUAaBBqANAg0AGkQaADSINAApMHA2mRc2/Ta\nJf6trNlHP9K1iX1OwYp2lfXXKgXHDO0aCzzSOEIDkAaBBiANAg1AGgQagDQINABpEGgA0iDQAKRB\noAFIo/aBtbavkfRhSdsi4vBq2SWS/lLS81WzCyPiW3X3nd0hRa32qqm36N7kW18pW9XSZQWNRnHQ\nbImfdm2xcePgq5gtBnGEdq2kpROWhaTLI2Jx9UOYAahd7YEWEfdLemmSX/EFDwAD1eRnaOfaftT2\nctv7NdgvgFmiqUC7UtLB6txu6DlJlzXUL4BZpJFAi4htUZF0taSjmugXwOzSSKDZnj/u6amS1jXR\nL4DZZRDDNm6QdJykA2xvknSxpCW2j1DnaucGSWfX3S8A1B5oEXHGJIuvqbsfAJiIGWtbomRMy61/\nVLKmt/dZSUcUjKv9+IcfKlrX9TtLBtaOou4vUlz3zq5t/nBNHbVA4qtPABIh0ACkQaABSINAA5AG\ngQYgDQINQBoEGoA0CDQAaTCwdoS88aDm+ioZ6PuOPUvXtm8flbRXxPqubU46c0P39dRRDCRxhAYg\nEQINQBoEGoA0CDQAaRBoANIg0ACkQaABSINAA5CGo2Rq0obYbk8xDbO7D2XdtfO+ghW9t4ZqCpW+\ndwq2rX2e7doi/nVx1zZzznuhjmJSi4ja3iAcoQFIg0ADkAaBBiANAg1AGgQagDQINABpEGgA0iDQ\nAKTBwNqWKBlZuPaPu//9ecdNO/ovZqS92r1JbO3aZMVb3ty1zZkbu79dW/S/V2sxsBYAJkGgAUiD\nQAOQBoEGIA0CDUAaBBqANAg0AGkQaADSqH1gre2Fkq6T9Dvq3OX+qoj4gu39Jf2npDdL2ijptIh4\necK/ZRjiNP68YObX5bt+WLCmg/ovZih+1rVFxJVd24yN/V33rgr+v+DNWo+2D6zdIemTEfG7ko6W\ndI7tt0m6QNLKiDhM0t3VcwCoTe2BFhFbImJN9Xi7pPWSFkg6SdKKqtkKSafU3TeA2W2gn6HZXiRp\nsaQHJc2L+OWX6LZKmjfIvgHMPgMLNNtzJd0i6fyIeGX876LzwR0fQQCo1UACzfae6oTZlyLitmrx\nVtsHVr+fL2nbIPoGMHvVHmju3GByuaQnIuKKcb+6XdKy6vEySbdN/LcA0I89BrDOYyR9XNJa26ur\nZRdK+pykm2z/haphGwPoG8AsVnugRcR3NPWR3wl19wcAuzFj7QgpGX244czunyK8+ZqnC9a0qKBN\nofhG1yY/v+ikrm1O/Hz3rr5dUk6L3vNo/8BaABgKAg1AGgQagDQINABpEGgA0iDQAKRBoAFIg0AD\nkAYDa5MpGqFYMPNt45ghdtZiYC0ATIJAA5AGgQYgDQINQBoEGoA0CDQAaRBoANIg0ACkMYh7CmCI\nigaftmgwNVAnjtAApEGgAUiDQAOQBoEGIA0CDUAaBBqANAg0AGkQaADSINAApEGgAUiDQAOQBoEG\nIA0CDUAaBBqANAg0AGkQaADSINAApEGgAUij9kCzvdD2PbYft/2Y7fOq5ZfY3mx7dfWztO6+Acxu\njprnl7d9oKQDI2KN7bmSHpF0iqTTJL0SEZdP82+Z7B6YZSLCda2r9pukRMQWSVuqx9ttr5e0oPp1\nbYUDwEQD/QzN9iJJiyV9t1p0ru1HbS+3vd8g+wYw+wws0KrTzZslnR8R2yVdKelgSUdIek7SZYPq\nG8DsVPtnaJJke09J/yXpmxFxxSS/XyTp6xFx+ITlfIYGzDJ1foY2iKuclrRc0hPjw8z2/HHNTpW0\nru6+Acxug7jK+T5J90laq9du5H2RpDPUOd0MSRsknR0RWyf8W47QgFmmziO0gZxy9opAA2afVp9y\nAsCwEGgA0iDQAKRBoAFIg0ADkAaBBiANAg1AGgQagDQINABpEGgA0iDQAKRBoAFIg0ADkAaBBiAN\nAg1AGgQagDQINABpEGgA0mjVFNwA0A+O0ACkQaABSINAA5BGawPN9lLbT9p+2vanh11PCdsbba+1\nvdr2Q8OuZzK2r7G91fa6ccv2t73S9lO277S93zBrnMwUdV9ie3P1eq+2vXSYNU5ke6Hte2w/bvsx\n2+dVy1v7ek9Tc6tf691aeVHA9pik70s6QdKzkh6WdEZErB9qYV3Y3iDpXRHx4rBrmYrtYyVtl3Rd\nRBxeLbtU0o8j4tLqj8frI+KCYdY50RR1XyzplYi4fKjFTcH2gZIOjIg1tudKekTSKZLOVEtf72lq\nPk0tfq13a+sR2lGSnomIjRGxQ9KNkk4eck2lartp6iBExP2SXpqw+CRJK6rHK9R5A7fKFHVLLX69\nI2JLRKypHm+XtF7SArX49Z6mZqnFr/VubQ20BZI2jXu+Wa+9qG0Wku6yvcr2WcMuZgbmRcTW6vFW\nSfOGWcwMnWv7UdvL23TqNpHtRZIWS3pQI/J6j6v5u9Wi1r/WbQ209p0HlzkmIhZLOlHSOdVp0kiJ\nzmcQo/L6XynpYElHSHpO0mXDLWdy1anbLZLOj4hXxv+ura93VfPN6tS8XSPyWrc10J6VtHDc84Xq\nHKW1WkQ8V/33eUm3qnPqPAq2Vp+dyPZ8SduGXE+RiNgWFUlXq4Wvt+091QmzL0XEbdXiVr/e42r+\n8u6aR+G1ltobaKskHWp7ke29JJ0u6fYh1zQt23vb3rd6vI+kD0paN/2/ao3bJS2rHi+TdNs0bVuj\nCoPdTlXLXm/blrRc0hMRccW4X7X29Z6q5ra/1ru18iqnJNk+UdIVksYkLY+Izw65pGnZPlidozJJ\n2kPS9W2s2fYNko6TdIA6n998RtLXJN0k6U2SNko6LSJeHlaNk5mk7oslLVHnFCgkbZB09rjPpobO\n9vsk3SdprV47rbxQ0kNq6es9Rc0XSTpDLX6td2ttoAHATLX1lBMAZoxAA5AGgQYgDQINQBoEGoA0\nCDQAaRBoANIg0ACkQaABSINAA5AGgQYgDQINQBoEGoA0CDQAaRBoANIg0ACkQaABSINAA5AGgQYg\nDQINQBoEGoA0CDQAaRBoANIg0ACkQaABSINAA5AGgQYgDQINQBoEGoA0CDQAaRBoANIg0ACkQaAB\nSINAA5AGgQYgDQINQBoEGoA0CDQAaRBoANIg0ACkQaABSINAA5AGgQYgDQINQBoEGoA0CDQAaRBo\nANIg0ACkQaABSINAA5AGgQYgDQINQBoEGoA0CDQAaRBoANIg0ACkQaABSINAA5AGgQYgDQINQBoE\nGoA0CDQAaRBoANIg0ACkQaABSINAA5AGgQYgDQINQBoEGoA0CDQAaRBoANIg0ACkQaABSINAA5AG\ngQYgDQINQBoEGoA0CDQAaRBoANIg0ACkQaABSINAA5AGgQYgDQINQBoEGoA0CDQAaRBoANIg0ACk\nQaABSINAA5AGgQYgDQINQBoEGoA0CDQAaRBoANIg0ACkQaABSINAA5AGgQYgDQINQBoEGoA0CDQA\naRBoANIg0ACkQaABSINAA5AGgQYgDQINQBoEGoA09hjUim3HoNYNYHaLCE+2vOcjNNtLbT9p+2nb\nn56i01/5ufjii39t2Sj+sB2hiEeKfnbt+u+in72knn/mTHhuu+gnYkfhz7D2xc7Cn+G/l+p5T5Vt\n73R6CjTbY5K+KGmppLdLOsP223pZFwDUpdcjtKMkPRMRGyNih6QbJZ1cX1kAMHO9BtoCSZvGPd9c\nLZvWkiVLeuyuXdiOdslwZSvLvhj2dvR6UaDoA/9LLrnkl4+XLFky9I2tC9vRLgRaewxiO+69917d\ne++3i9o6YuYXI20fLemSiFhaPb9Q0q6I+Py4NtHLujEqvlfUKuIXRe1eN+c9/RTzK3Z40gtgv2bX\nrrLaBjgYoItdhe0yRLpUur32mKLmq5yrJB1qe5HtvSSdLun2HtcFALXo6U9PRLxq+xOS7pA0Jml5\nRKyvtTIAmKGej6Uj4puSvlljLQDQl4F+ODBnTpZzewxarZ+2Fn52O2fOXnX2ihYgcQCkQaABSINA\nA5AGgQYgDQINQBoEGoA0CDQAaRBoANIg0ACkMdBvCjDbRl51/yX8QGG7QwravFy4rht5f6bT6xTc\nC23fY/tx24/ZPq/uwgBgpno9Qtsh6ZMRscb2XEmP2F7JjBsAhqmnI7SI2BIRa6rH2yWtl3RQnYUB\nwEz1/VGI7UWSFkt6sN91AUA/+gq06nTzZknnV0dqADA0/dxoeE9Jt0j6ckTcVl9JANCbXq9yWtJy\nSU9ExBX1lgQAven1CO0YSR+X9H7bq6ufpTXWBQAz1utNUr4jvmUAoGV6ui9n0YpthmGjWOns/iV/\ngX9euK7Su16ifeq+LycAtA6BBiANAg1AGgQagDQINABpEGgA0iDQAKRBoAFIg0ADkMZA7ykAlCr9\npsCrBW1K39S/KGyH0dHvfGhj1RfTv15XQQDQq35POc+X9IQkvrcJYOj6meDxjZI+JOlqSZN+URQA\nmtTPEdo/S/qUmLQAQEv0OmPtRyRti4jV4ugMQEv0eoT2Xkkn2d4g6QZJx9u+rr6yAGDm+p7g0fZx\nkv42Ij46YTkXClBsbmG7kmEbpZ+BMGxjdA16gkfCC8DQMQU3WoEjNMzEVEdoBBpGSskpBZfd8+Oe\nAgDSI9AApEGgAUiDQAOQBoEGIA0CDUAaBBqANAg0AGkQaADS4J4C6Ml+he3GCieXev6qhUXtfvb0\npq5tXvfXZxata2zRtUXtMDr6mbF2P9s3215v+wnbR9dZGADMVD9HaP8i6RsR8THbe0jap6aaAKAn\nPQWa7d+WdGxELJOkiHhV0v/WWRgAzFSvp5wHS3re9rW2v2f7P2zvXWdhADBTvQbaHpKOlPTvEXGk\npJ9KuqC2qgCgB70G2mZJmyPi4er5zeoEHAAMTU+BFhFbJG2yfVi16ARJj9dWFQD0oJ+rnOdKut72\nXpJ+IKls8A8ADEjPgRYRj0r6vRprAYC+cE8BtIJLb1dd8q4qXNeA3vpoAPcUAJAegQYgDQINQBoE\nGoA0CDQAaRBoANIg0ACkQaABSINAA5AG9xRAT+r+S3hIjaP2txSua3t9XaIl+rmnwIW2H7e9zvZX\nbP9GnYUBwEz1FGi2F0k6S9KREXG4pDFJf1JfWQAwc72ecv5E0g5Je9veKWlvSc/WVhUA9KDXCR5f\nlHSZpB9K+pGklyPirjoLA4CZ6vWU8y2S/kbSIkkHSZpr+09rrAsAZqzXiwLvlvRARLxQ3cLuq5Le\nW19ZADBzvQbak5KOtv2btq3OPQWeqK8sAJi5Xj9De1TSdZJWSVpbLb6qrqIAoBdMwY2e1D6wtsZ1\nbSlsx8Da0TXVFNx8UwA9OWBI6/u/gja8qWcvvssJIA0CDUAaBBqANAg0AGkQaADSINAApEGgAUiD\nQAOQBoEGIA0GVaMn22pe396F7X6roE3pm7r0r/muwnYYvmn3qe1rbG+1vW7csv1tr7T9lO07be83\n+DIBoLtuf6SulbR0wrILJK2MiMMk3V09B4ChmzbQIuJ+SS9NWHySpBXV4xWSThlAXQAwY71cFJgX\nEVurx1slzauxHgDoWV9XOaMzmRrzngFohV4CbavtAyXJ9nzVf8ELmBEmasRuvQTa7ZKWVY+XSbqt\nvnKAmSPQsFu3YRs3SHpA0lttb7J9pqTPSfqA7ackHV89B4ChG/Q9Bb43YfF8Sc8NpMNmsR01W1TY\nbrKBtT+S5h80bjt+VLiuFwvbNTSwtjX7ok9NbMeRU91TgJukABg5jQcaADSNL6cDSINAA5AGgQYg\njcYCzfZS20/aftr2p5vqt262N9pea3u17YeGXU+JLLOmTLEdl9jeXO2P1bYnTqbQOrYX2r7H9uO2\nH7N9XrV8pPbJNNsxtH3SyEUB22OSvi/pBEnPSnpY0hkRsX7gndfM9gZJ74qI0qv+Q2f7WHXGn14X\nEYdXyy6V9OOIuLT6A/P6iGj1zClTbMfFkl6JiMuHWtwMVN+0OTAi1tieK+kRdSZ5OFMjtE+m2Y7T\nNKR90tQR2lGSnomIjRGxQ9KNkk5uqO9BmPSScVtlmTVliu2QRm9/bImINdXj7ZLWS1qgEdsn02yH\nNKR90lSgLZC0adzzzXptw0dNSLrL9irbZw27mD5kmjXlXNuP2l7e9tO0iWwvkrRY0oMa4X0ybju+\nWy0ayj5pKtAyDXY7JiIWSzpR0jnVadBIG/FZU66UdLCkI9QZoX7ZcMspV52m3SLp/Ih4ZfzvRmmf\nVNtxszrbsV1D3CdNBdqzkhaOe75QnaO0kRMRz1X/fV7SreqcTo+iFLOmRMS2qEi6WiOyP2zvqU6Y\nfSkidk/wMHL7ZNx2fHn3dgxznzQVaKskHWp7ke29JJ2uzqwdI8X23rb3rR7vI+mDktZN/69aK8Ws\nKdX/+LudqhHYH7YtabmkJyLiinG/Gql9MtV2DHOfNPbVJ9snSrpC0pik5RHx2UY6rpHtg9U5KpM6\nNxe6fhS2o5o15ThJB6jz2cxnJH1N0k2S3iRpo6TTIuLlYdVYYpLtuFjSEnVObULSBklnj/scqpVs\nv0/SfZLW6rXTygslPaQR2idTbMdFks7QkPYJ3+UEkAbfFACQBoEGIA0CDUAaBBqANAg0AGkQaADS\nINAApPH/3jUAnTujG+0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f099ffc5ad0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "figsize(5,10)\n",
    "subplot(211,aspect=1)\n",
    "imshow(xs.reshape(28,28).T)\n",
    "subplot(212,aspect=1)\n",
    "imshow(pred.reshape(28,11).T, interpolation='none', vmin=0, vmax=1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}