lstm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import random\n",
    "from six.moves import range\n",
    "import string\n",
    "import os\n",
    "import zipfile\n",
    "from six.moves.urllib.request import urlretrieve\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "filename = 'text8.zip'\n",
    "\n",
    "def read_data(filename):\n",
    "    with zipfile.ZipFile(filename)as f:\n",
    "        name = f.namelist()[0]\n",
    "        data = tf.compat.as_str(f.read(name))\n",
    "    return data\n",
    "\n",
    "text = read_data(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99999000 ons anarchists advocate social relations based upon voluntary as\n",
      "1000  anarchism originated as a term of abuse first used against earl\n"
     ]
    }
   ],
   "source": [
    "# Now create a validation set\n",
    "valid_size=1000\n",
    "valid_text= text[:valid_size]\n",
    "train_text = text[valid_size:]\n",
    "train_size = len(train_text)\n",
    "print(train_size, train_text[:64])\n",
    "print(valid_size, valid_text[:64])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Utility func for char_to_id and vice versa\n",
    "\n",
    "vocabulary_size = len(string.ascii_lowercase) + 1\n",
    "first_letter_int  = ord(string.ascii_lowercase[0])\n",
    "\n",
    "def char2id(char):\n",
    "    if char in string.ascii_lowercase:\n",
    "        return ord(char) - first_letter_int + 1\n",
    "    elif char==' ':\n",
    "        return 0\n",
    "    else:\n",
    "        print('Unexpected Character %s' %char)\n",
    "        return 0\n",
    "    \n",
    "def id2char(id):\n",
    "    if id>0 :\n",
    "        return chr(id + first_letter_int -1)\n",
    "    else:\n",
    "        return ' '"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unexpected Character ,\n",
      "0\n",
      "a\n"
     ]
    }
   ],
   "source": [
    "print(char2id(','))\n",
    "print(id2char(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Func to generate a training batch for LSTM Model\n",
    "\n",
    "batch_size=64\n",
    "num_unrollings=10\n",
    "class BatchGenerator(object):\n",
    "    def __init__(self,text,batch_size,num_unrollings):\n",
    "        self._text=text\n",
    "        self._num_unrollings = num_unrollings\n",
    "        self._text_size = len(text)\n",
    "        self._batch_size = batch_size\n",
    "        num_batches = self._text_size // batch_size\n",
    "        self._cursor = [offset *  num_batches for offset in range(batch_size)]\n",
    "        self._last_batch = self._next_batch()\n",
    "    \n",
    "    def _next_batch(self):\n",
    "        batch = np.zeros(shape=(self._batch_size,vocabulary_size),dtype=np.float)\n",
    "        for b in range(self._batch_size):\n",
    "            batch[b,char2id(self._text[self._cursor[b]])]=1.0\n",
    "            self._cursor[b] = (self._cursor[b]+1) % self._text_size\n",
    "        return batch\n",
    "    \n",
    "    def next(self):\n",
    "        batches = [self._last_batch]\n",
    "        for step in range(self._num_unrollings):\n",
    "            batches.append(self._next_batch())\n",
    "        self._last_batch = batches[-1]\n",
    "        return batches\n",
    "    \n",
    "def characters(probabilities):\n",
    "    return [id2char(i) for i in np.argmax(probabilities,1)]\n",
    "    \n",
    "def batches2string(batches):\n",
    "    s=['']*batches[0].shape[0]\n",
    "    for b in batches:\n",
    "        s = [''.join(x) for x in zip(s,characters(b))]\n",
    "    return s\n",
    "\n",
    "train_batches = BatchGenerator(train_text,batch_size,num_unrollings)\n",
    "valid_batches = BatchGenerator(valid_text, 1, 1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def logprob(predictions,labels):\n",
    "    predictions[predictions< 1e-10] = 1e-10\n",
    "    return np.sum(np.multiply(labels,-np.log(predictions)))/labels.shape[0]\n",
    "\n",
    "\n",
    "def sample_distribution(distribution):\n",
    "    r = random.uniform(0,1)\n",
    "    s=0\n",
    "    for i in range(len(distribution)):\n",
    "        s += distribution[i]\n",
    "        if s > r:\n",
    "            return i\n",
    "    return len(distribution)-1\n",
    "\n",
    "def sample(prediction):\n",
    "    p = np.zeros(shape=[1,vocabulary_size],dtype=np.float)\n",
    "    p[0,sample_distribution(prediction[0])] = 1.0\n",
    "    return p\n",
    "\n",
    "def random_distribution():\n",
    "    b = np.random.uniform(0.0,1.0,size=[1,vocabulary_size])\n",
    "    return b/ np.sum(b,1)[:,None]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Now we'll design a simple LSTM Model\n",
    "\n",
    "\n",
    "num_nodes=64\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    # Define hyper params\n",
    "    # Input gate: inp,prev,bias\n",
    "    ix = tf.Variable(tf.truncated_normal([vocabulary_size,num_nodes],-0.1,0.1))\n",
    "    im = tf.Variable(tf.truncated_normal([num_nodes,num_nodes],-0.1,0.1))\n",
    "    ib = tf.Variable(tf.zeros([1,num_nodes]))\n",
    "    \n",
    "    # Forget gate: inp,prev,bias\n",
    "    fx = tf.Variable(tf.truncated_normal([vocabulary_size,num_nodes],-0.1,0.1))\n",
    "    fm = tf.Variable(tf.truncated_normal([num_nodes,num_nodes],-0.1,0.1))\n",
    "    fb = tf.Variable(tf.zeros([1,num_nodes]))\n",
    "    \n",
    "    # Memory cell: inp,state,bias\n",
    "    cx = tf.Variable(tf.truncated_normal([vocabulary_size,num_nodes],-0.1,0.1))\n",
    "    cm = tf.Variable(tf.truncated_normal([num_nodes,num_nodes],-0.1,0.1))\n",
    "    cb = tf.Variable(tf.zeros([1,num_nodes]))\n",
    "    \n",
    "    #Output Gate : inp,prev,bias\n",
    "    ox = tf.Variable(tf.truncated_normal([vocabulary_size,num_nodes],-0.1,0.1))\n",
    "    om = tf.Variable(tf.truncated_normal([num_nodes,num_nodes],-0.1,0.1))\n",
    "    ob = tf.Variable(tf.zeros([1,num_nodes]))\n",
    "    \n",
    "    #Variables saving states across unrollings\n",
    "    saved_output = tf.Variable(tf.zeros([batch_size,num_nodes]),trainable=False)\n",
    "    saved_state = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "    \n",
    "    # Classifier weights and bias\n",
    "    w = tf.Variable(tf.truncated_normal([num_nodes,vocabulary_size],-0.1,0.1))\n",
    "    b = tf.Variable(tf.zeros([vocabulary_size]))\n",
    "    \n",
    "    \n",
    "    #Def of cell computation\n",
    "    def lstm_cell(i,o,state):\n",
    "        input_gate = tf.sigmoid(tf.matmul(i,ix) + tf.matmul(o,im) + ib)\n",
    "        forget_gate = tf.sigmoid(tf.matmul(i,fx)+tf.matmul(o,fm) + fb)\n",
    "        update = tf.matmul(i,cx)+tf.matmul(o,cm) + cb\n",
    "        state = forget_gate * state + input_gate * tf.tanh(update)\n",
    "        output_gate = tf.sigmoid(tf.matmul(i,ox)+tf.matmul(o,om)+ob)\n",
    "        return output_gate * tf.tanh(state), state\n",
    "    \n",
    "    train_data = list()\n",
    "    for _ in range(num_unrollings+1):\n",
    "        train_data.append(tf.placeholder(dtype=tf.float32,shape=[batch_size,vocabulary_size]))\n",
    "    train_inputs = train_data[:num_unrollings]\n",
    "    train_labels = train_data[1:] # labels are inputs shifted by one time step.\n",
    "    \n",
    "    #Unrolled lstm loop\n",
    "    outputs = list()\n",
    "    output = saved_output\n",
    "    state = saved_state\n",
    "    for i in train_inputs:\n",
    "        output,state = lstm_cell(i,output,state)\n",
    "        outputs.append(state)\n",
    "    \n",
    "    # State savings across unrollings.\n",
    "    with tf.control_dependencies([saved_output.assign(output),saved_state.assign(state)]):\n",
    "        logits = tf.nn.xw_plus_b(tf.concat(outputs,axis=0),w,b)\n",
    "        loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=tf.concat(train_labels,axis=0),logits=logits))\n",
    "        \n",
    "    \n",
    "    #Optimizer\n",
    "    global_step = tf.Variable(0)\n",
    "    learning_rate = tf.train.exponential_decay(10.0,global_step,5000,0.1,staircase=True)\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    gradients,v = zip(*optimizer.compute_gradients(loss)) # * here to unpack coz comp_grads(..) returns list(tuple(gradient,variable))\n",
    "    gradients,_ = tf.clip_by_global_norm(gradients,1.25) # deal with exploding gradient problem\n",
    "    \n",
    "    optimizer = optimizer.apply_gradients(zip(gradients,v),global_step=global_step)\n",
    "    \n",
    "    \n",
    "    # predictions\n",
    "    train_prediction = tf.nn.softmax(logits)\n",
    "    \n",
    "    #sampling and validating eval..no unrolling since not back prop\n",
    "    \n",
    "    sample_input  = tf.placeholder(shape=[1,vocabulary_size],dtype=tf.float32)\n",
    "    saved_sample_output = tf.Variable(tf.zeros([1,num_nodes]))\n",
    "    saved_sample_state =  tf.Variable(tf.zeros([1,num_nodes]))\n",
    "    \n",
    "    reset_sample_state=tf.group(saved_sample_output.assign(tf.zeros([1,num_nodes])),saved_sample_state.assign(tf.zeros([1,num_nodes])))\n",
    "    sample_output,sample_state = lstm_cell(sample_input,saved_sample_output,saved_sample_state)\n",
    "    \n",
    "    with tf.control_dependencies([saved_sample_output.assign(sample_output),saved_sample_state.assign(sample_state)]):\n",
    "        sample_prediction  = tf.nn.softmax(tf.nn.xw_plus_b(sample_output, w, b)) # <- This guy is the fk'n classifier ..!! ding dong\n",
    "        \n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " <b>Refer to this diagram for above implementation details of lstm_cell [Forget,Input,Update(Memory),Output &lt;- Gates in this order]</b><br><img src='lstm.png' width=600px  height=400px/></br>\n",
    "<br>\n",
    " <p style=\"border-style:1px double red;border-radius: 10px;border-width: medium;\" > \n",
    " &nbsp;&nbsp;<span style=\"color:green\"><b> def </b></span> <span style=\"color:blue\">lstm_cell</span>(i,o,state):<br>\n",
    " &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; input_gate = tf.sigmoid(tf.matmul(i,ix) + tf.matmul(o,im) + ib) <br>\n",
    " &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; forget_gate = tf.sigmoid(tf.matmul(i,fx)+tf.matmul(o,fm) + fb) <br>\n",
    " &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; update = tf.matmul(i,cx)+tf.matmul(o,cm) + cb <br>\n",
    " &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; state = forget_gate * state + input_gate * tf.tanh(update) <br>\n",
    " &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; output_gate = tf.sigmoid(tf.matmul(i,ox)+tf.matmul(o,om)+ob) <br>\n",
    " &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; return output_gate * tf.tanh(state), state <br>\n",
    " </p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Global vars initialized\n",
      "Average loss at step 0: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity : 27.05\n",
      "================================================================================\n"
     ]
    },
    {
     "ename": "AxisError",
     "evalue": "axis 1 is out of bounds for array of dimension 1",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAxisError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-cae3e57e1299>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     25\u001b[0m             \u001b[0;32mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'='\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;36m80\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m                 \u001b[0msentence\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcharacters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m                 \u001b[0mfeed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrandom_distribution\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m                 \u001b[0msentence\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mcharacters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-6-8a64f1159c1d>\u001b[0m in \u001b[0;36mcharacters\u001b[0;34m(probabilities)\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mcharacters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprobabilities\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mid2char\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprobabilities\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mbatches2string\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/rahulchandran/tensorflow/lib/python2.7/site-packages/numpy/core/fromnumeric.pyc\u001b[0m in \u001b[0;36margmax\u001b[0;34m(a, axis, out)\u001b[0m\n\u001b[1;32m    961\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    962\u001b[0m     \"\"\"\n\u001b[0;32m--> 963\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_wrapfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'argmax'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    964\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    965\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/rahulchandran/tensorflow/lib/python2.7/site-packages/numpy/core/fromnumeric.pyc\u001b[0m in \u001b[0;36m_wrapfunc\u001b[0;34m(obj, method, *args, **kwds)\u001b[0m\n\u001b[1;32m     65\u001b[0m     \u001b[0;31m# a downstream library like 'pandas'.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mAttributeError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 67\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_wrapit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/rahulchandran/tensorflow/lib/python2.7/site-packages/numpy/core/fromnumeric.pyc\u001b[0m in \u001b[0;36m_wrapit\u001b[0;34m(obj, method, *args, **kwds)\u001b[0m\n\u001b[1;32m     45\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m         \u001b[0mwrap\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 47\u001b[0;31m     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     48\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mwrap\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmu\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAxisError\u001b[0m: axis 1 is out of bounds for array of dimension 1"
     ]
    }
   ],
   "source": [
    "num_steps = 7001\n",
    "summary_frequency= 100\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    tf.global_variables_initializer().run()\n",
    "    print('Global vars initialized')\n",
    "    mean_loss = 0\n",
    "    feed=\"\"\n",
    "    for step in range(num_steps):\n",
    "        batches = train_batches.next()\n",
    "        feed_dict = {}\n",
    "        for i in range(num_unrollings+1):\n",
    "            feed_dict[train_data[i]] = batches[i]\n",
    "        _,l,predictions,lr = session.run([optimizer,loss,train_prediction,learning_rate],feed_dict=feed_dict)\n",
    "        mean_loss += 1\n",
    "        if step%summary_frequency==0:\n",
    "            if step > 0:\n",
    "                mean_loss = mean_loss / summary_frequency\n",
    "            print('Average loss at step %d: %f learning rate: %f' % (step, mean_loss, lr))\n",
    "            mean_loss=0\n",
    "            labels = np.concatenate(list(batches)[1:])\n",
    "            print('Minibatch perplexity : %.2f' %float( np.exp(logprob(predictions,labels))))\n",
    "        if step % (summary_frequency * 10) ==0:\n",
    "            # generate samples\n",
    "            print('=' * 80)\n",
    "            for _ in range(5):\n",
    "                sentence = characters(feed)[0]\n",
    "                feed = sample(random_distribution())\n",
    "                sentence += characters(feed)[0]\n",
    "                reset_sample_state.run()\n",
    "                for _ in range(80):\n",
    "                    prediction = sample_prediction.eval({sample_input:feed})\n",
    "                    feed = sample(prediction)\n",
    "                    sentence += characters(feed)[0]\n",
    "                print(sentence)    \n",
    "            print('=' * 80)\n",
    "            #measure validation set perplexity\n",
    "            \n",
    "            reset_sample_state.run()\n",
    "            valid_logprob = 0\n",
    "            for _ in range(valid_size):\n",
    "                b = valid_batches.next()\n",
    "                predictions = sample_prediction.eval({sample_input: b[0]})\n",
    "                valid_logprob = valid_logprob + logprob(predictions, b[1])\n",
    "                \n",
    "            print('Validation set perplexity: %.2f' % float(np.exp(valid_logprob / valid_size)))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}