Deep RNN, Multilayered and Softmax Layer MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy.random as rnd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py:93: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    }
   ],
   "source": [
    "#CONSTRUCTION PHASE\n",
    "n_steps = 28\n",
    "n_inputs = 28 #28 features, 1 per pixel\n",
    "n_neurons = 150\n",
    "n_outputs = 10\n",
    "n_layers = 3\n",
    "\n",
    "learning_rate = 0.001\n",
    "\n",
    "#create input and output placeholders\n",
    "#X has shape [    t = 0              t = 1          t = 28 (n_steps)\n",
    "#             [[x1,x2,...,x28],[x1,x2,...,x28],...,[x1,x2,...,x28]]  instance 1\n",
    "#             [[x1,x2,...,x28],[x1,x2,...,x28],...,[x1,x2,...,x28]]  instance 2\n",
    "#             [[x1,x2,...,x28],[x1,x2,...,x28],...,[x1,x2,...,x28]]  instance 3\n",
    "#             ...\n",
    "#             [[x1,x2,...,x28],[x1,x2,...,x28],...,[x1,x2,...,x28]]  total instances in batch (None | Undefined)\n",
    "#            ]\n",
    "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "\n",
    "#Y is an integer 0 - 9\n",
    "y = tf.placeholder(tf.int32, [None])\n",
    "\n",
    "#create the cells and the layers of the network\n",
    "layers = [tf.contrib.rnn.BasicRNNCell(num_units=n_neurons) for layer in range(n_layers)]\n",
    "multi_layer_cell = tf.contrib.rnn.MultiRNNCell(layers, state_is_tuple=True)\n",
    "outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)\n",
    "\n",
    "#For classification, we only want the output activation at the last time step so we use 'outputs' \n",
    "#We transpose so that the time axis is first and use tf.gather() for selecting the last index, x28. \n",
    "\n",
    "#X has shape [    instance 1       instance 2      instance n (150)\n",
    "#             [[x1,x2,...,x28],[x1,x2,...,x28],...,[x1,x2,...,x28]]  t = 0\n",
    "#             [[x1,x2,...,x28],[x1,x2,...,x28],...,[x1,x2,...,x28]]  t = 1\n",
    "#             [[x1,x2,...,x28],[x1,x2,...,x28],...,[x1,x2,...,x28]]  t = 2\n",
    "#             ...\n",
    "#             [[x1,x2,...,x28],[x1,x2,...,x28],...,[x1,x2,...,x28]]  t = 28\n",
    "#            ]\n",
    "outputs = tf.transpose(outputs, [1, 0, 2])\n",
    "\n",
    "#gather the final values across t = 28\n",
    "last = tf.gather(outputs, int(outputs.get_shape()[0]) - 1)\n",
    "\n",
    "#Create Softmax layer with XEntropy \n",
    "logits = tf.layers.dense(last, n_outputs)\n",
    "xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,logits=logits)\n",
    "\n",
    "#Compute the loss, mean xentropy score\n",
    "loss = tf.reduce_mean(xentropy)\n",
    "\n",
    "#Set the optimizer\n",
    "#Here we use Adaptive Momement Estimation which keeps track of an exponentially decaying average of past gradients\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "#Compute accuracy\n",
    "#Does the highest logit correspond to the target class?\n",
    "#in_top_k returns a 1D tensor full of boolean values\n",
    "correct = tf.nn.in_top_k(logits, y, 1)\n",
    "#We cast to floating point numbers and then take the mean to compute an accuracy score\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "#Adds an op to initialize all variables in the model\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "#IMPORT MNIST DATASET\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"tmp/data/\")\n",
    "X_test = mnist.test.images.reshape((-1, n_steps, n_inputs))\n",
    "y_test = mnist.test.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.966667 Test accuracy: 0.9499\n",
      "1 Train accuracy: 0.986667 Test accuracy: 0.9585\n",
      "2 Train accuracy: 0.96 Test accuracy: 0.965\n",
      "3 Train accuracy: 0.986667 Test accuracy: 0.968\n",
      "4 Train accuracy: 0.96 Test accuracy: 0.9752\n",
      "5 Train accuracy: 0.973333 Test accuracy: 0.9727\n",
      "6 Train accuracy: 0.973333 Test accuracy: 0.9716\n",
      "7 Train accuracy: 0.96 Test accuracy: 0.9745\n",
      "8 Train accuracy: 1.0 Test accuracy: 0.971\n",
      "9 Train accuracy: 0.986667 Test accuracy: 0.9738\n",
      "10 Train accuracy: 0.973333 Test accuracy: 0.9753\n",
      "11 Train accuracy: 0.993333 Test accuracy: 0.9723\n",
      "12 Train accuracy: 0.986667 Test accuracy: 0.9716\n",
      "13 Train accuracy: 0.993333 Test accuracy: 0.9743\n",
      "14 Train accuracy: 1.0 Test accuracy: 0.9781\n",
      "15 Train accuracy: 0.986667 Test accuracy: 0.9766\n",
      "16 Train accuracy: 1.0 Test accuracy: 0.9775\n",
      "17 Train accuracy: 0.993333 Test accuracy: 0.9747\n",
      "18 Train accuracy: 0.993333 Test accuracy: 0.9777\n",
      "19 Train accuracy: 0.98 Test accuracy: 0.9778\n",
      "20 Train accuracy: 0.993333 Test accuracy: 0.9769\n",
      "21 Train accuracy: 0.993333 Test accuracy: 0.9763\n",
      "22 Train accuracy: 0.98 Test accuracy: 0.9737\n",
      "23 Train accuracy: 0.98 Test accuracy: 0.9724\n",
      "24 Train accuracy: 0.986667 Test accuracy: 0.9739\n",
      "25 Train accuracy: 0.993333 Test accuracy: 0.9763\n",
      "26 Train accuracy: 0.993333 Test accuracy: 0.978\n",
      "27 Train accuracy: 0.993333 Test accuracy: 0.9786\n",
      "28 Train accuracy: 1.0 Test accuracy: 0.977\n",
      "29 Train accuracy: 0.986667 Test accuracy: 0.9787\n",
      "30 Train accuracy: 0.993333 Test accuracy: 0.9734\n",
      "31 Train accuracy: 1.0 Test accuracy: 0.975\n",
      "32 Train accuracy: 0.986667 Test accuracy: 0.9779\n",
      "33 Train accuracy: 0.98 Test accuracy: 0.9805\n",
      "34 Train accuracy: 1.0 Test accuracy: 0.9818\n",
      "35 Train accuracy: 0.993333 Test accuracy: 0.9806\n",
      "36 Train accuracy: 0.986667 Test accuracy: 0.9759\n",
      "37 Train accuracy: 1.0 Test accuracy: 0.9822\n",
      "38 Train accuracy: 0.993333 Test accuracy: 0.9798\n",
      "39 Train accuracy: 0.993333 Test accuracy: 0.975\n",
      "40 Train accuracy: 0.986667 Test accuracy: 0.9791\n",
      "41 Train accuracy: 0.986667 Test accuracy: 0.9748\n",
      "42 Train accuracy: 0.993333 Test accuracy: 0.9792\n",
      "43 Train accuracy: 1.0 Test accuracy: 0.9802\n",
      "44 Train accuracy: 0.953333 Test accuracy: 0.9778\n",
      "45 Train accuracy: 1.0 Test accuracy: 0.9756\n",
      "46 Train accuracy: 0.98 Test accuracy: 0.9799\n",
      "47 Train accuracy: 0.973333 Test accuracy: 0.9766\n",
      "48 Train accuracy: 1.0 Test accuracy: 0.9771\n",
      "49 Train accuracy: 0.986667 Test accuracy: 0.9743\n",
      "50 Train accuracy: 1.0 Test accuracy: 0.9817\n",
      "51 Train accuracy: 0.993333 Test accuracy: 0.9782\n",
      "52 Train accuracy: 0.98 Test accuracy: 0.9728\n",
      "53 Train accuracy: 1.0 Test accuracy: 0.9773\n",
      "54 Train accuracy: 0.986667 Test accuracy: 0.9799\n",
      "55 Train accuracy: 0.986667 Test accuracy: 0.9705\n",
      "56 Train accuracy: 0.986667 Test accuracy: 0.9765\n",
      "57 Train accuracy: 1.0 Test accuracy: 0.9745\n",
      "58 Train accuracy: 0.993333 Test accuracy: 0.975\n",
      "59 Train accuracy: 0.973333 Test accuracy: 0.975\n",
      "60 Train accuracy: 1.0 Test accuracy: 0.9791\n",
      "61 Train accuracy: 0.98 Test accuracy: 0.9785\n",
      "62 Train accuracy: 0.993333 Test accuracy: 0.9755\n",
      "63 Train accuracy: 0.993333 Test accuracy: 0.9794\n",
      "64 Train accuracy: 1.0 Test accuracy: 0.9746\n",
      "65 Train accuracy: 0.986667 Test accuracy: 0.978\n",
      "66 Train accuracy: 0.973333 Test accuracy: 0.9776\n",
      "67 Train accuracy: 0.986667 Test accuracy: 0.9776\n",
      "68 Train accuracy: 0.986667 Test accuracy: 0.9645\n",
      "69 Train accuracy: 0.96 Test accuracy: 0.9765\n",
      "70 Train accuracy: 0.993333 Test accuracy: 0.976\n",
      "71 Train accuracy: 0.98 Test accuracy: 0.9758\n",
      "72 Train accuracy: 0.993333 Test accuracy: 0.9746\n",
      "73 Train accuracy: 0.986667 Test accuracy: 0.974\n",
      "74 Train accuracy: 0.966667 Test accuracy: 0.963\n",
      "75 Train accuracy: 0.986667 Test accuracy: 0.9773\n",
      "76 Train accuracy: 0.98 Test accuracy: 0.9785\n",
      "77 Train accuracy: 0.986667 Test accuracy: 0.9684\n",
      "78 Train accuracy: 0.986667 Test accuracy: 0.9652\n",
      "79 Train accuracy: 0.966667 Test accuracy: 0.9708\n",
      "80 Train accuracy: 0.96 Test accuracy: 0.9689\n",
      "81 Train accuracy: 0.986667 Test accuracy: 0.9712\n",
      "82 Train accuracy: 0.986667 Test accuracy: 0.9773\n",
      "83 Train accuracy: 0.986667 Test accuracy: 0.9757\n",
      "84 Train accuracy: 0.966667 Test accuracy: 0.9582\n",
      "85 Train accuracy: 0.993333 Test accuracy: 0.9779\n",
      "86 Train accuracy: 0.993333 Test accuracy: 0.9782\n",
      "87 Train accuracy: 0.98 Test accuracy: 0.9575\n",
      "88 Train accuracy: 0.98 Test accuracy: 0.9663\n",
      "89 Train accuracy: 0.986667 Test accuracy: 0.9671\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-064430c66bc8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m             \u001b[0;31m#feed in the batch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m             \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtraining_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mX_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0my_batch\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m         \u001b[0;31m#compute accuracy of RNN against Training Set and Test Set\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    787\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    788\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 789\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    790\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    791\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    995\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    996\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 997\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    998\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    999\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1130\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1131\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1132\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1133\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1134\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1137\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1138\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1139\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1140\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1141\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1119\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[1;32m   1120\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1121\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m   1122\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1123\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#TRAINING PHASE\n",
    "n_epochs = 100\n",
    "batch_size = 150\n",
    "\n",
    "#open a TensorFlow Session\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    \n",
    "    #Epoch is a single pass through the entire training set, followed by testing of the verification set.\n",
    "    for epoch in range(n_epochs):\n",
    "        #Number of batches, here we exhaust the training set\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):   \n",
    "\n",
    "            #obtain batch of specified batch size\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size) \n",
    "            \n",
    "            #reshape the array to size n_steps * n_inputs\n",
    "            X_batch = X_batch.reshape((-1, n_steps, n_inputs)) \n",
    "            \n",
    "            #feed in the batch\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        \n",
    "        #compute accuracy of RNN against Training Set and Test Set\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}