Deep Network Comparison.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Neural Network Comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Load necessary libraries\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import tensorflow.contrib.slim as slim\n",
    "import input_data\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load CIFAR Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To obtain the CIFAR10 dataset, go here: https://www.cs.toronto.edu/~kriz/cifar.html\n",
    "\n",
    "The training data is stored in 5 separate files, and we will alternate between them during training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def unpickle(file):\n",
    "    import cPickle\n",
    "    fo = open(file, 'rb')\n",
    "    dict = cPickle.load(fo)\n",
    "    fo.close()\n",
    "    return dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "currentCifar = 1\n",
    "cifar = unpickle('./cifar10/data_batch_1')\n",
    "cifarT = unpickle('./cifar10/test_batch')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "total_layers = 25 #Specify how deep we want our network\n",
    "units_between_stride = total_layers / 5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RegularNet\n",
    "A Deep Neural Network composed exclusively of regular and strided convolutional layers. While this architecture works well for relatively shallow networks, it becomes increasingly more difficult to train as the network depth increases."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "input_layer = tf.placeholder(shape=[None,32,32,3],dtype=tf.float32,name='input')\n",
    "label_layer = tf.placeholder(shape=[None],dtype=tf.int32)\n",
    "label_oh = slim.layers.one_hot_encoding(label_layer,10)\n",
    "\n",
    "layer1 = slim.conv2d(input_layer,64,[3,3],normalizer_fn=slim.batch_norm,scope='conv_'+str(0))\n",
    "for i in range(5):\n",
    "    for j in range(units_between_stride):\n",
    "        layer1 = slim.conv2d(layer1,64,[3,3],normalizer_fn=slim.batch_norm,scope='conv_'+str((j+1) + (i*units_between_stride)))\n",
    "    layer1 = slim.conv2d(layer1,64,[3,3],stride=[2,2],normalizer_fn=slim.batch_norm,scope='conv_s_'+str(i))\n",
    "    \n",
    "top = slim.conv2d(layer1,10,[3,3],normalizer_fn=slim.batch_norm,activation_fn=None,scope='conv_top')\n",
    "\n",
    "output = slim.layers.softmax(slim.layers.flatten(top))\n",
    "\n",
    "loss = tf.reduce_mean(-tf.reduce_sum(label_oh * tf.log(output) + 1e-10, axis=[1]))\n",
    "trainer = tf.train.AdamOptimizer(learning_rate=0.001)\n",
    "update = trainer.minimize(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ResNet\n",
    "An implementation of a Residual Network as described in [Identity Mappings in Deep Residual Networks](https://arxiv.org/abs/1603.05027)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def resUnit(input_layer,i):\n",
    "    with tf.variable_scope(\"res_unit\"+str(i)):\n",
    "        part1 = slim.batch_norm(input_layer,activation_fn=None)\n",
    "        part2 = tf.nn.relu(part1)\n",
    "        part3 = slim.conv2d(part2,64,[3,3],activation_fn=None)\n",
    "        part4 = slim.batch_norm(part3,activation_fn=None)\n",
    "        part5 = tf.nn.relu(part4)\n",
    "        part6 = slim.conv2d(part5,64,[3,3],activation_fn=None)\n",
    "        output = input_layer + part6\n",
    "        return output\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "input_layer = tf.placeholder(shape=[None,32,32,3],dtype=tf.float32,name='input')\n",
    "label_layer = tf.placeholder(shape=[None],dtype=tf.int32)\n",
    "label_oh = slim.layers.one_hot_encoding(label_layer,10)\n",
    "\n",
    "layer1 = slim.conv2d(input_layer,64,[3,3],normalizer_fn=slim.batch_norm,scope='conv_'+str(0))\n",
    "for i in range(5):\n",
    "    for j in range(units_between_stride):\n",
    "        layer1 = resUnit(layer1,j + (i*units_between_stride))\n",
    "    layer1 = slim.conv2d(layer1,64,[3,3],stride=[2,2],normalizer_fn=slim.batch_norm,scope='conv_s_'+str(i))\n",
    "    \n",
    "top = slim.conv2d(layer1,10,[3,3],normalizer_fn=slim.batch_norm,activation_fn=None,scope='conv_top')\n",
    "\n",
    "output = slim.layers.softmax(slim.layers.flatten(top))\n",
    "\n",
    "loss = tf.reduce_mean(-tf.reduce_sum(label_oh * tf.log(output) + 1e-10, axis=[1]))\n",
    "trainer = tf.train.AdamOptimizer(learning_rate=0.001)\n",
    "update = trainer.minimize(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## HighwayNet\n",
    "An implementation of a Highway Network as desribed in [Highway Networks](https://arxiv.org/abs/1505.00387)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def highwayUnit(input_layer,i):\n",
    "    with tf.variable_scope(\"highway_unit\"+str(i)):\n",
    "        H = slim.conv2d(input_layer,64,[3,3])\n",
    "        T = slim.conv2d(input_layer,64,[3,3], #We initialize with a negative bias to push the network to use the skip connection\n",
    "            biases_initializer=tf.constant_initializer(-1.0),activation_fn=tf.nn.sigmoid)\n",
    "        output = H*T + input_layer*(1.0-T)\n",
    "        return output\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "input_layer = tf.placeholder(shape=[None,32,32,3],dtype=tf.float32,name='input')\n",
    "label_layer = tf.placeholder(shape=[None],dtype=tf.int32)\n",
    "label_oh = slim.layers.one_hot_encoding(label_layer,10)\n",
    "\n",
    "layer1 = slim.conv2d(input_layer,64,[3,3],normalizer_fn=slim.batch_norm,scope='conv_'+str(0))\n",
    "for i in range(5):\n",
    "    for j in range(units_between_stride):\n",
    "        layer1 = highwayUnit(layer1,j + (i*units_between_stride))\n",
    "    layer1 = slim.conv2d(layer1,64,[3,3],stride=[2,2],normalizer_fn=slim.batch_norm,scope='conv_s_'+str(i))\n",
    "    \n",
    "top = slim.conv2d(layer1,10,[3,3],normalizer_fn=slim.batch_norm,activation_fn=None,scope='conv_top')\n",
    "\n",
    "output = slim.layers.softmax(slim.layers.flatten(top))\n",
    "\n",
    "loss = tf.reduce_mean(-tf.reduce_sum(label_oh * tf.log(output) + 1e-10, axis=[1]))\n",
    "trainer = tf.train.AdamOptimizer(learning_rate=0.001)\n",
    "update = trainer.minimize(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DenseNet\n",
    "An implementation of a Dense Network as described in [Densely Connected Convolutional Networks](https://arxiv.org/abs/1608.06993)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def denseBlock(input_layer,i,j):\n",
    "    with tf.variable_scope(\"dense_unit\"+str(i)):\n",
    "        nodes = []\n",
    "        a = slim.conv2d(input_layer,64,[3,3],normalizer_fn=slim.batch_norm)\n",
    "        nodes.append(a)\n",
    "        for z in range(j):\n",
    "            b = slim.conv2d(tf.concat(nodes,3),64,[3,3],normalizer_fn=slim.batch_norm)\n",
    "            nodes.append(b)\n",
    "        return b\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "input_layer = tf.placeholder(shape=[None,32,32,3],dtype=tf.float32,name='input')\n",
    "label_layer = tf.placeholder(shape=[None],dtype=tf.int32)\n",
    "label_oh = slim.layers.one_hot_encoding(label_layer,10)\n",
    "\n",
    "layer1 = slim.conv2d(input_layer,64,[3,3],normalizer_fn=slim.batch_norm,scope='conv_'+str(0))\n",
    "for i in range(5):\n",
    "    layer1 = denseBlock(layer1,i,units_between_stride)\n",
    "    layer1 = slim.conv2d(layer1,64,[3,3],stride=[2,2],normalizer_fn=slim.batch_norm,scope='conv_s_'+str(i))\n",
    "    \n",
    "top = slim.conv2d(layer1,10,[3,3],normalizer_fn=slim.batch_norm,activation_fn=None,scope='conv_top')\n",
    "\n",
    "output = slim.layers.softmax(slim.layers.flatten(top))\n",
    "\n",
    "loss = tf.reduce_mean(-tf.reduce_sum(label_oh * tf.log(output) + 1e-10, axis=[1]))\n",
    "trainer = tf.train.AdamOptimizer(learning_rate=0.001)\n",
    "update = trainer.minimize(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize the network graph\n",
    "We can call the Tensorflow Board to provide a graphical representation of our network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from IPython.display import clear_output, Image, display, HTML\n",
    "\n",
    "def strip_consts(graph_def, max_const_size=32):\n",
    "    \"\"\"Strip large constant values from graph_def.\"\"\"\n",
    "    strip_def = tf.GraphDef()\n",
    "    for n0 in graph_def.node:\n",
    "        n = strip_def.node.add() \n",
    "        n.MergeFrom(n0)\n",
    "        if n.op == 'Const':\n",
    "            tensor = n.attr['value'].tensor\n",
    "            size = len(tensor.tensor_content)\n",
    "            if size > max_const_size:\n",
    "                tensor.tensor_content = \"<stripped %d bytes>\"%size\n",
    "    return strip_def\n",
    "\n",
    "def show_graph(graph_def, max_const_size=32):\n",
    "    \"\"\"Visualize TensorFlow graph.\"\"\"\n",
    "    if hasattr(graph_def, 'as_graph_def'):\n",
    "        graph_def = graph_def.as_graph_def()\n",
    "    strip_def = strip_consts(graph_def, max_const_size=max_const_size)\n",
    "    code = \"\"\"\n",
    "        <script>\n",
    "          function load() {{\n",
    "            document.getElementById(\"{id}\").pbtxt = {data};\n",
    "          }}\n",
    "        </script>\n",
    "        <link rel=\"import\" href=\"https://tensorboard.appspot.com/tf-graph-basic.build.html\" onload=load()>\n",
    "        <div style=\"height:600px\">\n",
    "          <tf-graph-basic id=\"{id}\"></tf-graph-basic>\n",
    "        </div>\n",
    "    \"\"\".format(data=repr(str(strip_def)), id='graph'+str(np.random.rand()))\n",
    "\n",
    "    iframe = \"\"\"\n",
    "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"{}\"></iframe>\n",
    "    \"\"\".format(code.replace('\"', '&quot;'))\n",
    "    display(HTML(iframe))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "show_graph(tf.get_default_graph().as_graph_def())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "batch_size = 64\n",
    "currentCifar = 1\n",
    "total_steps = 20000\n",
    "l = []\n",
    "a = []\n",
    "aT = []\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    i = 0\n",
    "    draw = range(10000)\n",
    "    while i < total_steps:\n",
    "        if i % (10000/batch_size) != 0:\n",
    "            batch_index = np.random.choice(draw,size=batch_size,replace=False)\n",
    "        else:\n",
    "            draw = range(10000)\n",
    "            if currentCifar == 5:\n",
    "                currentCifar = 1\n",
    "                print \"Switched CIFAR set to \" + str(currentCifar)\n",
    "            else:\n",
    "                currentCifar = currentCifar + 1\n",
    "                print \"Switched CIFAR set to \" + str(currentCifar)\n",
    "            cifar = unpickle('./cifar10/data_batch_'+str(currentCifar))\n",
    "            batch_index = np.random.choice(draw,size=batch_size,replace=False)\n",
    "        x = cifar['data'][batch_index]\n",
    "        x = np.reshape(x,[batch_size,32,32,3],order='F')\n",
    "        x = (x/256.0)\n",
    "        x = (x - np.mean(x,axis=0)) / np.std(x,axis=0)\n",
    "        y = np.reshape(np.array(cifar['labels'])[batch_index],[batch_size,1])\n",
    "        _,lossA,yP,LO = sess.run([update,loss,output,label_oh],feed_dict={input_layer:x,label_layer:np.hstack(y)})\n",
    "        accuracy = np.sum(np.equal(np.hstack(y),np.argmax(yP,1)))/float(len(y))\n",
    "        l.append(lossA)\n",
    "        a.append(accuracy)\n",
    "        if i % 10 == 0: print \"Step: \" + str(i) + \" Loss: \" + str(lossA) + \" Accuracy: \" + str(accuracy)\n",
    "        if i % 100 == 0: \n",
    "            point = np.random.randint(0,10000-500)\n",
    "            xT = cifarT['data'][point:point+500]\n",
    "            xT = np.reshape(xT,[500,32,32,3],order='F')\n",
    "            xT = (xT/256.0)\n",
    "            xT = (xT - np.mean(xT,axis=0)) / np.std(xT,axis=0)\n",
    "            yT = np.reshape(np.array(cifarT['labels'])[point:point+500],[500])\n",
    "            lossT,yP = sess.run([loss,output],feed_dict={input_layer:xT,label_layer:yT})\n",
    "            accuracy = np.sum(np.equal(yT,np.argmax(yP,1)))/float(len(yT))\n",
    "            aT.append(accuracy)\n",
    "            print \"Test set accuracy: \" + str(accuracy)\n",
    "        i+= 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.plot(l) #Plot training loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.plot(a) #Plot training accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.plot(aT) #Plot test accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.max(aT) #Best test accuracy"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}