MNIST_CNN_working.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Start TensorFlow Interactive Session - builds a computation graph that runs with the graph \n",
    "import tensorflow as tf\n",
    "sess = tf.InteractiveSession() \n",
    "\n",
    "# Check version of TensorFlow\n",
    "tf_version = tf.__version__\n",
    "print(\"Current version of Tensorflow is %s. \" % tf_version) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import libraries for plotting \n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "print(\"Current version of numpy is 1.12.1\") \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#Load MNIST data stored in NumPy Arrays \n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets('MNIST_data', one_hot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### Plotting images of MNIST Data:\n",
    "\n",
    "From the one-hot vectors, we can find the true label of an image by taking the max value of the one-hot vector using numpy.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# MNIST image constants \n",
    "img_size = 28                # 28x28 pixelsin dimensions\n",
    "img_size_flat = 784          # 1D array\n",
    "img_shape = (28, 28)         # tuple used to reshape arrays \n",
    "num_channels = 1             # greyscale = 1 \n",
    "num_classes = 10             # 10 classes for 10 digits \n",
    "\n",
    "# Max value of the one-hot vector using np \n",
    "mnist.test.cls = np.argmax(mnist.test.labels, axis=1)\n",
    "\n",
    "def plot_images(images, cls_true, cls_pred=None):\n",
    "    assert len(images) == len(cls_true) == 9\n",
    "    \n",
    "    # Create figure with 3x3 sub-plots.\n",
    "    fig, axes = plt.subplots(3, 3)\n",
    "    fig.subplots_adjust(hspace=0.3, wspace=0.3)\n",
    "\n",
    "    for i, ax in enumerate(axes.flat):\n",
    "        # Plot image.\n",
    "        ax.imshow(images[i].reshape(img_shape), cmap='binary')\n",
    "\n",
    "        # Show true and predicted classes.\n",
    "        if cls_pred is None:\n",
    "            xlabel = \"True: {0}\".format(cls_true[i])\n",
    "        else:\n",
    "            xlabel = \"True: {0}, Pred: {1}\".format(cls_true[i], cls_pred[i])\n",
    "\n",
    "        # Show the classes as the label on the x-axis.\n",
    "        ax.set_xlabel(xlabel)\n",
    "        \n",
    "        # Remove ticks from the plot.\n",
    "        ax.set_xticks([])\n",
    "        ax.set_yticks([])\n",
    "    \n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the first images from the test-set.\n",
    "images = mnist.test.images[0:9]\n",
    "\n",
    "# Get the true classes for those images.\n",
    "cls_true = mnist.test.cls[0:9]\n",
    "\n",
    "# Plot the images and labels using our helper-function above.\n",
    "plot_images(images=images, cls_true=cls_true)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 1: SoftMax Regression Model \n",
    "\n",
    "First start with softmax regression model with a single linear layer. Build a computational graph by creating nodes. The input images node is a placeholder (x) which is a 2D tensor of floats. The target output node is a placeholder (y_) which is a 2D tensor for storing the image and its one-hot 10D vector."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = tf.placeholder(tf.float32, shape=[None, 784]) # 784 = number of pixels in a 28x28 MNIST image \n",
    "y_ = tf.placeholder(tf.float32, shape=[None, 10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check the datatype of our placeholders x and y."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the model parameters for the model - the variables that are modified by the computational graph. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Weights W and biases b \n",
    "W = tf.Variable(tf.zeros([784,10]))\n",
    "b = tf.Variable(tf.zeros([10]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Initalize the variables W and b so they can be used within a session.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use a linear regression model to predict output value (y)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y = tf.matmul(x,W) + b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add a cross entropy loss function where the true probability is the true label (y_), and the given distribution is the predicted value (y).\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to train the model, we need to reduce the cost/loss. In this case, the cross entropy loss function can be minimized using the steepest gradient descent algorithm with a 0.5 step size. The function train_step updates parameters repeatedly as it computes gradients, causing the parameter to update steps, and applies update steps to parameters.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Train the model\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# The function feed_dict replaces placeholders (x, y_) with training examples \n",
    "for _ in range(1000):\n",
    "  batch = mnist.train.next_batch(100)\n",
    "  train_step.run(feed_dict={x: batch[0], y_: batch[1]})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluate our softmax regression model. Compare the true and predicted labels. Casting is required since correct_prediction gives us list of booleans but we want a float.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Find max entry along every row using argmax where y =  predicted label, y_ = true label\n",
    "correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "answer1 = accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels})\n",
    "print ('Accuracy of our softmax regression model in making predictions: %1.4f.' % answer1 )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 2:  Multilayer Convolutional Network \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Helper functions weight_Variable and bias_variable are implemented for readability since we will be calculating the weights and bias for multiple layers.\n",
    "\n",
    "def weight_variable(shape):\n",
    "  initial = tf.truncated_normal(shape, stddev=0.1)\n",
    "  return tf.Variable(initial)\n",
    "\n",
    "def bias_variable(shape):\n",
    "  initial = tf.constant(0.1, shape=shape)\n",
    "  return tf.Variable(initial)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Convolutions of stride 1 and no padding\n",
    "def conv2d(x, W):\n",
    "  return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')\n",
    "\n",
    "# Max pooling over 2x2 blocks\n",
    "def max_pool_2x2(x):\n",
    "  return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],\n",
    "                        strides=[1, 2, 2, 1], padding='SAME')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The first convolutional layer outputs 32 features from the input. It does this for each 5x5 patch. The image size gets reduced to 14x14.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# first convolutional layer and bias vector with a component for each output channel\n",
    "W_conv1 = weight_variable([5, 5, 1, 32])\n",
    "b_conv1 = bias_variable([32])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Reshape x to a 4D tensor \n",
    "x_image = tf.reshape(x, [-1, 28, 28, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Convolve x_image with weight tensor, add bias, then apply the ReLU function and max pool \n",
    "h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)\n",
    "h_pool1 = max_pool_2x2(h_conv1) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The second convolutional layer computes 64 features from the input. It does this for each 5x5 patch. The image size gets reduced to 7x7.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "W_conv2 = weight_variable([5, 5, 32, 64])\n",
    "b_conv2 = bias_variable([64])\n",
    "h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)\n",
    "h_pool2 = max_pool_2x2(h_conv2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The densely connected layer processes the entire image by adding a fully-connected layer with 1024 neurons. The tensor is reshaped from the pooling layer into a batch of vectors, then multiplied by the weight matrix. Bias is added and then ReLU is applied. \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 1024 neurons \n",
    "W_fc1 = weight_variable([7 * 7 * 64, 1024])\n",
    "b_fc1 = bias_variable([1024])\n",
    "h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64])\n",
    "h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To reduce overfitting, dropout is applied before the readout layer. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Dropout is on during training and off during testing \n",
    "keep_prob = tf.placeholder(tf.float32)\n",
    "h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Readout layer \n",
    "W_fc2 = weight_variable([1024, 10])\n",
    "b_fc2 = bias_variable([10])\n",
    "y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To train and evaluate the model for a CNN , changes were made from the softmax regression model above. Specifically,  we have an ADAM optimizer instead of a gradient descent optimizer, the dropout rate is controlled by a new parameter \"keep_prob\", and logging is added to every 100th iteration during training. Here, tf.Session is used rather than tf.InteractiveSession for readability purposes to seperate the model specification and model fitting phases. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 0, training accuracy 0.12\n",
      "step 100, training accuracy 0.72\n",
      "step 200, training accuracy 0.92\n",
      "step 300, training accuracy 0.96\n",
      "step 400, training accuracy 0.94\n",
      "step 500, training accuracy 0.92\n",
      "step 600, training accuracy 0.9\n",
      "step 700, training accuracy 0.96\n",
      "step 800, training accuracy 0.9\n",
      "step 900, training accuracy 0.96\n",
      "step 1000, training accuracy 0.96\n",
      "step 1100, training accuracy 0.96\n",
      "step 1200, training accuracy 0.94\n",
      "step 1300, training accuracy 0.98\n",
      "step 1400, training accuracy 0.98\n",
      "step 1500, training accuracy 0.96\n",
      "step 1600, training accuracy 1\n",
      "step 1700, training accuracy 1\n",
      "step 1800, training accuracy 0.98\n",
      "step 1900, training accuracy 1\n",
      "step 2000, training accuracy 0.98\n",
      "step 2100, training accuracy 1\n",
      "step 2200, training accuracy 0.98\n",
      "step 2300, training accuracy 0.96\n",
      "step 2400, training accuracy 1\n",
      "step 2500, training accuracy 0.98\n",
      "step 2600, training accuracy 1\n",
      "step 2700, training accuracy 0.92\n",
      "step 2800, training accuracy 0.96\n",
      "step 2900, training accuracy 0.96\n",
      "step 3000, training accuracy 1\n",
      "step 3100, training accuracy 0.96\n",
      "step 3200, training accuracy 1\n",
      "step 3300, training accuracy 0.94\n",
      "step 3400, training accuracy 0.98\n",
      "step 3500, training accuracy 1\n",
      "step 3600, training accuracy 0.98\n",
      "step 3700, training accuracy 0.98\n",
      "step 3800, training accuracy 1\n",
      "step 3900, training accuracy 1\n",
      "step 4000, training accuracy 1\n",
      "step 4100, training accuracy 0.94\n",
      "step 4200, training accuracy 0.96\n",
      "step 4300, training accuracy 1\n",
      "step 4400, training accuracy 1\n",
      "step 4500, training accuracy 0.98\n",
      "step 4600, training accuracy 0.98\n",
      "step 4700, training accuracy 0.96\n",
      "step 4800, training accuracy 0.96\n",
      "step 4900, training accuracy 1\n",
      "step 5000, training accuracy 0.98\n",
      "step 5100, training accuracy 1\n",
      "step 5200, training accuracy 1\n",
      "step 5300, training accuracy 1\n",
      "step 5400, training accuracy 0.98\n",
      "step 5500, training accuracy 1\n",
      "step 5600, training accuracy 1\n",
      "step 5700, training accuracy 1\n",
      "step 5800, training accuracy 1\n",
      "step 5900, training accuracy 1\n",
      "step 6000, training accuracy 1\n",
      "step 6100, training accuracy 1\n",
      "step 6200, training accuracy 1\n",
      "step 6300, training accuracy 1\n",
      "step 6400, training accuracy 1\n",
      "step 6500, training accuracy 1\n",
      "step 6600, training accuracy 0.98\n",
      "step 6700, training accuracy 1\n",
      "step 6800, training accuracy 1\n",
      "step 6900, training accuracy 1\n",
      "step 7000, training accuracy 1\n",
      "step 7100, training accuracy 1\n",
      "step 7200, training accuracy 1\n",
      "step 7300, training accuracy 1\n",
      "step 7400, training accuracy 1\n",
      "step 7500, training accuracy 1\n",
      "step 7600, training accuracy 1\n",
      "step 7700, training accuracy 1\n",
      "step 7800, training accuracy 1\n",
      "step 7900, training accuracy 1\n",
      "step 8000, training accuracy 1\n",
      "step 8100, training accuracy 1\n",
      "step 8200, training accuracy 1\n",
      "step 8300, training accuracy 1\n",
      "step 8400, training accuracy 1\n",
      "step 8500, training accuracy 1\n",
      "step 8600, training accuracy 1\n",
      "step 8700, training accuracy 1\n",
      "step 8800, training accuracy 1\n",
      "step 8900, training accuracy 1\n",
      "step 9000, training accuracy 1\n",
      "step 9100, training accuracy 0.98\n",
      "step 9200, training accuracy 1\n",
      "step 9300, training accuracy 1\n",
      "step 9400, training accuracy 0.98\n",
      "step 9500, training accuracy 1\n",
      "step 9600, training accuracy 1\n",
      "step 9700, training accuracy 1\n",
      "step 9800, training accuracy 1\n",
      "step 9900, training accuracy 1\n",
      "step 10000, training accuracy 1\n",
      "step 10100, training accuracy 1\n",
      "step 10200, training accuracy 1\n",
      "step 10300, training accuracy 1\n",
      "step 10400, training accuracy 1\n",
      "step 10500, training accuracy 1\n",
      "step 10600, training accuracy 1\n",
      "step 10700, training accuracy 1\n",
      "step 10800, training accuracy 1\n",
      "step 10900, training accuracy 1\n",
      "step 11000, training accuracy 1\n",
      "step 11100, training accuracy 1\n",
      "step 11200, training accuracy 1\n",
      "step 11300, training accuracy 1\n",
      "step 11400, training accuracy 1\n",
      "step 11500, training accuracy 1\n",
      "step 11600, training accuracy 1\n",
      "step 11700, training accuracy 1\n",
      "step 11800, training accuracy 1\n",
      "step 11900, training accuracy 1\n",
      "step 12000, training accuracy 1\n",
      "step 12100, training accuracy 0.98\n",
      "step 12200, training accuracy 1\n",
      "step 12300, training accuracy 1\n",
      "step 12400, training accuracy 0.98\n",
      "step 12500, training accuracy 1\n",
      "step 12600, training accuracy 1\n",
      "step 12700, training accuracy 1\n",
      "step 12800, training accuracy 1\n",
      "step 12900, training accuracy 1\n",
      "step 13000, training accuracy 1\n",
      "step 13100, training accuracy 1\n",
      "step 13200, training accuracy 1\n",
      "step 13300, training accuracy 1\n",
      "step 13400, training accuracy 1\n",
      "step 13500, training accuracy 1\n",
      "step 13600, training accuracy 0.96\n",
      "step 13700, training accuracy 1\n",
      "step 13800, training accuracy 1\n",
      "step 13900, training accuracy 1\n",
      "step 14000, training accuracy 1\n",
      "step 14100, training accuracy 1\n",
      "step 14200, training accuracy 1\n",
      "step 14300, training accuracy 1\n",
      "step 14400, training accuracy 1\n",
      "step 14500, training accuracy 1\n",
      "step 14600, training accuracy 1\n",
      "step 14700, training accuracy 1\n",
      "step 14800, training accuracy 1\n",
      "step 14900, training accuracy 1\n",
      "step 15000, training accuracy 1\n",
      "step 15100, training accuracy 1\n",
      "step 15200, training accuracy 1\n",
      "step 15300, training accuracy 1\n",
      "step 15400, training accuracy 1\n",
      "step 15500, training accuracy 1\n",
      "step 15600, training accuracy 1\n",
      "step 15700, training accuracy 1\n",
      "step 15800, training accuracy 1\n",
      "step 15900, training accuracy 1\n",
      "step 16000, training accuracy 1\n",
      "step 16100, training accuracy 1\n",
      "step 16200, training accuracy 1\n",
      "step 16300, training accuracy 1\n",
      "step 16400, training accuracy 0.98\n",
      "step 16500, training accuracy 1\n",
      "step 16600, training accuracy 1\n",
      "step 16700, training accuracy 1\n",
      "step 16800, training accuracy 1\n",
      "step 16900, training accuracy 1\n",
      "step 17000, training accuracy 1\n",
      "step 17100, training accuracy 1\n",
      "step 17200, training accuracy 1\n",
      "step 17300, training accuracy 1\n",
      "step 17400, training accuracy 1\n",
      "step 17500, training accuracy 1\n",
      "step 17600, training accuracy 1\n",
      "step 17700, training accuracy 1\n",
      "step 17800, training accuracy 0.98\n",
      "step 17900, training accuracy 1\n",
      "step 18000, training accuracy 1\n",
      "step 18100, training accuracy 1\n",
      "step 18200, training accuracy 1\n",
      "step 18300, training accuracy 1\n",
      "step 18400, training accuracy 1\n",
      "step 18500, training accuracy 1\n",
      "step 18600, training accuracy 1\n",
      "step 18700, training accuracy 1\n",
      "step 18800, training accuracy 1\n",
      "step 18900, training accuracy 1\n",
      "step 19000, training accuracy 1\n",
      "step 19100, training accuracy 1\n",
      "step 19200, training accuracy 1\n",
      "step 19300, training accuracy 1\n",
      "step 19400, training accuracy 1\n",
      "step 19500, training accuracy 0.98\n",
      "step 19600, training accuracy 1\n",
      "step 19700, training accuracy 1\n",
      "step 19800, training accuracy 1\n",
      "step 19900, training accuracy 1\n",
      "test accuracy 0.9909\n"
     ]
    }
   ],
   "source": [
    "# Train the model \n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))\n",
    "\n",
    "train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
    "correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "\n",
    "# \"with\" block in python: tf.session is destroyed once block is exited\n",
    "with tf.Session() as sess:\n",
    "  sess.run(tf.global_variables_initializer())\n",
    "  for i in range(20000):\n",
    "    batch = mnist.train.next_batch(50)\n",
    "    if i % 100 == 0:\n",
    "      train_accuracy = accuracy.eval(feed_dict={\n",
    "          x: batch[0], y_: batch[1], keep_prob: 1.0})\n",
    "      print('step %d, training accuracy %g' % (i, train_accuracy))\n",
    "    train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 1.0})\n",
    "\n",
    "  print('test accuracy %g' % accuracy.eval(feed_dict={\n",
    "      x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0}))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [cats]",
   "language": "python",
   "name": "Python [cats]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}