MNIST-Conv2d.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "nbpresent": {
     "id": "e98280a2-2321-4290-9f94-77ee5d39e2f7"
    }
   },
   "source": [
    "# Simple MNIST Model\n",
    "\n",
    "In this notebook we will create a simple model to predicts what letter is written in a 28x28 image based on the [MNIST dataset](http://yann.lecun.com/exdb/mnist/).  This is done via a two fully connected layer Neural Network on a flattened version of the image."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nbpresent": {
     "id": "81f31bb2-0012-405d-acb2-5fc4ceeb463d"
    }
   },
   "source": [
    "## Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "nbpresent": {
     "id": "ea336b2d-4ba4-4054-92ad-bf420b56709a"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download Mnist Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True, reshape=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explore Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_image(image):\n",
    "    plt.imshow(image.reshape((28, 28)), cmap='gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label: [0 0 0 0 1 0 0 0 0 0]\n",
      "Argmax: 4\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAADTJJREFUeJzt3W/MVOWZx/HfTyxvaGPQKhIR6VayWaPR6qMxYf2zqTS4aYJ9UVNIDNpmQYLJor5Y4xuMG4wx2+5uYtIEApYmhbYRUVKbbYnZLG4sIJim2LIUUCgIgSU2lL7QBrj2xXPYPOIz98wzc2bO4PX9JGZmzjUz58rg77nPzH1mbkeEAORzSdMNAGgG4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kNSlg9yZbU4nBPosItzJ/Xoa+W3Ps73X9n7bT/XyXAAGy92e2297kqTfS5or6YiktyUtiIjfFR7DyA/02SBG/jsk7Y+I9yLiL5J+LGl+D88HYIB6Cf81kg6PuX2k2vYJthfb3ml7Zw/7AlCzXj7wG+/Q4lOH9RGxStIqicN+YJj0MvIfkXTtmNszJB3trR0Ag9JL+N+WNNv2l2xPlvQtSZvraQtAv3V92B8RZ2w/JukXkiZJWhsRv62tMwB91fVUX1c74z0/0HcDOckHwMWL8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkBrpEN4bPI488UqyvWLGiWF+4cGGx/tZbb024JwwGIz+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJNXTPL/tg5JOSzor6UxEjNTRFAbn6quvLtZnzpxZrD/55JPFOvP8w6uOk3z+LiJO1vA8AAaIw34gqV7DH5J+aXuX7cV1NARgMHo97J8TEUdtXyVpi+3/iYitY+9Q/VHgDwMwZHoa+SPiaHV5QtImSXeMc59VETHCh4HAcOk6/Lan2P7C+euSvibp3boaA9BfvRz2T5O0yfb551kfEf9RS1cA+q7r8EfEe5JurrEXNOCee+7p6fGXXMKE0cWKfzkgKcIPJEX4gaQIP5AU4QeSIvxAUvx092fcpZeW/4kvu+yynp5/6tSpxXppKvDcuXM97Ru9YeQHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaSY5/+Ma7eE9u23316sHz58uFi/++67i/XZs2e3rO3du7f4WPQXIz+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJMU8/2fAnXfe2bL2+OOPFx/76quvFusvv/xysb5+/fpi/YYbbmhZY56/WYz8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5BU23l+22slfV3SiYi4sdp2uaSfSJol6aCkByPij/1rM7d2v72/ZMmSlrWZM2cWH7t06dJifdasWcV6O72uC4D+6WTk/4GkeRdse0rSGxExW9Ib1W0AF5G24Y+IrZI+vGDzfEnrquvrJD1Qc18A+qzb9/zTIuKYJFWXV9XXEoBB6Pu5/bYXS1rc7/0AmJhuR/7jtqdLUnV5otUdI2JVRIxExEiX+wLQB92Gf7OkRdX1RZJeq6cdAIPSNvy2N0j6laS/tn3E9nckPS9pru19kuZWtwFcRNq+54+IBS1KX625F7Tw3HPPFesPPfRQy9ry5cuLj922bVux3us8/5tvvtnT49E/nOEHJEX4gaQIP5AU4QeSIvxAUoQfSIqf7h4CDz/8cLH+6KOPFusbNmxoWXvxxRe7aak2d911V8vagQMHBtgJLsTIDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJMc8/BG677bZi/dSpU8X66tWr62wHSTDyA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBSzPMPwAsvvFCsL1u2rFhfuXJlsb5169YJ99SpGTNmFOvHjx8v1g8fPlxnO6gRIz+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJNV2nt/2Wklfl3QiIm6stj0j6R8k/W91t6cj4uf9avJit2BBq1XOR61du7ZYf/bZZ+tsZ0LmzJlTrF955ZXF+pkzZ+ps5xOuv/76rh97xRVXFOvbt2/v+rkvFp2M/D+QNG+c7f8aEbdU/xF84CLTNvwRsVXShwPoBcAA9fKe/zHbv7G91vbU2joCMBDdhv/7kr4s6RZJxyR9t9UdbS+2vdP2zi73BaAPugp/RByPiLMRcU7Sakl3FO67KiJGImKk2yYB1K+r8NuePubmNyS9W087AAalk6m+DZLulfRF20ckrZB0r+1bJIWkg5KW9LFHAH3giBjczuzB7WyA5s+fX6xv3LixWP/444+L9V27dhXru3fvblnbv39/8bE33XRTsb5w4cJiffLkycX6gQMHWtbWrFlTfOzNN99crLd73UvOnj1brM+dO7dY37ZtW9f77reIcCf34ww/ICnCDyRF+IGkCD+QFOEHkiL8QFJM9dWg3Vd2X3rppWK93XTZZ9WhQ4eK9RMnThTrr7/+erF++vTplrUtW7YUH7tv375ivd30bJOY6gNQRPiBpAg/kBThB5Ii/EBShB9IivADSTHPPwC33nprsX7//fcPqJNPmzRpUrH+xBNPFOs7duwo1pcuXdqydvLkyeJjT506VaxjfMzzAygi/EBShB9IivADSRF+ICnCDyRF+IGkmOdH0aZNm4r1KVOmFOvz5o23wPOoc+fOddUTypjnB1BE+IGkCD+QFOEHkiL8QFKEH0iK8ANJXdruDravlfRDSVdLOidpVUT8u+3LJf1E0ixJByU9GBF/7F+rGEb33XdfsV5ak+Cjjz6qux1MQCcj/xlJT0bE30i6U9Iy2zdIekrSGxExW9Ib1W0AF4m24Y+IYxHxTnX9tKQ9kq6RNF/Suupu6yQ90K8mAdRvQu/5bc+S9BVJ2yVNi4hj0ugfCElX1d0cgP5p+57/PNufl7RR0vKI+JPd0enDsr1Y0uLu2gPQLx2N/LY/p9Hg/ygiXqk2H7c9vapPlzTuqooRsSoiRiJipI6GAdSjbfg9OsSvkbQnIr43prRZ0qLq+iJJr9XfHoB+6eSwf46khyTttv3ratvTkp6X9FPb35H0B0nf7E+LGGbt3v5dd911LWt79+6tux1MQNvwR8R/S2r1L/zVetsBMCic4QckRfiBpAg/kBThB5Ii/EBShB9IquPTe5HT+++/X6y3++n3Q4cO1dkOasTIDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJMc+Poh07dhTrH3zwQbHOMtzDi5EfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Jyu+9j17oze3A7A5KKiI7W0mPkB5Ii/EBShB9IivADSRF+ICnCDyRF+IGk2obf9rW2/9P2Htu/tf2P1fZnbH9g+9fVf3/f/3YB1KXtST62p0uaHhHv2P6CpF2SHpD0oKQ/R8S/dLwzTvIB+q7Tk3za/pJPRByTdKy6ftr2HknX9NYegKZN6D2/7VmSviJpe7XpMdu/sb3W9tQWj1lse6ftnT11CqBWHZ/bb/vzkv5L0sqIeMX2NEknJYWkf9boW4Nvt3kODvuBPuv0sL+j8Nv+nKSfSfpFRHxvnPosST+LiBvbPA/hB/qsti/22LakNZL2jA1+9UHged+Q9O5EmwTQnE4+7f9bSW9K2i3p/O8wPy1pgaRbNHrYf1DSkurDwdJzMfIDfVbrYX9dCD/Qf3yfH0AR4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+IKm2P+BZs5OSDo25/cVq2zAa1t6GtS+J3rpVZ2/XdXrHgX6f/1M7t3dGxEhjDRQMa2/D2pdEb91qqjcO+4GkCD+QVNPhX9Xw/kuGtbdh7Uuit2410luj7/kBNKfpkR9AQxoJv+15tvfa3m/7qSZ6aMX2Qdu7q5WHG11irFoG7YTtd8dsu9z2Ftv7qstxl0lrqLehWLm5sLJ0o6/dsK14PfDDftuTJP1e0lxJRyS9LWlBRPxuoI20YPugpJGIaHxO2Pbdkv4s6YfnV0Oy/YKkDyPi+eoP59SI+Kch6e0ZTXDl5j711mpl6YfV4GtX54rXdWhi5L9D0v6IeC8i/iLpx5LmN9DH0IuIrZI+vGDzfEnrquvrNPo/z8C16G0oRMSxiHinun5a0vmVpRt97Qp9NaKJ8F8j6fCY20c0XEt+h6Rf2t5le3HTzYxj2vmVkarLqxru50JtV24epAtWlh6a166bFa/r1kT4x1tNZJimHOZExK2S7pe0rDq8RWe+L+nLGl3G7Zik7zbZTLWy9EZJyyPiT032MtY4fTXyujUR/iOSrh1ze4akow30Ma6IOFpdnpC0SaNvU4bJ8fOLpFaXJxru5/9FxPGIOBsR5yStVoOvXbWy9EZJP4qIV6rNjb924/XV1OvWRPjfljTb9pdsT5b0LUmbG+jjU2xPqT6Ike0pkr6m4Vt9eLOkRdX1RZJea7CXTxiWlZtbrSythl+7YVvxupGTfKqpjH+TNEnS2ohYOfAmxmH7rzQ62kuj33hc32RvtjdIulej3/o6LmmFpFcl/VTSTEl/kPTNiBj4B28tertXE1y5uU+9tVpZersafO3qXPG6ln44ww/IiTP8gKQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8k9X/9dtvyinCPXAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7ff3394348d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "index = random.randint(0, len(mnist.train.images))\n",
    "\n",
    "image = mnist.train.images[index]\n",
    "label = mnist.train.labels[index]\n",
    "\n",
    "print('Label: {}\\nArgmax: {}'.format(label.astype(int), np.argmax(label)))\n",
    "\n",
    "show_image(image)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nbpresent": {
     "id": "4829090c-07c5-4964-a84a-701a5b89f294"
    }
   },
   "source": [
    "## Define Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reset graph\n",
    "tf.reset_default_graph()\n",
    "# Set reuse to AUTO_REUSE, this allows you to run tf.get_variable() more than once.\n",
    "tf.get_variable_scope()._reuse = tf.AUTO_REUSE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Learning rate\n",
    "learning_rate = 0.001\n",
    "# Batch Size\n",
    "batch_size = 128\n",
    "# Epochs\n",
    "epochs = 2000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nbpresent": {
     "id": "91eb9701-a56a-4695-ab94-8d9eb2c98206"
    }
   },
   "source": [
    "#### Model Inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "nbpresent": {
     "id": "9229f34c-08e4-4285-b2f0-750cdaa195b0"
    }
   },
   "outputs": [],
   "source": [
    "# Input Images\n",
    "features = tf.placeholder(tf.float32, [None, 28, 28, 1], name='features')\n",
    "\n",
    "# True Values\n",
    "labels = tf.placeholder(tf.float32, [None, 10], name='labels')\n",
    "\n",
    "# Keep Probability\n",
    "keep_prob = tf.placeholder(tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nbpresent": {
     "id": "a12af5e5-d37f-40b3-83e3-4bca21ab0eca"
    }
   },
   "source": [
    "#### Layer 1: Convolution/Max Pool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "nbpresent": {
     "id": "6688439c-cd74-4d5f-9f54-ac8eeb169518"
    }
   },
   "outputs": [],
   "source": [
    "# Shape = 28x28x1\n",
    "\n",
    "# Weights\n",
    "W_1 = tf.get_variable('W_1', shape=[5, 5, 1, 32], initializer=tf.truncated_normal_initializer)\n",
    "# Bias\n",
    "b_1 = tf.get_variable('b_1', shape=[32], initializer=tf.zeros_initializer)\n",
    "# Create Layer\n",
    "layer_1 = tf.nn.conv2d(features, W_1, strides=[1, 1, 1, 1], padding='SAME') + b_1\n",
    "# Activate\n",
    "layer_1 = tf.nn.relu(layer_1)\n",
    "\n",
    "# Shape = 28x28x32\n",
    "\n",
    "# Max Pool\n",
    "layer_1 = tf.nn.max_pool(layer_1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "\n",
    "# Shape = 14x14x32"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Layer 2: Convolution/Max Pool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Shape = 14x14x32\n",
    "\n",
    "# Weights\n",
    "W_2 = tf.get_variable('W_2', shape=[5, 5, 32, 64], initializer=tf.truncated_normal_initializer)\n",
    "# Bias\n",
    "b_2 = tf.get_variable('b_2', shape=[64], initializer=tf.zeros_initializer)\n",
    "# Create Layer\n",
    "layer_2 = tf.nn.conv2d(layer_1, W_2, strides=[1, 1, 1, 1], padding='SAME') + b_2\n",
    "# Activate\n",
    "layer_2 = tf.nn.relu(layer_2)\n",
    "\n",
    "# Shape = 14x14x64\n",
    "\n",
    "# Max Pool\n",
    "layer_2 = tf.nn.max_pool(layer_2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "\n",
    "# Shape = 7x7x64"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Reshape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Shape = 7x7x64\n",
    "\n",
    "flattened = tf.reshape(layer_2, shape=[-1, 7*7*64])\n",
    "\n",
    "# Shape = (7*7*64)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Layer 3: Fully Connected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Weights\n",
    "W_3 = tf.get_variable('W_3', shape=[7*7*64, 1024], initializer=tf.truncated_normal_initializer)\n",
    "# Bias\n",
    "b_3 = tf.get_variable('b_3', shape=[1024], initializer=tf.zeros_initializer)\n",
    "# Create Layer\n",
    "layer_3 = tf.matmul(flattened, W_3) + b_3\n",
    "# Activate\n",
    "layer_3 = tf.nn.relu(layer_3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Layer 4: Fully Connected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Weights\n",
    "W_4 = tf.get_variable('W_4', shape=[1024, 10], initializer=tf.truncated_normal_initializer)\n",
    "# Bias\n",
    "b_4 = tf.get_variable('b_4', shape=[10], initializer=tf.zeros_initializer)\n",
    "# Create Layer\n",
    "layer_4 = tf.matmul(layer_3, W_4) + b_4\n",
    "\n",
    "# No Activation on last layer, since that's covered by Softmax in the loss function.\n",
    "logits = layer_4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Loss Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "nbpresent": {
     "id": "77576169-b0c3-4b16-87fb-1d132a2d715e"
    }
   },
   "outputs": [],
   "source": [
    "# Cross entropy\n",
    "cross_entropys = tf.nn.softmax_cross_entropy_with_logits_v2(labels=labels, logits=logits)\n",
    "\n",
    "average_cross_entropy = tf.reduce_mean(cross_entropys)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optimizer\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "\n",
    "# Training Step\n",
    "train_step = optimizer.minimize(average_cross_entropy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "correct_prediction = tf.equal(tf.argmax(labels, 1), tf.argmax(logits, 1))\n",
    "\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nbpresent": {
     "id": "96345023-61d9-4ed4-bd92-0b2558ae6b6d"
    }
   },
   "source": [
    "## Train Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Saver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "saver = tf.train.Saver()\n",
    "save_path = './checkpoints/conv2d.ckpt'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run Tensorflow Session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:   0 Train Accuracy: 0.188\n",
      "Epoch: 100 Train Accuracy: 0.898\n",
      "Epoch: 200 Train Accuracy: 0.930\n",
      "Epoch: 300 Train Accuracy: 0.961\n",
      "Epoch: 400 Train Accuracy: 0.930\n",
      "Epoch: 500 Train Accuracy: 0.938\n",
      "Epoch: 600 Train Accuracy: 0.977\n",
      "Epoch: 700 Train Accuracy: 0.961\n",
      "Epoch: 800 Train Accuracy: 0.945\n",
      "Epoch: 900 Train Accuracy: 0.977\n",
      "Epoch:1000 Train Accuracy: 0.945\n",
      "Epoch:1100 Train Accuracy: 0.969\n",
      "Epoch:1200 Train Accuracy: 0.984\n",
      "Epoch:1300 Train Accuracy: 0.945\n",
      "Epoch:1400 Train Accuracy: 0.977\n",
      "Epoch:1500 Train Accuracy: 0.969\n",
      "Epoch:1600 Train Accuracy: 0.984\n",
      "Epoch:1700 Train Accuracy: 0.969\n",
      "Epoch:1800 Train Accuracy: 0.961\n",
      "Epoch:1900 Train Accuracy: 0.984\n",
      "Epoch:2000 Train Accuracy: 0.969\n",
      "Test Accuracy: 0.976\n"
     ]
    }
   ],
   "source": [
    "print_every = 100\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    for e in range(epochs+1):\n",
    "        \n",
    "        batch_features, batch_labels = mnist.train.next_batch(batch_size)\n",
    "        \n",
    "        if e % print_every == 0:\n",
    "            acc = sess.run(accuracy, feed_dict={\n",
    "                features: batch_features,\n",
    "                labels: batch_labels\n",
    "            })\n",
    "\n",
    "            print('Epoch:{:4} Train Accuracy: {:.3f}'.format(e, acc))\n",
    "        \n",
    "        acc = sess.run(train_step, feed_dict={\n",
    "            features: batch_features,\n",
    "            labels: batch_labels\n",
    "        })\n",
    "    \n",
    "    \n",
    "    acc = sess.run(accuracy, feed_dict={\n",
    "        features: mnist.test.images,\n",
    "        labels: mnist.test.labels\n",
    "    })\n",
    "    print('Test Accuracy: {:.3f}'.format(acc))\n",
    "    \n",
    "    saver.save(sess, save_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nbpresent": {
     "id": "063c912e-a9b6-4142-865b-6eb4c246a65e"
    }
   },
   "source": [
    "## Prediction Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "nbpresent": {
     "id": "edca66a2-2ab3-4b41-9d7b-12f43d8e6c6a"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./checkpoints/conv2d.ckpt\n",
      "\n",
      "Label: [0 0 0 0 0 0 0 1 0 0]\n",
      "Argmax: 7\n",
      "\n",
      "Logit: [-39703 -15803 -10419   2560 -27520 -24457 -37041  25493 -17513   -403]\n",
      "Argmax: 7\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAADbxJREFUeJzt3W2IHfUVx/HfURPRNGBijA0mNVF8qMSQxCU+xEeq1ZZALBIxL+oWSrcvFCz2RSWgjUhBSlIrvhA2ZG2E1ihoNAapVRFsRcRV89TGPBA3JjVmW1OtilhMTl/spF3j3v+9mTtzZ3bP9wOy9865M3O85Lczd/8z92/uLgDxHFd1AwCqQfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwR1Qid3ZmZcTgiUzN2tlde1deQ3sxvMbLuZ7TKzu9rZFoDOsrzX9pvZ8ZJ2SLpO0j5Jb0ha6u5/S6zDkR8oWSeO/Ask7XL33e7+H0lrJS1uY3sAOqid8J8hae+w5/uyZV9hZj1m1m9m/W3sC0DB2vmD30inFl87rXf3Xkm9Eqf9QJ20c+TfJ2nGsOfTJb3fXjsAOqWd8L8h6Rwzm2Vm4yXdIml9MW0BKFvu0353/9LMbpf0vKTjJfW5+18L6wxAqXIP9eXaGZ/5gdJ15CIfAKMX4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0HlnqJbksxsQNInkg5J+tLdu4poCkD52gp/5hp3/2cB2wHQQZz2A0G1G36X9Ccze9PMeopoCEBntHvav9Dd3zezqZJeMLN33P2V4S/IfinwiwGoGXP3YjZktlzSp+6+IvGaYnYGoCF3t1Zel/u038wmmNnEI48lfVfS1rzbA9BZ7Zz2ny5pnZkd2c4f3P2PhXQFoHSFnfa3tDNO+4HSlX7aD2B0I/xAUIQfCIrwA0ERfiAowg8EVcRdfRjDTjgh/U/krLPOyr3tyy67LFmfP39+sj579uxk/ZprrmlY27NnT3LdBQsWJOuDg4PJ+mjAkR8IivADQRF+ICjCDwRF+IGgCD8QFOEHgmKcfww499xzc9UkacmSJcn6jBkzkvWrrroqWa/S4cOHG9aajdN/9tlnRbdTOxz5gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAoxvkLMG7cuGQ9Nd4sSddee22yfs899yTr559/fsPaKaecklw3qokTJybr48ePT9bHwnUAHPmBoAg/EBThB4Ii/EBQhB8IivADQRF+IKim4/xm1idpkaRBd5+dLZss6XFJMyUNSLrZ3f9VXpvtmzlzZrJ+xRVXJOup75hftGhRct0XX3wxWb/kkkuS9Wb35Jep3fveZ82aVWQ7hWn2/9Xs2oyxoJUj/+8k3XDUsrskveTu50h6KXsOYBRpGn53f0XSwaMWL5a0Jnu8RtKNBfcFoGR5P/Of7u77JSn7ObW4lgB0QunX9ptZj6SesvcD4NjkPfIfMLNpkpT9bPjXE3fvdfcud+/KuS8AJcgb/vWSurPH3ZKeKaYdAJ3SNPxm9pik1ySdZ2b7zOzHku6XdJ2Z7ZR0XfYcwChi7t65nZmVtrNm3y//9ttvJ+uTJk0qsp2Oeu655xrWNmzYkFx3x44dyfq7776brH/00UfJ+rx58xrWpkyZklx37dq1yXozH3zwQcPanDlzkut++OGHbe27Su5urbyOK/yAoAg/EBThB4Ii/EBQhB8IivADQY2Zr+4+6aSTkvV2h/JSQ6Kff/55ct2+vr5k/dlnn03Wt23blqynhrQOHTqUXLdsL7/8csPavffeW+q+X3311Ya10TyUVxSO/EBQhB8IivADQRF+ICjCDwRF+IGgCD8Q1Ji5pffMM89M1jdt2pSsv/baa8l6b29vw9q6deuS60Z26qmnNqzt3bs3ue6JJ57Y1r5TtxNv3ry5rW3XGbf0Akgi/EBQhB8IivADQRF+ICjCDwRF+IGgxsz9/Hv27EnWFy5cmKw3m2p6YGDgWFsK4bTTTkvWn3766Ya1dsfxu7u7k/WtW7e2tf2xjiM/EBThB4Ii/EBQhB8IivADQRF+ICjCDwTV9H5+M+uTtEjSoLvPzpYtl/QTSf/IXrbM3RvPE/3/bXXuywPQEY888kiyfuutt+bedrNrKy699NJkfXBwMPe+R7Mi7+f/naQbRlj+gLvPzf5rGnwA9dI0/O7+iqSDHegFQAe185n/djPbbGZ9ZtbeXFgAOi5v+B+WdLakuZL2S1rZ6IVm1mNm/WbWn3NfAEqQK/zufsDdD7n7YUmrJC1IvLbX3bvcvStvkwCKlyv8ZjZt2NMfSOL2KWCUaXpLr5k9JulqSVPMbJ+kX0q62szmSnJJA5J+WmKPAErQNPzuvnSExatL6AU1dPLJJyfrF154YWn7XrVqVbIedRy/KFzhBwRF+IGgCD8QFOEHgiL8QFCEHwhqzHx1N8oxZ86cZL3Z1Ojt2L59e2nbBkd+ICzCDwRF+IGgCD8QFOEHgiL8QFCEHwiKcX4k3XTTTcn65MmTc2979+7dyfqGDRtybxvNceQHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaCaTtFd6M6Yort2pk6dmqzv2rUrWZ8wYULufc+fPz9Z37RpU+5tR1bkFN0AxiDCDwRF+IGgCD8QFOEHgiL8QFCEHwiq6f38ZjZD0qOSvinpsKRed3/QzCZLelzSTEkDkm5293+V1yrKsHp1erb1dsbxJWnnzp0Na++8805b20Z7Wjnyfynp5+7+bUmXSLrNzC6QdJekl9z9HEkvZc8BjBJNw+/u+939rezxJ5K2STpD0mJJa7KXrZF0Y1lNAijeMX3mN7OZkuZJel3S6e6+Xxr6BSEpfZ0ogFpp+Tv8zOwbkp6U9DN3/7dZS5cPy8x6JPXkaw9AWVo68pvZOA0F//fu/lS2+ICZTcvq0yQNjrSuu/e6e5e7dxXRMIBiNA2/DR3iV0va5u6/GVZaL6k7e9wt6Zni2wNQlqa39JrZ5ZL+LGmLhob6JGmZhj73PyHpW5Lek7TE3Q822Ra39HbYHXfckayvWLEiWT/uuPTxYeXKlcn63Xff3bD2xRdfJNdFPq3e0tv0M7+7/0VSo41951iaAlAfXOEHBEX4gaAIPxAU4QeCIvxAUIQfCIopuseA8847r2HtzjvvTK7bbBy/mf7+/mSdsfz64sgPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0Exzj8GXHDBBQ1r06dPb2vbBw8mv6JBzz//fFvbR3U48gNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIzzjwEXX3xxadt+6KGHkvWPP/64tH2jXBz5gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAoc/f0C8xmSHpU0jclHZbU6+4PmtlyST+R9I/spcvc/bkm20rvDCO68sork/XUPfXjx49va98XXXRRsr5x48a2to/iubu18rpWLvL5UtLP3f0tM5so6U0zeyGrPeDuK/I2CaA6TcPv7vsl7c8ef2Jm2ySdUXZjAMp1TJ/5zWympHmSXs8W3W5mm82sz8wmNVinx8z6zSw9rxOAjmo5/Gb2DUlPSvqZu/9b0sOSzpY0V0NnBitHWs/de929y927CugXQEFaCr+ZjdNQ8H/v7k9JkrsfcPdD7n5Y0ipJC8prE0DRmobfzEzSaknb3P03w5ZPG/ayH0jaWnx7AMrSyl/7F0r6oaQtZnZkXGeZpKVmNleSSxqQ9NNSOoSuv/76ZL2d4bz77rsvWd+yZUvubaPeWvlr/18kjTRumBzTB1BvXOEHBEX4gaAIPxAU4QeCIvxAUIQfCKrpLb2F7oxbeoHStXpLL0d+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiq01N0/1PSnmHPp2TL6qiuvdW1L4ne8iqytzNbfWFHL/L52s7N+uv63X517a2ufUn0lldVvXHaDwRF+IGgqg5/b8X7T6lrb3XtS6K3vCrprdLP/ACqU/WRH0BFKgm/md1gZtvNbJeZ3VVFD42Y2YCZbTGzjVVPMZZNgzZoZluHLZtsZi+Y2c7s54jTpFXU23Iz+3v23m00s+9X1NsMM3vZzLaZ2V/N7I5seaXvXaKvSt63jp/2m9nxknZIuk7SPklvSFrq7n/raCMNmNmApC53r3xM2MyulPSppEfdfXa27NeSDrr7/dkvzknu/oua9LZc0qdVz9ycTSgzbfjM0pJulPQjVfjeJfq6WRW8b1Uc+RdI2uXuu939P5LWSlpcQR+15+6vSDp41OLFktZkj9do6B9PxzXorRbcfb+7v5U9/kTSkZmlK33vEn1VoorwnyFp77Dn+1SvKb9d0p/M7E0z66m6mRGcnk2bfmT69KkV93O0pjM3d9JRM0vX5r3LM+N10aoI/0hfMVSnIYeF7j5f0vck3Zad3qI1Lc3c3CkjzCxdC3lnvC5aFeHfJ2nGsOfTJb1fQR8jcvf3s5+DktapfrMPHzgySWr2c7Difv6nTjM3jzSztGrw3tVpxusqwv+GpHPMbJaZjZd0i6T1FfTxNWY2IftDjMxsgqTvqn6zD6+X1J097pb0TIW9fEVdZm5uNLO0Kn7v6jbjdSUX+WRDGb+VdLykPnf/VcebGIGZnaWho700dMfjH6rszcwek3S1hu76OiDpl5KelvSEpG9Jek/SEnfv+B/eGvR2tYZOXf83c/ORz9gd7u1ySX+WtEXS4WzxMg19vq7svUv0tVQVvG9c4QcExRV+QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeC+i8ivvt8Mo7UhAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7ff336923358>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, save_path)\n",
    "    \n",
    "    index = random.randint(0, len(mnist.train.images))\n",
    "    image = mnist.train.images[index]\n",
    "    label = mnist.train.labels[index]\n",
    "    \n",
    "    logit = sess.run(logits, feed_dict={\n",
    "        features: [image],\n",
    "        labels: [label]\n",
    "    })[0]\n",
    "    \n",
    "    # This is just to make the print statement nicer\n",
    "    label = label.astype(int)\n",
    "    logit = logit.astype(int)\n",
    "\n",
    "    print('\\nLabel: {}\\nArgmax: {}\\n\\nLogit: {}\\nArgmax: {}'.format(label, np.argmax(label), logit, np.argmax(logit)))\n",
    "    \n",
    "    show_image(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow-gpu]",
   "language": "python",
   "name": "conda-env-tensorflow-gpu-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}