Convolutional_Neural_Network-TensorFlow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center><h1> Convolutional Neural Network - TensorFlow </h1><center>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import time\n",
    "import os\n",
    "import cv2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/MNIST/train-images-idx3-ubyte.gz\n",
      "Extracting data/MNIST/train-labels-idx1-ubyte.gz\n",
      "Extracting data/MNIST/t10k-images-idx3-ubyte.gz\n",
      "Extracting data/MNIST/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "data = input_data.read_data_sets('data/MNIST/', one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of:\n",
      "- Training-set:\t\t55000\n",
      "- Test-set:\t\t10000\n",
      "- Validation-set:\t5000\n"
     ]
    }
   ],
   "source": [
    "print(\"Size of:\")\n",
    "print(\"- Training-set:\\t\\t{}\".format(len(data.train.labels)))\n",
    "print(\"- Test-set:\\t\\t{}\".format(len(data.test.labels)))\n",
    "print(\"- Validation-set:\\t{}\".format(len(data.validation.labels)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Placeholder variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Placeholder variable for the input images\n",
    "x = tf.placeholder(tf.float32, shape=[None, 28*28], name='X')\n",
    "# Reshape it into [num_images, img_height, img_width, num_channels]\n",
    "x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
    "\n",
    "# Placeholder variable for the true labels associated with the images\n",
    "y_true = tf.placeholder(tf.float32, shape=[None, 10], name='y_true')\n",
    "y_true_cls = tf.argmax(y_true, dimension=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function for creating a new Convolution Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def new_conv_layer(input, num_input_channels, filter_size, num_filters, name):\n",
    "    \n",
    "    with tf.variable_scope(name) as scope:\n",
    "        # Shape of the filter-weights for the convolution\n",
    "        shape = [filter_size, filter_size, num_input_channels, num_filters]\n",
    "\n",
    "        # Create new weights (filters) with the given shape\n",
    "        weights = tf.Variable(tf.truncated_normal(shape, stddev=0.05))\n",
    "\n",
    "        # Create new biases, one for each filter\n",
    "        biases = tf.Variable(tf.constant(0.05, shape=[num_filters]))\n",
    "\n",
    "        # TensorFlow operation for convolution\n",
    "        layer = tf.nn.conv2d(input=input, filter=weights, strides=[1, 1, 1, 1], padding='SAME')\n",
    "\n",
    "        # Add the biases to the results of the convolution.\n",
    "        layer += biases\n",
    "        \n",
    "        return layer, weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function for creating a new Pooling Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def new_pool_layer(input, name):\n",
    "    \n",
    "    with tf.variable_scope(name) as scope:\n",
    "        # TensorFlow operation for convolution\n",
    "        layer = tf.nn.max_pool(value=input, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "        \n",
    "        return layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function for creating a new ReLU Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def new_relu_layer(input, name):\n",
    "    \n",
    "    with tf.variable_scope(name) as scope:\n",
    "        # TensorFlow operation for convolution\n",
    "        layer = tf.nn.relu(input)\n",
    "        \n",
    "        return layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function for creating a new Fully connected Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def new_fc_layer(input, num_inputs, num_outputs, name):\n",
    "    \n",
    "    with tf.variable_scope(name) as scope:\n",
    "\n",
    "        # Create new weights and biases.\n",
    "        weights = tf.Variable(tf.truncated_normal([num_inputs, num_outputs], stddev=0.05))\n",
    "        biases = tf.Variable(tf.constant(0.05, shape=[num_outputs]))\n",
    "        \n",
    "        # Multiply the input and weights, and then add the bias-values.\n",
    "        layer = tf.matmul(input, weights) + biases\n",
    "        \n",
    "        return layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Convolutional Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Convolutional Layer 1\n",
    "layer_conv1, weights_conv1 = new_conv_layer(input=x_image, num_input_channels=1, filter_size=5, num_filters=6, name =\"conv1\")\n",
    "\n",
    "# Pooling Layer 1\n",
    "layer_pool1 = new_pool_layer(layer_conv1, name=\"pool1\")\n",
    "\n",
    "# RelU layer 1\n",
    "layer_relu1 = new_relu_layer(layer_pool1, name=\"relu1\")\n",
    "\n",
    "# Convolutional Layer 2\n",
    "layer_conv2, weights_conv2 = new_conv_layer(input=layer_relu1, num_input_channels=6, filter_size=5, num_filters=16, name= \"conv2\")\n",
    "\n",
    "# Pooling Layer 2\n",
    "layer_pool2 = new_pool_layer(layer_conv2, name=\"pool2\")\n",
    "\n",
    "# RelU layer 2\n",
    "layer_relu2 = new_relu_layer(layer_pool2, name=\"relu2\")\n",
    "\n",
    "# Flatten Layer\n",
    "num_features = layer_relu2.get_shape()[1:4].num_elements()\n",
    "layer_flat = tf.reshape(layer_relu2, [-1, num_features])\n",
    "\n",
    "# Fully-Connected Layer 1\n",
    "layer_fc1 = new_fc_layer(layer_flat, num_inputs=num_features, num_outputs=128, name=\"fc1\")\n",
    "\n",
    "# RelU layer 3\n",
    "layer_relu3 = new_relu_layer(layer_fc1, name=\"relu3\")\n",
    "\n",
    "# Fully-Connected Layer 2\n",
    "layer_fc2 = new_fc_layer(input=layer_relu3, num_inputs=128, num_outputs=10, name=\"fc2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Use Softmax function to normalize the output\n",
    "with tf.variable_scope(\"Softmax\"):\n",
    "    y_pred = tf.nn.softmax(layer_fc2)\n",
    "    y_pred_cls = tf.argmax(y_pred, dimension=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cost Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Use Cross entropy cost function\n",
    "with tf.name_scope(\"cross_ent\"):\n",
    "    cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=layer_fc2, labels=y_true)\n",
    "    cost = tf.reduce_mean(cross_entropy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Use Adam Optimizer\n",
    "with tf.name_scope(\"optimizer\"):\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(cost)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Accuracy\n",
    "with tf.name_scope(\"accuracy\"):\n",
    "    correct_prediction = tf.equal(y_pred_cls, y_true_cls)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Trainable variables\n",
    "vart_list = tf.trainable_variables()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Initialize the FileWriter\n",
    "writer = tf.summary.FileWriter(\"Training_FileWriter/\")\n",
    "writer1 = tf.summary.FileWriter(\"Validation_FileWriter/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Summary name conv1/Variable:0 is illegal; using conv1/Variable_0 instead.\n",
      "INFO:tensorflow:Summary name conv1/Variable_1:0 is illegal; using conv1/Variable_1_0 instead.\n",
      "INFO:tensorflow:Summary name conv2/Variable:0 is illegal; using conv2/Variable_0 instead.\n",
      "INFO:tensorflow:Summary name conv2/Variable_1:0 is illegal; using conv2/Variable_1_0 instead.\n",
      "INFO:tensorflow:Summary name fc1/Variable:0 is illegal; using fc1/Variable_0 instead.\n",
      "INFO:tensorflow:Summary name fc1/Variable_1:0 is illegal; using fc1/Variable_1_0 instead.\n",
      "INFO:tensorflow:Summary name fc2/Variable:0 is illegal; using fc2/Variable_0 instead.\n",
      "INFO:tensorflow:Summary name fc2/Variable_1:0 is illegal; using fc2/Variable_1_0 instead.\n"
     ]
    }
   ],
   "source": [
    "# Add the cost and accuracy to summary\n",
    "tf.summary.scalar('loss', cost)\n",
    "tf.summary.scalar('accuracy', accuracy)\n",
    "\n",
    "# Add the trainable variables to summary \n",
    "for var in vart_list:\n",
    "    tf.summary.histogram(var.name, var)\n",
    "\n",
    "# Merge all summaries together\n",
    "merged_summary = tf.summary.merge_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_epochs = 100\n",
    "batch_size = 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TensorFlow Session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 completed : Time usage 4207 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.7146727267585018\n",
      "\t- Validation Accuracy:\t0.8826000094413757\n",
      "Epoch 2 completed : Time usage 104 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9010181837732142\n",
      "\t- Validation Accuracy:\t0.9214000105857849\n",
      "Epoch 3 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.925072729695927\n",
      "\t- Validation Accuracy:\t0.9380000233650208\n",
      "Epoch 4 completed : Time usage 96 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.941181819005446\n",
      "\t- Validation Accuracy:\t0.9495999813079834\n",
      "Epoch 5 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9519272735985843\n",
      "\t- Validation Accuracy:\t0.9580000042915344\n",
      "Epoch 6 completed : Time usage 97 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9599636405164546\n",
      "\t- Validation Accuracy:\t0.9667999744415283\n",
      "Epoch 7 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9657090966268019\n",
      "\t- Validation Accuracy:\t0.9700000286102295\n",
      "Epoch 8 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9696363708105954\n",
      "\t- Validation Accuracy:\t0.9721999764442444\n",
      "Epoch 9 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9728727354786613\n",
      "\t- Validation Accuracy:\t0.9751999974250793\n",
      "Epoch 10 completed : Time usage 96 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9748181918534365\n",
      "\t- Validation Accuracy:\t0.9764000177383423\n",
      "Epoch 11 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9776727378368377\n",
      "\t- Validation Accuracy:\t0.9765999913215637\n",
      "Epoch 12 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.978654556166042\n",
      "\t- Validation Accuracy:\t0.978600025177002\n",
      "Epoch 13 completed : Time usage 97 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9800545558062467\n",
      "\t- Validation Accuracy:\t0.9796000123023987\n",
      "Epoch 14 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9810181923346086\n",
      "\t- Validation Accuracy:\t0.9797999858856201\n",
      "Epoch 15 completed : Time usage 102 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9823454656384208\n",
      "\t- Validation Accuracy:\t0.9818000197410583\n",
      "Epoch 16 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.983581829179417\n",
      "\t- Validation Accuracy:\t0.9818000197410583\n",
      "Epoch 17 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9838363734158603\n",
      "\t- Validation Accuracy:\t0.9824000000953674\n",
      "Epoch 18 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9845454646240581\n",
      "\t- Validation Accuracy:\t0.984000027179718\n",
      "Epoch 19 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9856181926077062\n",
      "\t- Validation Accuracy:\t0.9843999743461609\n",
      "Epoch 20 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9858181927420876\n",
      "\t- Validation Accuracy:\t0.9847999811172485\n",
      "Epoch 21 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9867090996828947\n",
      "\t- Validation Accuracy:\t0.9850000143051147\n",
      "Epoch 22 completed : Time usage 97 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9870909188010476\n",
      "\t- Validation Accuracy:\t0.98580002784729\n",
      "Epoch 23 completed : Time usage 96 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9878181920268319\n",
      "\t- Validation Accuracy:\t0.9865999817848206\n",
      "Epoch 24 completed : Time usage 97 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9882909178733825\n",
      "\t- Validation Accuracy:\t0.9855999946594238\n",
      "Epoch 25 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9882727362892845\n",
      "\t- Validation Accuracy:\t0.9868000149726868\n",
      "Epoch 26 completed : Time usage 97 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9892000080238689\n",
      "\t- Validation Accuracy:\t0.9876000285148621\n",
      "Epoch 27 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9895636447993191\n",
      "\t- Validation Accuracy:\t0.9869999885559082\n",
      "Epoch 28 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9898727349801497\n",
      "\t- Validation Accuracy:\t0.9873999953269958\n",
      "Epoch 29 completed : Time usage 97 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9902000084790317\n",
      "\t- Validation Accuracy:\t0.9829999804496765\n",
      "Epoch 30 completed : Time usage 96 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9906000085310502\n",
      "\t- Validation Accuracy:\t0.9873999953269958\n",
      "Epoch 31 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9912727351622148\n",
      "\t- Validation Accuracy:\t0.9876000285148621\n",
      "Epoch 32 completed : Time usage 101 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.991363643732938\n",
      "\t- Validation Accuracy:\t0.9882000088691711\n",
      "Epoch 33 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9914727346463637\n",
      "\t- Validation Accuracy:\t0.9846000075340271\n",
      "Epoch 34 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9919272795590487\n",
      "\t- Validation Accuracy:\t0.9887999892234802\n",
      "Epoch 35 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9926363703337583\n",
      "\t- Validation Accuracy:\t0.9876000285148621\n",
      "Epoch 36 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9924909158186479\n",
      "\t- Validation Accuracy:\t0.9890000224113464\n",
      "Epoch 37 completed : Time usage 96 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9927454608136957\n",
      "\t- Validation Accuracy:\t0.989799976348877\n",
      "Epoch 38 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9929818246581338\n",
      "\t- Validation Accuracy:\t0.9879999756813049\n",
      "Epoch 39 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9931454608657143\n",
      "\t- Validation Accuracy:\t0.9872000217437744\n",
      "Epoch 40 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.993709096475081\n",
      "\t- Validation Accuracy:\t0.9883999824523926\n",
      "Epoch 41 completed : Time usage 97 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.993709096475081\n",
      "\t- Validation Accuracy:\t0.9843999743461609\n",
      "Epoch 42 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9942181873321533\n",
      "\t- Validation Accuracy:\t0.9882000088691711\n",
      "Epoch 43 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9944909141280435\n",
      "\t- Validation Accuracy:\t0.9890000224113464\n",
      "Epoch 44 completed : Time usage 97 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.994581823348999\n",
      "\t- Validation Accuracy:\t0.9878000020980835\n",
      "Epoch 45 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9947636411406777\n",
      "\t- Validation Accuracy:\t0.9879999756813049\n",
      "Epoch 46 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9946363686431537\n",
      "\t- Validation Accuracy:\t0.989799976348877\n",
      "Epoch 47 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9948909138549458\n",
      "\t- Validation Accuracy:\t0.9891999959945679\n",
      "Epoch 48 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.995600004196167\n",
      "\t- Validation Accuracy:\t0.9883999824523926\n",
      "Epoch 49 completed : Time usage 97 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9956909130920063\n",
      "\t- Validation Accuracy:\t0.9883999824523926\n",
      "Epoch 50 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.996236367225647\n",
      "\t- Validation Accuracy:\t0.9890000224113464\n",
      "Epoch 51 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9960909127105366\n",
      "\t- Validation Accuracy:\t0.9896000027656555\n",
      "Epoch 52 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.99580000389706\n",
      "\t- Validation Accuracy:\t0.9887999892234802\n",
      "Epoch 53 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9965636396408081\n",
      "\t- Validation Accuracy:\t0.9883999824523926\n",
      "Epoch 54 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9962181853164326\n",
      "\t- Validation Accuracy:\t0.9896000027656555\n",
      "Epoch 55 completed : Time usage 100 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9965636396408081\n",
      "\t- Validation Accuracy:\t0.989799976348877\n",
      "Epoch 56 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9970727300643921\n",
      "\t- Validation Accuracy:\t0.9901999831199646\n",
      "Epoch 57 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.997381820678711\n",
      "\t- Validation Accuracy:\t0.9890000224113464\n",
      "Epoch 58 completed : Time usage 100 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9970363664627075\n",
      "\t- Validation Accuracy:\t0.9887999892234802\n",
      "Epoch 59 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9973454570770264\n",
      "\t- Validation Accuracy:\t0.9873999953269958\n",
      "Epoch 60 completed : Time usage 102 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9977090930938721\n",
      "\t- Validation Accuracy:\t0.9890000224113464\n",
      "Epoch 61 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.997327275167812\n",
      "\t- Validation Accuracy:\t0.9882000088691711\n",
      "Epoch 62 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9975454568862915\n",
      "\t- Validation Accuracy:\t0.9887999892234802\n",
      "Epoch 63 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9981454563140869\n",
      "\t- Validation Accuracy:\t0.989799976348877\n",
      "Epoch 64 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9979818201065064\n",
      "\t- Validation Accuracy:\t0.989799976348877\n",
      "Epoch 65 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9980000019073486\n",
      "\t- Validation Accuracy:\t0.9869999885559082\n",
      "Epoch 66 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9983272743225098\n",
      "\t- Validation Accuracy:\t0.9876000285148621\n",
      "Epoch 67 completed : Time usage 100 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9984545469284057\n",
      "\t- Validation Accuracy:\t0.9890000224113464\n",
      "Epoch 68 completed : Time usage 100 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9981636381149293\n",
      "\t- Validation Accuracy:\t0.9887999892234802\n",
      "Epoch 69 completed : Time usage 101 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9984363651275635\n",
      "\t- Validation Accuracy:\t0.9894000291824341\n",
      "Epoch 70 completed : Time usage 102 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9984363651275635\n",
      "\t- Validation Accuracy:\t0.9894000291824341\n",
      "Epoch 71 completed : Time usage 114 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9986545467376708\n",
      "\t- Validation Accuracy:\t0.9886000156402588\n",
      "Epoch 72 completed : Time usage 100 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9987636375427246\n",
      "\t- Validation Accuracy:\t0.9890000224113464\n",
      "Epoch 73 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9988000011444091\n",
      "\t- Validation Accuracy:\t0.9890000224113464\n",
      "Epoch 74 completed : Time usage 124 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9988363647460937\n",
      "\t- Validation Accuracy:\t0.9891999959945679\n",
      "Epoch 75 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9990181827545166\n",
      "\t- Validation Accuracy:\t0.9896000027656555\n",
      "Epoch 76 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9986909103393554\n",
      "\t- Validation Accuracy:\t0.989799976348877\n",
      "Epoch 77 completed : Time usage 97 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9990181827545166\n",
      "\t- Validation Accuracy:\t0.9891999959945679\n",
      "Epoch 78 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.998981819152832\n",
      "\t- Validation Accuracy:\t0.9878000020980835\n",
      "Epoch 79 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9990909099578857\n",
      "\t- Validation Accuracy:\t0.9890000224113464\n",
      "Epoch 80 completed : Time usage 100 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9990363645553589\n",
      "\t- Validation Accuracy:\t0.9879999756813049\n",
      "Epoch 81 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9992545461654663\n",
      "\t- Validation Accuracy:\t0.9886000156402588\n",
      "Epoch 82 completed : Time usage 102 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.999236364364624\n",
      "\t- Validation Accuracy:\t0.9891999959945679\n",
      "Epoch 83 completed : Time usage 102 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9993090915679932\n",
      "\t- Validation Accuracy:\t0.9868000149726868\n",
      "Epoch 84 completed : Time usage 101 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.999236364364624\n",
      "\t- Validation Accuracy:\t0.9886000156402588\n",
      "Epoch 85 completed : Time usage 169 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9992545461654663\n",
      "\t- Validation Accuracy:\t0.9890000224113464\n",
      "Epoch 86 completed : Time usage 369 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9992727279663086\n",
      "\t- Validation Accuracy:\t0.9879999756813049\n",
      "Epoch 87 completed : Time usage 199 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9995090913772583\n",
      "\t- Validation Accuracy:\t0.9904000163078308\n",
      "Epoch 88 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9994727277755737\n",
      "\t- Validation Accuracy:\t0.9894000291824341\n",
      "Epoch 89 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9994727277755737\n",
      "\t- Validation Accuracy:\t0.9900000095367432\n",
      "Epoch 90 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9994545459747315\n",
      "\t- Validation Accuracy:\t0.9900000095367432\n",
      "Epoch 91 completed : Time usage 104 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9994363641738891\n",
      "\t- Validation Accuracy:\t0.9876000285148621\n",
      "Epoch 92 completed : Time usage 96 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9994363641738891\n",
      "\t- Validation Accuracy:\t0.9886000156402588\n",
      "Epoch 93 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9995818185806274\n",
      "\t- Validation Accuracy:\t0.9891999959945679\n",
      "Epoch 94 completed : Time usage 97 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9996909093856812\n",
      "\t- Validation Accuracy:\t0.9891999959945679\n",
      "Epoch 95 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9995818185806274\n",
      "\t- Validation Accuracy:\t0.9891999959945679\n",
      "Epoch 96 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9997090911865234\n",
      "\t- Validation Accuracy:\t0.9894000291824341\n",
      "Epoch 97 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.999745454788208\n",
      "\t- Validation Accuracy:\t0.9887999892234802\n",
      "Epoch 98 completed : Time usage 98 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9997636365890503\n",
      "\t- Validation Accuracy:\t0.9896000027656555\n",
      "Epoch 99 completed : Time usage 99 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9996545457839966\n",
      "\t- Validation Accuracy:\t0.9901999831199646\n",
      "Epoch 100 completed : Time usage 97 seconds\n",
      "\tAccuracy:\n",
      "\t- Training Accuracy:\t0.9998545455932617\n",
      "\t- Validation Accuracy:\t0.9896000027656555\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    # Initialize all variables\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    # Add the model graph to TensorBoard\n",
    "    writer.add_graph(sess.graph)\n",
    "    \n",
    "    # Loop over number of epochs\n",
    "    for epoch in range(num_epochs):\n",
    "        \n",
    "        start_time = time.time()\n",
    "        train_accuracy = 0\n",
    "        \n",
    "        for batch in range(0, int(len(data.train.labels)/batch_size)):\n",
    "            \n",
    "            # Get a batch of images and labels\n",
    "            x_batch, y_true_batch = data.train.next_batch(batch_size)\n",
    "            \n",
    "            # Put the batch into a dict with the proper names for placeholder variables\n",
    "            feed_dict_train = {x: x_batch, y_true: y_true_batch}\n",
    "            \n",
    "            # Run the optimizer using this batch of training data.\n",
    "            sess.run(optimizer, feed_dict=feed_dict_train)\n",
    "            \n",
    "            # Calculate the accuracy on the batch of training data\n",
    "            train_accuracy += sess.run(accuracy, feed_dict=feed_dict_train)\n",
    "            \n",
    "            # Generate summary with the current batch of data and write to file\n",
    "            summ = sess.run(merged_summary, feed_dict=feed_dict_train)\n",
    "            writer.add_summary(summ, epoch*int(len(data.train.labels)/batch_size) + batch)\n",
    "        \n",
    "          \n",
    "        train_accuracy /= int(len(data.train.labels)/batch_size)\n",
    "        \n",
    "        # Generate summary and validate the model on the entire validation set\n",
    "        summ, vali_accuracy = sess.run([merged_summary, accuracy], feed_dict={x:data.validation.images, y_true:data.validation.labels})\n",
    "        writer1.add_summary(summ, epoch)\n",
    "        \n",
    "\n",
    "        end_time = time.time()\n",
    "        \n",
    "        # Save checkpoint of the model\n",
    "        checkpoint_name = os.path.join(\"Checkpoints/\", 'model_epoch'+str(epoch+1)+'.ckpt')\n",
    "        save_path = tf.train.Saver().save(sess, checkpoint_name)  \n",
    "        \n",
    "        print(\"Epoch \"+str(epoch+1)+\" completed : Time usage \"+str(int(end_time-start_time))+\" seconds\")\n",
    "        print(\"\\tAccuracy:\")\n",
    "        print (\"\\t- Training Accuracy:\\t{}\".format(train_accuracy))\n",
    "        print (\"\\t- Validation Accuracy:\\t{}\".format(vali_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from Checkpoints/model_epoch100.ckpt\n"
     ]
    }
   ],
   "source": [
    "# Restore variables weight and calculate accuracy on the batch of validation data \n",
    "with tf.Session() as sess:\n",
    "    \n",
    "    # Restore the pretrained weights\n",
    "    tf.train.Saver().restore(sess, \"Checkpoints/model_epoch100.ckpt\")\n",
    "    \n",
    "    # Feed Dictionary\n",
    "    feed_dict={x:data.validation.images[78:90], y_true:data.validation.labels[78:90]}\n",
    "    \n",
    "    # Make the Predictions on the batch of validaton data\n",
    "    prediction_labels = sess.run(y_pred_cls, feed_dict=feed_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Visualize classification using OpenCV\n",
    "for i, image in enumerate(feed_dict[x]):\n",
    "    image = np.array(image).reshape(28,28)\n",
    "    height, width = image.shape[:2]\n",
    "    image  = cv2.resize(image, (width*10, height*10), interpolation = cv2.INTER_CUBIC)\n",
    "    \n",
    "    cv2.putText(image, \"True class: \"+ str(np.argmax(feed_dict[y_true], axis=1)[i]), (20,30) , cv2.FONT_HERSHEY_SIMPLEX, .5, 1)\n",
    "    cv2.putText(image, \"Predict class: \"+str(prediction_labels[i]),                  (140,30), cv2.FONT_HERSHEY_SIMPLEX, .5, 1)\n",
    "    \n",
    "    cv2.imshow(\"Images\",image)\n",
    "    cv2.waitKey(3000)\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}