DL_computer_vision/MNIST/01_Lenet_MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading the MNIST data, which comes pre-loaded with TensorFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "\n",
      " Image Shape: (28, 28, 1) \n",
      "\n",
      "Training Set:   55000 samples\n",
      "Validation Set: 5000 samples\n",
      "Test Set:       10000 samples\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "mnist = input_data.read_data_sets(\"MNIST_data/\", reshape=False)\n",
    "X_train, y_train           = mnist.train.images, mnist.train.labels\n",
    "X_validation, y_validation = mnist.validation.images, mnist.validation.labels\n",
    "X_test, y_test             = mnist.test.images, mnist.test.labels\n",
    "\n",
    "assert(len(X_train) == len(y_train))\n",
    "assert(len(X_validation) == len(y_validation))\n",
    "assert(len(X_test) == len(y_test))\n",
    "\n",
    "print(\"\\n Image Shape: {} \\n\".format(X_train[0].shape))\n",
    "print(\"Training Set:   {} samples\".format(len(X_train)))\n",
    "print(\"Validation Set: {} samples\".format(len(X_validation)))\n",
    "print(\"Test Set:       {} samples\".format(len(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updated Image Shape: (32, 32, 1)\n"
     ]
    }
   ],
   "source": [
    "## MNIST dataset has as 28x28x1 shape.\n",
    "## Architecture we are suing for this example only accepts 32x32x? images (? = color channels)\n",
    "# Loaded data set size is 28x28 we would need to be reshaped to 1x32x32x1\n",
    "import numpy as np\n",
    "img_size = 32\n",
    "\n",
    "if X_train[0].shape[1] is not img_size:\n",
    "    X_train      = np.pad(X_train, ((0,0),(2,2),(2,2),(0,0)), 'constant')\n",
    "    X_validation = np.pad(X_validation, ((0,0),(2,2),(2,2),(0,0)), 'constant')\n",
    "    X_test       = np.pad(X_test, ((0,0),(2,2),(2,2),(0,0)), 'constant')\n",
    "    \n",
    "print(\"Updated Image Shape: {}\".format(X_train[0].shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# How does this data set looks like?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAFsAAABZCAYAAABR/liSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAABVBJREFUeJztnEFsVFUUhr9fRrtpQ2hKDUFixRjCAlKJuDGBBikRQ0Ib0EiIKDHghqTutK4kbEjQbg2YkggxEZNCBhZEXOhCFgYkjVJB20gRtGCKC+tsmjrHxbwZKm3pdN7Mmenr/ZLJzLy+eef0nz9n7r0578rMCPjwSLUTWEgEsR0JYjsSxHYkiO1IENuRILYjscSW9JKknyUNSXqvXEklFZU6qZG0CPgFaAduA5eAXWb2U/nSSxapGJ99Hhgys18BJH0ObAdmFFtSYqerZqbZzolTRpYDtya9vx0d+x+S9ku6LOlyjFiJII6zp/smpzjXzI4BxyDZzi6GOM6+DayY9P4J4I946SSbOGJfAp6R9JSkx4DXgLPlSSuZlFxGzGxC0gHgS2ARcNzMBsqWWQIpeehXUrAE1+xKj0YCcySI7Uicod+8Z/HixQBcuHABgEwmw6ZNmyoWb0GK3djYCEBfXx8Aa9asAWDv3r0VjRvKiCMLztkNDQ0cPHgQgI0bNwJw6NAhAE6dOlXR2MHZnpiZ24Pc2klVH+l02rLZrGWzWUun05ZOpy2VSlkqlYp13WL+/wVTRrZs2QLA2rVruXfvHgBdXV0ATExMuOQQyogjiXd2XV0dANu2bQOgqamJ9vZ2AIaHh11zCc52JPELUSdPngRg9+7dAAwMDBQmMeUkLETVGImt2evWrQPuj0Ju3rwJQGdnZ9VySqzYZ86cAWDp0qUA9Pb2AjA0NFS1nEIZcSSRzu7o6KC5uRmAixcvAtDd3V3NlIDgbFcS5exVq1YBcOLEicJk5ujRowCMjo5WLa88iRhnb9iwAYAjR44AsH79eu7cuQNAS0sLAOPj45UIXSCMs2uMRJSRHTt2ADlH59m5cydQeUfPheBsRxJRs2/dyjXTLl+ea6K9ceMGra2tAIyNjVUi5BTKUrMlrZD0taRrkgYkdUXHGyV9JWkwel5SjqSTzKzOlrQMWGZmVyQ1AN8DHcCbwF9mdji6xWOJmb07y7XK7uzz588X1j+knLm6u7sLPSGrV68uHAO4fv16uVMAinP2rD+QZjYCjESvxyRdI9f0vh1oi077FPgGeKjYlaCurq4gcp59+/axcuVK4P4XkC8ne/bs8U1wEnMajUhqAZ4FvgMej74IzGxEUvMMn9kP7I+XZjIoWmxJ9UAf8I6Z/f2gm2aiGnce5F0dxfcIWRRFDf0kPUpO6M/M7HR0+G5Uz/N1/c/KpJgcZnW2chbuBa6ZWc+kP50F3gAOR8/pimQYk/ykpqenZ5YzK08xZeQF4HXgR0n90bH3yYn8haS3gN+AVyqTYnIoZjTyLdPfGQbwYnnTmTvnzp2jra1tyvFMJgPA1q1bAejv759yjjfzfgZZX1/P6dO5n5HNmzcDMDg4WFgvuXr1arlDTktY9asx5r2za4Xg7BojiO1IENuRILYjQWxHgtiOBLEdCWI7EsR2JIjtSBDbEe+OqFEgEz3XOk0Un+eTxZzkuhAFIOmymT3nGrQEKpFnKCOOBLEdqYbYx6oQsxTKnqd7zV7IhDLiiJvYtbzX9kM6dT+Q9Luk/ujxcqw4HmWk1vfafkin7qvAP2b2YTnieDm7sNe2mY0D+b22awIzGzGzK9HrMSDfqVtWvMQuaq/tWuCBTl2AA5J+kHQ8bsO/l9hF7bVdbR7s1AU+Bp4GWsn1qH8U5/peYtf8XtvTdeqa2V0z+9fMssAn5MphyXiJXdN7bc/UqZtviY7oBGL1srms+s2DvbZn6tTdJamVXMkbBt6OEyTMIB0JM0hHgtiOBLEdCWI7EsR2JIjtSBDbkSC2I/8Bg2Zbi8BWWJMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 72x72 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "index = random.randint(0, len(X_train))\n",
    "image = X_train[index].squeeze()\n",
    "\n",
    "plt.figure(figsize=(1,1))\n",
    "plt.imshow(image, cmap=\"gray\")\n",
    "print(y_train[index])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# For this example I am using LeNet architecture \n",
    "# http://yann.lecun.com/exdb/publis/pdf/lecun-01a.pdf\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.contrib.layers import flatten\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "# Using tf.truncated_normal to initilize weights and biases varibles\n",
    "# with random normal distribution \n",
    "mu = 0\n",
    "sigma = 0.1\n",
    "\n",
    "conv1_W = tf.Variable(tf.truncated_normal(shape=(5, 5, 1, 6), mean = mu, stddev = sigma))\n",
    "conv1_b = tf.Variable(tf.zeros(6))\n",
    "\n",
    "conv2_W = tf.Variable(tf.truncated_normal(shape=(5, 5, 6, 16), mean = mu, stddev = sigma))\n",
    "conv2_b = tf.Variable(tf.zeros(16))\n",
    "\n",
    "fc1_W = tf.Variable(tf.truncated_normal(shape=(400, 120), mean = mu, stddev = sigma))\n",
    "fc1_b = tf.Variable(tf.zeros(120))\n",
    "\n",
    "fc2_W  = tf.Variable(tf.truncated_normal(shape=(120, 84), mean = mu, stddev = sigma))\n",
    "fc2_b  = tf.Variable(tf.zeros(84))\n",
    "\n",
    "fc3_W  = tf.Variable(tf.truncated_normal(shape=(84, 10), mean = mu, stddev = sigma))\n",
    "fc3_b  = tf.Variable(tf.zeros(10))\n",
    "\n",
    "def LeNet(x):     \n",
    "    \n",
    "    print(x.shape)\n",
    "    #Layer 1: Convolutional. Input = 32x32x1. Output = 28x28x6.\n",
    "    conv1   = tf.nn.conv2d(x, conv1_W, strides=[1, 1, 1, 1], padding='VALID') + conv1_b\n",
    "    conv1 = tf.nn.relu(conv1)\n",
    "    #Pooling. Input = 28x28x6. Output = 14x14x6.\n",
    "    conv1 = tf.nn.max_pool(conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='VALID')\n",
    "\n",
    "    #Layer 2: Convolutional. Output = 10x10x16.\n",
    "    conv2   = tf.nn.conv2d(conv1, conv2_W, strides=[1, 1, 1, 1], padding='VALID') + conv2_b\n",
    "    conv2 = tf.nn.relu(conv2)\n",
    "    #Pooling. Input = 10x10x16. Output = 5x5x16.\n",
    "    conv2 = tf.nn.max_pool(conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='VALID')\n",
    "    #Flatten. Input = 5x5x16. Output = 400.\n",
    "    fc0   = flatten(conv2)\n",
    "    \n",
    "    #Layer 3: Fully Connected. Input = 400. Output = 120.\n",
    "    fc1   = tf.matmul(fc0, fc1_W) + fc1_b\n",
    "    fc1    = tf.nn.relu(fc1)\n",
    "\n",
    "    #Layer 4: Fully Connected. Input = 120. Output = 84.\n",
    "    fc2    = tf.matmul(fc1, fc2_W) + fc2_b\n",
    "    fc2    = tf.nn.relu(fc2)\n",
    "\n",
    "    #Layer 5: Fully Connected. Input = 84. Output = 10.\n",
    "    logits = tf.matmul(fc2, fc3_W) + fc3_b\n",
    "    \n",
    "    return logits\n",
    "\n",
    "#Placeholder variables\n",
    "x = tf.placeholder(tf.float32, (None, img_size, img_size, 1))\n",
    "y = tf.placeholder(tf.int32, (None))\n",
    "\n",
    "save_path = 'model/model.ckpt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(?, 32, 32, 1)\n",
      "WARNING:tensorflow:From <ipython-input-5-41b9b231c9ad>:10: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 10\n",
    "BATCH_SIZE = 128\n",
    "\n",
    "rate = 0.001\n",
    "one_hot_y = tf.one_hot(y, 10)\n",
    "\n",
    "# Get logits\n",
    "logits = LeNet(x)\n",
    "# Computes softmax cross entropy between logits and labels\n",
    "cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=one_hot_y, logits=logits)\n",
    "#Calculate loss\n",
    "loss_operation = tf.reduce_mean(cross_entropy)\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate = rate)\n",
    "training_operation = optimizer.minimize(loss_operation)\n",
    "\n",
    "# Add an op to initialize the variables.\n",
    "init_op = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#set validation system \n",
    "correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(one_hot_y, 1))\n",
    "accuracy_operation = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "def evaluate(X_data, y_data):\n",
    "    num_examples = len(X_data)\n",
    "    total_accuracy = 0\n",
    "    sess = tf.get_default_session()\n",
    "    for offset in range(0, num_examples, BATCH_SIZE):\n",
    "        batch_x, batch_y = X_data[offset:offset+BATCH_SIZE], y_data[offset:offset+BATCH_SIZE]\n",
    "        accuracy = sess.run(accuracy_operation, feed_dict={x: batch_x, y: batch_y})\n",
    "        total_accuracy += (accuracy * len(batch_x))\n",
    "    return total_accuracy / num_examples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Start tensorflow training session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[name: \"/device:CPU:0\"\n",
       " device_type: \"CPU\"\n",
       " memory_limit: 268435456\n",
       " locality {\n",
       " }\n",
       " incarnation: 5520820191998512049]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.python.client import device_lib\n",
    "device_lib.list_local_devices()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training...\n",
      "\n",
      "21.43s EPOCH 1 Accuracy = 0.965 \n",
      "\n",
      "42.25s EPOCH 2 Accuracy = 0.971 \n",
      "\n",
      "63.25s EPOCH 3 Accuracy = 0.978 \n",
      "\n",
      "83.84s EPOCH 4 Accuracy = 0.979 \n",
      "\n",
      "105.22s EPOCH 5 Accuracy = 0.982 \n",
      "\n",
      "126.50s EPOCH 6 Accuracy = 0.985 \n",
      "\n",
      "148.22s EPOCH 7 Accuracy = 0.984 \n",
      "\n",
      "169.53s EPOCH 8 Accuracy = 0.986 \n",
      "\n",
      "190.75s EPOCH 9 Accuracy = 0.983 \n",
      "\n",
      "212.38s EPOCH 10 Accuracy = 0.986 \n",
      "\n",
      "Model saved\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "beginTime = time.time()\n",
    "with tf.device('/cpu:0'):\n",
    "    with tf.Session() as sess:\n",
    "        sess.run(init_op)\n",
    "        dataLen = len(X_train)\n",
    "\n",
    "        print(\"Training...\")\n",
    "        print()\n",
    "        for i in range(EPOCHS):\n",
    "            for offset in range(0, dataLen, BATCH_SIZE):\n",
    "                end = offset + BATCH_SIZE\n",
    "                batch_x, batch_y = X_train[offset:end], y_train[offset:end]\n",
    "                sess.run(training_operation, feed_dict={x: batch_x, y: batch_y})\n",
    "\n",
    "            validation_accuracy = evaluate(X_validation, y_validation)\n",
    "\n",
    "            endTime = time.time()     \n",
    "            print(\"{:5.2f}s EPOCH {} Accuracy = {:.3f} \\n\".format(\\\n",
    "            endTime - beginTime, i+1, validation_accuracy))\n",
    "\n",
    "        save_path = saver.save(sess, 'model/model.ckpt')\n",
    "        print(\"Model saved\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(?, 32, 32, 1)\n",
      "Training...\n",
      "\n",
      "22.07s EPOCH 1 Accuracy = 0.911 \n",
      "\n",
      "Model saved\n"
     ]
    }
   ],
   "source": [
    "#RESER GRAPH\n",
    "\n",
    "\n",
    "EPOCHS = 100\n",
    "BATCH_SIZE = 64\n",
    "rate = 0.0001\n",
    "one_hot_y = tf.one_hot(y, 10)\n",
    "\n",
    "# Get logits\n",
    "logits = LeNet(x)\n",
    "# Computes softmax cross entropy between logits and labels\n",
    "cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=one_hot_y, logits=logits)\n",
    "#Calculate loss\n",
    "loss_operation = tf.reduce_mean(cross_entropy)\n",
    "optimizer = tf.train.AdamOptimizer(rate=1e-4)\n",
    "training_operation = optimizer.minimize(loss_operation)\n",
    "\n",
    "# Add an op to initialize the variables.\n",
    "init_op = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "import time\n",
    "beginTime = time.time()\n",
    "with tf.device('/cpu:0'):\n",
    "    with tf.Session() as sess:\n",
    "        sess.run(init_op)\n",
    "        dataLen = len(X_train)\n",
    "\n",
    "        print(\"Training...\")\n",
    "        print()\n",
    "        for i in range(EPOCHS):\n",
    "            for offset in range(0, dataLen, BATCH_SIZE):\n",
    "                end = offset + BATCH_SIZE\n",
    "                batch_x, batch_y = X_train[offset:end], y_train[offset:end]\n",
    "                sess.run(training_operation, feed_dict={x: batch_x, y: batch_y})\n",
    "                \n",
    "            if i % 10 == 0:\n",
    "                validation_accuracy = evaluate(X_validation, y_validation)\n",
    "\n",
    "                endTime = time.time()     \n",
    "                print(\"{:5.2f}s EPOCH {} Accuracy = {:.3f} \\n\".format(\\\n",
    "                endTime - beginTime, i+1, validation_accuracy))\n",
    "\n",
    "        save_path = saver.save(sess, 'model/model.ckpt')\n",
    "        print(\"Model saved\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:nightly]",
   "language": "python",
   "name": "conda-env-nightly-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}