MNIST_TensorFlowSlim.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple LeNet-5 convolutional MNIST model example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import gzip\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "\n",
    "import numpy\n",
    "from six.moves import urllib\n",
    "from six.moves import xrange\n",
    "import tensorflow as tf\n",
    "\n",
    "SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/'\n",
    "WORK_DIRECTORY = 'data'\n",
    "IMAGE_SIZE = 28\n",
    "NUM_CHANNELS = 1\n",
    "PIXEL_DEPTH = 255\n",
    "NUM_LABELS = 10\n",
    "VALIDATION_SIZE = 5000\n",
    "SEED = 66478 # set to None for random seed\n",
    "BATCH_SIZE = 64\n",
    "NUM_EPOCHS = 10\n",
    "EVAL_BATCH_SIZE = 64\n",
    "EVAL_FREQUENCY = 100\n",
    "\n",
    "#tf.app.flags.DEFINE_boolean(\"self_test\", False, \"True if running a self test.\")\n",
    "FLAGS = tf.app.flags.FLAGS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download the MNIST dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "def maybe_download(filename):\n",
    "  \"\"\"Download the data from Yann's website, unless it's already here.\"\"\"\n",
    "  if not tf.gfile.Exists(WORK_DIRECTORY):\n",
    "    tf.gfile.MakeDirs(WORK_DIRECTORY)\n",
    "  filepath = os.path.join(WORK_DIRECTORY, filename)\n",
    "  if not tf.gfile.Exists(filepath):\n",
    "    filepath, _ = urllib.request.urlretrieve(SOURCE_URL + filename, filepath)\n",
    "    with tf.gfile.GFile(filepath) as f:\n",
    "      size = f.Size()\n",
    "    print('Successfully downloaded', filename, size, 'bytes.')\n",
    "  return filepath\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "train_data_filename = maybe_download('train-images-idx3-ubyte.gz')\n",
    "train_labels_filename = maybe_download('train-labels-idx1-ubyte.gz')\n",
    "test_data_filename = maybe_download('t10k-images-idx3-ubyte.gz')\n",
    "test_labels_filename = maybe_download('t10k-labels-idx1-ubyte.gz')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def extract_data(filename, num_images):\n",
    "  \"\"\"Extract the images into a 4D tensor [image index, y, x, channels].\n",
    "  Values are rescaled from [0, 255] down to [-0.5, 0.5].\n",
    "  \"\"\"\n",
    "  print('Extracting', filename)\n",
    "  with gzip.open(filename) as bytestream:\n",
    "    bytestream.read(16)\n",
    "    buf = bytestream.read(IMAGE_SIZE * IMAGE_SIZE * num_images)\n",
    "    data = numpy.frombuffer(buf, dtype=numpy.uint8).astype(numpy.float32)\n",
    "    data = (data - (PIXEL_DEPTH / 2.0)) / PIXEL_DEPTH\n",
    "    data = data.reshape(num_images, IMAGE_SIZE, IMAGE_SIZE, 1)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def extract_labels(filename, num_images):\n",
    "  \"\"\"Extract the labels into a vector of int64 label IDs.\"\"\"\n",
    "  print('Extracting', filename)\n",
    "  with gzip.open(filename) as bytestream:\n",
    "    bytestream.read(8)\n",
    "    buf = bytestream.read(1 * num_images)\n",
    "    labels = numpy.frombuffer(buf, dtype=numpy.uint8).astype(numpy.int64)\n",
    "  return labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def dense_to_one_hot(labels_dense, num_classes):\n",
    "    num_labels = labels_dense.shape[0]\n",
    "    index_offset = numpy.arange(num_labels)*num_classes\n",
    "    labels_one_hot = numpy.zeros((num_labels, num_classes))\n",
    "    labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1\n",
    "    return labels_one_hot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/train-images-idx3-ubyte.gz\n",
      "Extracting data/train-labels-idx1-ubyte.gz\n",
      "Extracting data/t10k-images-idx3-ubyte.gz\n",
      "Extracting data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "train_data = extract_data(train_data_filename, 60000)\n",
    "train_labels = extract_labels(train_labels_filename, 60000)\n",
    "#train_labels = dense_to_one_hot(train_labels, NUM_LABELS)\n",
    "test_data = extract_data(test_data_filename, 10000)\n",
    "test_labels = extract_labels(test_labels_filename, 10000)\n",
    "#test_labels = dense_to_one_hot(test_labels, NUM_LABELS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate training and validation set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "validation_data = train_data[:VALIDATION_SIZE, ...]\n",
    "validation_labels = train_labels[:VALIDATION_SIZE]\n",
    "train_data = train_data[VALIDATION_SIZE:, ...]\n",
    "train_labels = train_labels[VALIDATION_SIZE:]\n",
    "train_labels = dense_to_one_hot(train_labels, NUM_LABELS)\n",
    "num_epochs = NUM_EPOCHS\n",
    "train_size = train_labels.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Define Place holders for inputs and outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_data_node = tf.placeholder(tf.float32, shape=(BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS))\n",
    "train_labels_node = tf.placeholder(tf.int64, shape= (BATCH_SIZE,NUM_LABELS))\n",
    "eval_data = tf.placeholder(tf.float32, shape=(EVAL_BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "conv1_weights = tf.Variable(tf.truncated_normal([5,5,NUM_CHANNELS, 32], # 5x5 filter, depth 32\n",
    "                                               stddev=0.1, seed=SEED))\n",
    "conv1_biases = tf.Variable(tf.zeros([32]))\n",
    "conv2_weights = tf.Variable(tf.truncated_normal([5,5,32,64], stddev=0.1, seed=SEED))\n",
    "conv2_biases = tf.Variable(tf.constant(0.1, shape=[64]))\n",
    "fc1_weights = tf.Variable(tf.truncated_normal([IMAGE_SIZE // 4 * IMAGE_SIZE // 4 * 64, 512], stddev=0.1, seed=SEED))\n",
    "fc1_biases = tf.Variable(tf.constant(0.1, shape=[512]))\n",
    "fc2_weights = tf.Variable(tf.truncated_normal([512, NUM_LABELS], stddev=0.1, seed=SEED))\n",
    "fc2_biases = tf.Variable(tf.constant(0.1, shape=[NUM_LABELS]))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def model(data, train=False):\n",
    "    conv = tf.nn.conv2d(data, conv1_weights, strides=[1,1,1,1], padding='SAME')\n",
    "    relu = tf.nn.relu(tf.nn.bias_add(conv, conv1_biases))\n",
    "    pool = tf.nn.max_pool(relu, ksize=[1,2,2,1], strides=[1,2,2,1], padding='SAME')\n",
    "    conv = tf.nn.conv2d(pool, conv2_weights, strides=[1,1,1,1], padding='SAME')\n",
    "    relu = tf.nn.relu(tf.nn.bias_add(conv, conv2_biases))\n",
    "    pool = tf.nn.max_pool(relu, ksize=[1,2,2,1], strides=[1,2,2,1], padding='SAME')\n",
    "    pool_shape = pool.get_shape().as_list()\n",
    "    reshape = tf.reshape(pool, [pool_shape[0], pool_shape[1]*pool_shape[2]*pool_shape[3]])\n",
    "    hidden = tf.nn.relu(tf.matmul(reshape, fc1_weights)+fc1_biases)\n",
    "    if train:\n",
    "        hidden = tf.nn.dropout(hidden, 0.5, seed=SEED)\n",
    "    return tf.matmul(hidden, fc2_weights)+fc2_biases\n",
    "\n",
    "logits = model(train_data_node, True)\n",
    "loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits, train_labels_node))\n",
    "#### L2 regularization for fully connected parameters\n",
    "regularizers = (tf.nn.l2_loss(fc1_weights)+tf.nn.l2_loss(fc1_biases)\n",
    "                +tf.nn.l2_loss(fc2_weights)+tf.nn.l2_loss(fc2_biases))\n",
    "loss += 5e-4 * regularizers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Model using TensorFlow-Slim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tensorflow.contrib.slim as slim\n",
    "def model_slim(data, train=False):\n",
    "    if train:\n",
    "        reuse = None\n",
    "    else:\n",
    "        reuse = True\n",
    "    with slim.arg_scope([slim.conv2d], padding='SAME', activation_fn=tf.nn.relu,\n",
    "                      weights_initializer=tf.truncated_normal_initializer(stddev=0.01),\n",
    "                      weights_regularizer=slim.l2_regularizer(0.0005)):\n",
    "        net = slim.layers.conv2d(data, 32, [5, 5], 1,scope='conv1', reuse=reuse)\n",
    "        net = slim.layers.max_pool2d(net, [2,2], scope='pool1')\n",
    "        net = slim.layers.conv2d(net, 64, [5, 5],scope='conv2', reuse=reuse)\n",
    "        net = slim.layers.max_pool2d(net, [2,2], scope='pool2')\n",
    "    net = slim.layers.flatten(net, scope='flatten3')\n",
    "    net = slim.layers.fully_connected(net, 512, scope='fc1', reuse=reuse)\n",
    "    if train:\n",
    "        net = tf.nn.dropout(net, 0.5, seed=SEED)\n",
    "    net = slim.layers.fully_connected(net, NUM_LABELS, activation_fn=None, scope='fc2', reuse=reuse)\n",
    "    return net\n",
    "\n",
    "logits = model_slim(train_data_node, True)\n",
    "loss = slim.losses.softmax_cross_entropy(logits, train_labels_node)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "batch = tf.Variable(0)\n",
    "learning_rate = tf.train.exponential_decay(0.01, batch*BATCH_SIZE, train_size, 0.95, staircase=True)\n",
    "optimizer = tf.train.MomentumOptimizer(learning_rate, 0.9).minimize(loss, global_step=batch)\n",
    "\n",
    "train_prediction = tf.nn.softmax(logits)\n",
    "eval_prediction = tf.nn.softmax(model_slim(eval_data))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def eval_in_batches(data, sess):\n",
    "    size = data.shape[0]\n",
    "    if size < EVAL_BATCH_SIZE:\n",
    "        raise ValueErro('batch size for evals larger than dataset: %d' %size)\n",
    "    predictions = numpy.ndarray(shape=(size, NUM_LABELS), dtype=numpy.float32)\n",
    "    for begin in xrange(0, size, EVAL_BATCH_SIZE):\n",
    "        end = begin + EVAL_BATCH_SIZE\n",
    "        if end <= size:\n",
    "            predictions[begin:end,:] = sess.run(eval_prediction, feed_dict={eval_data:data[begin:end,...]})\n",
    "        else:\n",
    "            batch_predictions = sess.run(eval_prediction, feed_dict={eval_data:data[-EVAL_BATCH_SIZE:,...]})\n",
    "    return predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Error rate function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def error_rate(predictions, labels):\n",
    "    return 100.0 - (100.0 * numpy.sum(numpy.argmax(predictions,1)== labels) / predictions.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Session to run training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Step 0 (epoch 0.00), 5.8 ms\n",
      "Minibatch loss: 2.303, learning rate: 0.010000\n",
      "Validation error: 88.4%\n",
      "Step 100 (epoch 0.00), 235.4 ms\n",
      "Minibatch loss: 0.999, learning rate: 0.010000\n",
      "Validation error: 25.0%\n",
      "Step 200 (epoch 0.00), 267.5 ms\n",
      "Minibatch loss: 0.438, learning rate: 0.010000\n",
      "Validation error: 6.5%\n",
      "Step 300 (epoch 0.00), 263.0 ms\n",
      "Minibatch loss: 0.270, learning rate: 0.010000\n",
      "Validation error: 4.3%\n",
      "Step 400 (epoch 0.00), 266.9 ms\n",
      "Minibatch loss: 0.369, learning rate: 0.010000\n",
      "Validation error: 3.5%\n",
      "Step 500 (epoch 0.00), 253.8 ms\n",
      "Minibatch loss: 0.272, learning rate: 0.010000\n",
      "Validation error: 3.2%\n",
      "Step 600 (epoch 0.00), 246.1 ms\n",
      "Minibatch loss: 0.088, learning rate: 0.010000\n",
      "Validation error: 3.7%\n",
      "Step 700 (epoch 0.00), 248.9 ms\n",
      "Minibatch loss: 0.069, learning rate: 0.010000\n",
      "Validation error: 2.5%\n",
      "Step 800 (epoch 0.00), 288.0 ms\n",
      "Minibatch loss: 0.109, learning rate: 0.010000\n",
      "Validation error: 2.9%\n",
      "Step 900 (epoch 0.00), 297.8 ms\n",
      "Minibatch loss: 0.058, learning rate: 0.009500\n",
      "Validation error: 2.1%\n",
      "Step 1000 (epoch 0.00), 312.2 ms\n",
      "Minibatch loss: 0.093, learning rate: 0.009500\n",
      "Validation error: 2.0%\n",
      "Step 1100 (epoch 0.00), 288.4 ms\n",
      "Minibatch loss: 0.025, learning rate: 0.009500\n",
      "Validation error: 1.9%\n",
      "Step 1200 (epoch 0.00), 279.4 ms\n",
      "Minibatch loss: 0.224, learning rate: 0.009500\n",
      "Validation error: 1.7%\n",
      "Step 1300 (epoch 0.00), 248.8 ms\n",
      "Minibatch loss: 0.040, learning rate: 0.009500\n",
      "Validation error: 1.8%\n",
      "Step 1400 (epoch 0.00), 259.4 ms\n",
      "Minibatch loss: 0.076, learning rate: 0.009500\n",
      "Validation error: 1.8%\n",
      "Step 1500 (epoch 0.00), 250.4 ms\n",
      "Minibatch loss: 0.138, learning rate: 0.009500\n",
      "Validation error: 1.7%\n",
      "Step 1600 (epoch 0.00), 251.4 ms\n",
      "Minibatch loss: 0.015, learning rate: 0.009500\n",
      "Validation error: 1.7%\n",
      "Step 1700 (epoch 0.00), 247.3 ms\n",
      "Minibatch loss: 0.029, learning rate: 0.009500\n",
      "Validation error: 1.8%\n",
      "Step 1800 (epoch 0.00), 244.1 ms\n",
      "Minibatch loss: 0.039, learning rate: 0.009025\n",
      "Validation error: 1.6%\n",
      "Step 1900 (epoch 0.00), 246.3 ms\n",
      "Minibatch loss: 0.030, learning rate: 0.009025\n",
      "Validation error: 1.4%\n",
      "Step 2000 (epoch 0.00), 244.5 ms\n",
      "Minibatch loss: 0.030, learning rate: 0.009025\n",
      "Validation error: 1.5%\n",
      "Step 2100 (epoch 0.00), 247.0 ms\n",
      "Minibatch loss: 0.055, learning rate: 0.009025\n",
      "Validation error: 1.4%\n",
      "Step 2200 (epoch 0.00), 242.1 ms\n",
      "Minibatch loss: 0.014, learning rate: 0.009025\n",
      "Validation error: 1.3%\n",
      "Step 2300 (epoch 0.00), 243.6 ms\n",
      "Minibatch loss: 0.028, learning rate: 0.009025\n",
      "Validation error: 1.5%\n",
      "Step 2400 (epoch 0.00), 246.1 ms\n",
      "Minibatch loss: 0.019, learning rate: 0.009025\n",
      "Validation error: 1.5%\n",
      "Step 2500 (epoch 0.00), 243.9 ms\n",
      "Minibatch loss: 0.030, learning rate: 0.009025\n",
      "Validation error: 1.5%\n",
      "Step 2600 (epoch 0.00), 242.8 ms\n",
      "Minibatch loss: 0.012, learning rate: 0.008574\n",
      "Validation error: 1.6%\n",
      "Step 2700 (epoch 0.00), 244.0 ms\n",
      "Minibatch loss: 0.062, learning rate: 0.008574\n",
      "Validation error: 1.4%\n",
      "Step 2800 (epoch 0.00), 240.7 ms\n",
      "Minibatch loss: 0.039, learning rate: 0.008574\n",
      "Validation error: 1.3%\n",
      "Step 2900 (epoch 0.00), 239.5 ms\n",
      "Minibatch loss: 0.055, learning rate: 0.008574\n",
      "Validation error: 1.4%\n",
      "Step 3000 (epoch 0.00), 241.2 ms\n",
      "Minibatch loss: 0.049, learning rate: 0.008574\n",
      "Validation error: 1.3%\n",
      "Step 3100 (epoch 0.00), 237.0 ms\n",
      "Minibatch loss: 0.044, learning rate: 0.008574\n",
      "Validation error: 1.2%\n",
      "Step 3200 (epoch 0.00), 241.8 ms\n",
      "Minibatch loss: 0.017, learning rate: 0.008574\n",
      "Validation error: 1.3%\n",
      "Step 3300 (epoch 0.00), 235.2 ms\n",
      "Minibatch loss: 0.005, learning rate: 0.008574\n",
      "Validation error: 1.3%\n",
      "Step 3400 (epoch 0.00), 242.2 ms\n",
      "Minibatch loss: 0.010, learning rate: 0.008574\n",
      "Validation error: 1.3%\n",
      "Step 3500 (epoch 0.00), 237.4 ms\n",
      "Minibatch loss: 0.012, learning rate: 0.008145\n",
      "Validation error: 1.2%\n",
      "Step 3600 (epoch 0.00), 244.2 ms\n",
      "Minibatch loss: 0.002, learning rate: 0.008145\n",
      "Validation error: 1.1%\n",
      "Step 3700 (epoch 0.00), 241.9 ms\n",
      "Minibatch loss: 0.002, learning rate: 0.008145\n",
      "Validation error: 1.3%\n",
      "Step 3800 (epoch 0.00), 238.7 ms\n",
      "Minibatch loss: 0.014, learning rate: 0.008145\n",
      "Validation error: 1.2%\n",
      "Step 3900 (epoch 0.00), 246.6 ms\n",
      "Minibatch loss: 0.055, learning rate: 0.008145\n",
      "Validation error: 1.2%\n",
      "Step 4000 (epoch 0.00), 235.5 ms\n",
      "Minibatch loss: 0.055, learning rate: 0.008145\n",
      "Validation error: 1.0%\n",
      "Step 4100 (epoch 0.00), 241.8 ms\n",
      "Minibatch loss: 0.035, learning rate: 0.008145\n",
      "Validation error: 1.3%\n",
      "Step 4200 (epoch 0.00), 238.1 ms\n",
      "Minibatch loss: 0.070, learning rate: 0.008145\n",
      "Validation error: 1.3%\n",
      "Step 4300 (epoch 0.00), 239.8 ms\n",
      "Minibatch loss: 0.047, learning rate: 0.007738\n",
      "Validation error: 1.1%\n",
      "Step 4400 (epoch 0.00), 236.4 ms\n",
      "Minibatch loss: 0.039, learning rate: 0.007738\n",
      "Validation error: 1.2%\n",
      "Step 4500 (epoch 0.00), 234.8 ms\n",
      "Minibatch loss: 0.083, learning rate: 0.007738\n",
      "Validation error: 1.1%\n",
      "Step 4600 (epoch 0.00), 239.0 ms\n",
      "Minibatch loss: 0.020, learning rate: 0.007738\n",
      "Validation error: 1.0%\n",
      "Step 4700 (epoch 0.00), 237.8 ms\n",
      "Minibatch loss: 0.001, learning rate: 0.007738\n",
      "Validation error: 1.2%\n",
      "Step 4800 (epoch 0.00), 238.0 ms\n",
      "Minibatch loss: 0.017, learning rate: 0.007738\n",
      "Validation error: 1.0%\n",
      "Step 4900 (epoch 0.00), 237.3 ms\n",
      "Minibatch loss: 0.011, learning rate: 0.007738\n",
      "Validation error: 1.1%\n",
      "Step 5000 (epoch 0.00), 237.8 ms\n",
      "Minibatch loss: 0.076, learning rate: 0.007738\n",
      "Validation error: 1.1%\n",
      "Step 5100 (epoch 0.00), 233.9 ms\n",
      "Minibatch loss: 0.012, learning rate: 0.007738\n",
      "Validation error: 1.2%\n",
      "Step 5200 (epoch 0.00), 236.1 ms\n",
      "Minibatch loss: 0.093, learning rate: 0.007351\n",
      "Validation error: 1.1%\n",
      "Step 5300 (epoch 0.00), 242.3 ms\n",
      "Minibatch loss: 0.009, learning rate: 0.007351\n",
      "Validation error: 1.1%\n",
      "Step 5400 (epoch 0.00), 241.5 ms\n",
      "Minibatch loss: 0.006, learning rate: 0.007351\n",
      "Validation error: 1.1%\n",
      "Step 5500 (epoch 0.00), 236.6 ms\n",
      "Minibatch loss: 0.009, learning rate: 0.007351\n",
      "Validation error: 1.1%\n",
      "Step 5600 (epoch 0.00), 239.8 ms\n",
      "Minibatch loss: 0.001, learning rate: 0.007351\n",
      "Validation error: 1.1%\n",
      "Step 5700 (epoch 0.00), 238.5 ms\n",
      "Minibatch loss: 0.005, learning rate: 0.007351\n",
      "Validation error: 1.0%\n",
      "Step 5800 (epoch 0.00), 236.0 ms\n",
      "Minibatch loss: 0.003, learning rate: 0.007351\n",
      "Validation error: 1.2%\n",
      "Step 5900 (epoch 0.00), 235.2 ms\n",
      "Minibatch loss: 0.003, learning rate: 0.007351\n",
      "Validation error: 1.0%\n",
      "Step 6000 (epoch 0.00), 238.6 ms\n",
      "Minibatch loss: 0.010, learning rate: 0.007351\n",
      "Validation error: 1.3%\n",
      "Step 6100 (epoch 0.00), 243.1 ms\n",
      "Minibatch loss: 0.005, learning rate: 0.006983\n",
      "Validation error: 1.1%\n",
      "Step 6200 (epoch 0.00), 234.1 ms\n",
      "Minibatch loss: 0.001, learning rate: 0.006983\n",
      "Validation error: 1.0%\n",
      "Step 6300 (epoch 0.00), 237.3 ms\n",
      "Minibatch loss: 0.036, learning rate: 0.006983\n",
      "Validation error: 1.1%\n",
      "Step 6400 (epoch 0.00), 241.3 ms\n",
      "Minibatch loss: 0.044, learning rate: 0.006983\n",
      "Validation error: 1.1%\n",
      "Step 6500 (epoch 0.00), 233.7 ms\n",
      "Minibatch loss: 0.006, learning rate: 0.006983\n",
      "Validation error: 1.0%\n",
      "Step 6600 (epoch 0.00), 239.0 ms\n",
      "Minibatch loss: 0.018, learning rate: 0.006983\n",
      "Validation error: 1.1%\n",
      "Step 6700 (epoch 0.00), 235.9 ms\n",
      "Minibatch loss: 0.007, learning rate: 0.006983\n",
      "Validation error: 1.0%\n",
      "Step 6800 (epoch 0.00), 233.3 ms\n",
      "Minibatch loss: 0.007, learning rate: 0.006983\n",
      "Validation error: 1.0%\n",
      "Step 6900 (epoch 0.00), 234.6 ms\n",
      "Minibatch loss: 0.001, learning rate: 0.006634\n",
      "Validation error: 1.1%\n",
      "Step 7000 (epoch 0.00), 234.6 ms\n",
      "Minibatch loss: 0.006, learning rate: 0.006634\n",
      "Validation error: 1.1%\n",
      "Step 7100 (epoch 0.00), 246.5 ms\n",
      "Minibatch loss: 0.006, learning rate: 0.006634\n",
      "Validation error: 1.0%\n",
      "Step 7200 (epoch 0.00), 253.8 ms\n",
      "Minibatch loss: 0.007, learning rate: 0.006634\n",
      "Validation error: 1.0%\n",
      "Step 7300 (epoch 0.00), 257.5 ms\n",
      "Minibatch loss: 0.047, learning rate: 0.006634\n",
      "Validation error: 0.9%\n",
      "Step 7400 (epoch 0.00), 243.5 ms\n",
      "Minibatch loss: 0.000, learning rate: 0.006634\n",
      "Validation error: 0.9%\n",
      "Step 7500 (epoch 0.00), 251.7 ms\n",
      "Minibatch loss: 0.023, learning rate: 0.006634\n",
      "Validation error: 1.0%\n",
      "Step 7600 (epoch 0.00), 239.3 ms\n",
      "Minibatch loss: 0.123, learning rate: 0.006634\n",
      "Validation error: 1.0%\n",
      "Step 7700 (epoch 0.00), 237.9 ms\n",
      "Minibatch loss: 0.006, learning rate: 0.006634\n",
      "Validation error: 1.1%\n",
      "Step 7800 (epoch 0.00), 237.3 ms\n",
      "Minibatch loss: 0.003, learning rate: 0.006302\n",
      "Validation error: 1.1%\n",
      "Step 7900 (epoch 0.00), 235.3 ms\n",
      "Minibatch loss: 0.001, learning rate: 0.006302\n",
      "Validation error: 1.0%\n",
      "Step 8000 (epoch 0.00), 239.0 ms\n",
      "Minibatch loss: 0.024, learning rate: 0.006302\n",
      "Validation error: 1.0%\n",
      "Step 8100 (epoch 0.00), 233.8 ms\n",
      "Minibatch loss: 0.007, learning rate: 0.006302\n",
      "Validation error: 1.0%\n",
      "Step 8200 (epoch 0.00), 239.0 ms\n",
      "Minibatch loss: 0.007, learning rate: 0.006302\n",
      "Validation error: 1.0%\n",
      "Step 8300 (epoch 0.00), 224.0 ms\n",
      "Minibatch loss: 0.013, learning rate: 0.006302\n",
      "Validation error: 1.0%\n",
      "Step 8400 (epoch 0.00), 137.7 ms\n",
      "Minibatch loss: 0.003, learning rate: 0.006302\n",
      "Validation error: 0.9%\n",
      "Step 8500 (epoch 0.00), 136.5 ms\n",
      "Minibatch loss: 0.020, learning rate: 0.006302\n",
      "Validation error: 0.9%\n",
      "Test error: 0.9%\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "with tf.Session() as sess:\n",
    "    tf.initialize_all_variables().run()\n",
    "    print('Initialized')\n",
    "    for step in xrange(int(num_epochs*train_size) // BATCH_SIZE):\n",
    "        offset = (step * BATCH_SIZE) % (train_size - BATCH_SIZE)\n",
    "        batch_data = train_data[offset:(offset + BATCH_SIZE),...]\n",
    "        batch_labels = train_labels[offset:(offset + BATCH_SIZE), ...]\n",
    "        feed_dict = {train_data_node: batch_data, train_labels_node: batch_labels}\n",
    "        _, l, lr, predictions = sess.run([optimizer, loss, learning_rate, train_prediction], feed_dict=feed_dict)\n",
    "        if step % EVAL_FREQUENCY == 0:\n",
    "            elapsed_time = time.time() - start_time\n",
    "            start_time = time.time()\n",
    "            print('Step %d (epoch %.2f), %.1f ms' % (step, float(step) % BATCH_SIZE / train_size, \n",
    "                                                     1000 * elapsed_time / EVAL_FREQUENCY))\n",
    "            print('Minibatch loss: %.3f, learning rate: %.6f' % (l, lr))\n",
    "            #print('Minibatch error: %.1f%%' % error_rate(predictions, batch_labels))\n",
    "            print('Validation error: %.1f%%' % error_rate(eval_in_batches(validation_data, sess), validation_labels))\n",
    "            sys.stdout.flush()\n",
    "    test_error = error_rate(eval_in_batches(test_data, sess), test_labels)\n",
    "    print('Test error: %.1f%%' % test_error)     "
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}