3c_MLP_MNIST_topNresults_160712.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\* *[Notice] I wrote thie code while following the examples in [Choi's Tesorflow-101 tutorial](https://github.com/sjchoi86/Tensorflow-101). And,  as I know, most of Choi's examples originally come from [Aymeric Damien's](https://github.com/aymericdamien/TensorFlow-Examples/) and  [Nathan Lintz's ](https://github.com/nlintz/TensorFlow-Tutorials) tutorials.*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Multilayer Perceptron with MNIST data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "#%matplotlib inline  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load MNIST data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/train-images-idx3-ubyte.gz\n",
      "Extracting data/train-labels-idx1-ubyte.gz\n",
      "Extracting data/t10k-images-idx3-ubyte.gz\n",
      "Extracting data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist      = input_data.read_data_sets('data', one_hot=True)\n",
    "X_train   = mnist.train.images\n",
    "Y_train = mnist.train.labels\n",
    "X_test    = mnist.test.images\n",
    "Y_test  = mnist.test.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of (X_train, X_test, Y_train, Y_test)\n",
      "((55000, 784), (10000, 784), (55000, 10), (10000, 10))\n"
     ]
    }
   ],
   "source": [
    "dimX = X_train.shape[1]\n",
    "dimY = Y_train.shape[1]\n",
    "nTrain = X_train.shape[0]\n",
    "nTest = X_test.shape[0]\n",
    "print (\"Shape of (X_train, X_test, Y_train, Y_test)\")\n",
    "print (X_train.shape, X_test.shape, Y_train.shape, Y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define my neural network structure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nLayer0 = dimX\n",
    "nLayer1 = 256\n",
    "nLayer2 = 256\n",
    "nLayer3 =  dimY\n",
    "sigma_init = 0.1   # For randomized initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "W = {\n",
    "    'W1': tf.Variable(tf.random_normal([nLayer0, nLayer1], stddev = sigma_init)),\n",
    "    'W2': tf.Variable(tf.random_normal([nLayer1, nLayer2], stddev = sigma_init)),\n",
    "    'W3': tf.Variable(tf.random_normal([nLayer2, nLayer3], stddev = sigma_init))\n",
    "}\n",
    "b = {\n",
    "    'b1': tf.Variable(tf.random_normal([nLayer1])),\n",
    "    'b2': tf.Variable(tf.random_normal([nLayer2])),\n",
    "    'b3': tf.Variable(tf.random_normal([nLayer3]))\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def model_myNN(_X, _W, _b):\n",
    "    Layer1 = tf.nn.sigmoid(tf.add(tf.matmul(_X,_W['W1']), _b['b1']))\n",
    "    Layer2 = tf.nn.sigmoid(tf.add(tf.matmul(Layer1,_W['W2']), _b['b2']))\n",
    "    Layer3 = tf.add(tf.matmul(Layer2,_W['W3']), _b['b3'])    \n",
    "    #Layer3 = tf.nn.sigmoid(tf.add(tf.matmul(Layer2,_W['W3']), _b['b3']))\n",
    "    return Layer3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = tf.placeholder(tf.float32, [None, dimX], name=\"input\")\n",
    "Y= tf.placeholder(tf.float32, [None, dimY], name=\"output\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "Y_pred = model_myNN(X, W, b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Define loss function, optimizer, measurer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use *softmax_cross_entropy()* and *AdamOptimizer()*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(Y_pred, Y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.001\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)\n",
    "# optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)\n",
    "training_epochs = 10\n",
    "display_epoch = 1\n",
    "batch_size = 100   # For each time, we will use 100 samples to update parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation - Top1, Top2, Top3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "correct_prediction = tf.equal(tf.argmax(Y_pred, 1), tf.argmax(Y, 1))    \n",
    "accuracy0 = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "accuracy1 = tf.reduce_mean(tf.cast(tf.nn.in_top_k(Y_pred,tf.argmax(Y, 1), k=1), \"float\"))\n",
    "accuracy2 = tf.reduce_mean(tf.cast(tf.nn.in_top_k(Y_pred,tf.argmax(Y, 1), k=2), \"float\"))\n",
    "accuracy3 = tf.reduce_mean(tf.cast(tf.nn.in_top_k(Y_pred,tf.argmax(Y, 1), k=3), \"float\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "is_top1 = tf.equal(tf.nn.top_k(Y_pred, k=1)[1][:,0], tf.cast(tf.argmax(Y, 1), \"int32\"))\n",
    "is_top2 = tf.equal(tf.nn.top_k(Y_pred, k=2)[1][:,1], tf.cast(tf.argmax(Y, 1), \"int32\"))\n",
    "is_top3 = tf.equal(tf.nn.top_k(Y_pred, k=3)[1][:,2], tf.cast(tf.argmax(Y, 1), \"int32\"))\n",
    "is_in_top1 = is_top1\n",
    "is_in_top2 = tf.logical_or(is_in_top1, is_top2)\n",
    "is_in_top3 = tf.logical_or(is_in_top2, is_top3)\n",
    "                   \n",
    "accuracy11 = tf.reduce_mean(tf.cast(is_in_top1, \"float\"))\n",
    "accuracy22 = tf.reduce_mean(tf.cast(is_in_top2, \"float\"))\n",
    "accuracy33 = tf.reduce_mean(tf.cast(is_in_top3, \"float\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run the session"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "We use *with* for load a TF session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Because of the memory allocation problem in evaluation\n",
    "divide_train = 45;\n",
    "divide_test = 5;\n",
    "nTrainSub = (int)(nTrain/divide_train);\n",
    "nTestSub = (int)(nTest/divide_test);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(epoch 1)\n",
      "[Loss / Tranin / Test] 0.2417 / 0.9292 / 0.9323\n",
      "[in_top_k: 1] [Train / Test] 0.9292 / 0.9323\n",
      "[in_top_k: 2] [Train / Test] 0.9719 / 0.9730\n",
      "[in_top_k: 3] [Train / Test] 0.9866 / 0.9863\n",
      "[top_k: 1] [Train / Test] 0.9292 / 0.9323\n",
      "[top_k: 2] [Train / Test] 0.9719 / 0.9730\n",
      "[top_k: 3] [Train / Test] 0.9866 / 0.9863\n",
      "[Test accuracy1]  0.9323\n",
      " \n",
      "(epoch 2)\n",
      "[Loss / Tranin / Test] 0.1680 / 0.9498 / 0.9499\n",
      "[in_top_k: 1] [Train / Test] 0.9498 / 0.9499\n",
      "[in_top_k: 2] [Train / Test] 0.9825 / 0.9815\n",
      "[in_top_k: 3] [Train / Test] 0.9921 / 0.9908\n",
      "[top_k: 1] [Train / Test] 0.9498 / 0.9499\n",
      "[top_k: 2] [Train / Test] 0.9825 / 0.9815\n",
      "[top_k: 3] [Train / Test] 0.9921 / 0.9908\n",
      "[Test accuracy1]  0.9499\n",
      " \n",
      "(epoch 3)\n",
      "[Loss / Tranin / Test] 0.1288 / 0.9624 / 0.9577\n",
      "[in_top_k: 1] [Train / Test] 0.9624 / 0.9577\n",
      "[in_top_k: 2] [Train / Test] 0.9877 / 0.9861\n",
      "[in_top_k: 3] [Train / Test] 0.9945 / 0.9939\n",
      "[top_k: 1] [Train / Test] 0.9624 / 0.9577\n",
      "[top_k: 2] [Train / Test] 0.9877 / 0.9861\n",
      "[top_k: 3] [Train / Test] 0.9945 / 0.9939\n",
      "[Test accuracy1]  0.9577\n",
      " \n",
      "(epoch 4)\n",
      "[Loss / Tranin / Test] 0.0974 / 0.9718 / 0.9656\n",
      "[in_top_k: 1] [Train / Test] 0.9718 / 0.9656\n",
      "[in_top_k: 2] [Train / Test] 0.9912 / 0.9893\n",
      "[in_top_k: 3] [Train / Test] 0.9965 / 0.9949\n",
      "[top_k: 1] [Train / Test] 0.9718 / 0.9656\n",
      "[top_k: 2] [Train / Test] 0.9912 / 0.9893\n",
      "[top_k: 3] [Train / Test] 0.9965 / 0.9949\n",
      "[Test accuracy1]  0.9656\n",
      " \n",
      "(epoch 5)\n",
      "[Loss / Tranin / Test] 0.0763 / 0.9777 / 0.9705\n",
      "[in_top_k: 1] [Train / Test] 0.9777 / 0.9705\n",
      "[in_top_k: 2] [Train / Test] 0.9937 / 0.9893\n",
      "[in_top_k: 3] [Train / Test] 0.9975 / 0.9961\n",
      "[top_k: 1] [Train / Test] 0.9777 / 0.9705\n",
      "[top_k: 2] [Train / Test] 0.9937 / 0.9893\n",
      "[top_k: 3] [Train / Test] 0.9975 / 0.9961\n",
      "[Test accuracy1]  0.9705\n",
      " \n",
      "(epoch 6)\n",
      "[Loss / Tranin / Test] 0.0625 / 0.9813 / 0.9716\n",
      "[in_top_k: 1] [Train / Test] 0.9813 / 0.9716\n",
      "[in_top_k: 2] [Train / Test] 0.9953 / 0.9912\n",
      "[in_top_k: 3] [Train / Test] 0.9984 / 0.9969\n",
      "[top_k: 1] [Train / Test] 0.9813 / 0.9716\n",
      "[top_k: 2] [Train / Test] 0.9953 / 0.9912\n",
      "[top_k: 3] [Train / Test] 0.9984 / 0.9969\n",
      "[Test accuracy1]  0.9716\n",
      " \n",
      "(epoch 7)\n",
      "[Loss / Tranin / Test] 0.0510 / 0.9851 / 0.9721\n",
      "[in_top_k: 1] [Train / Test] 0.9851 / 0.9721\n",
      "[in_top_k: 2] [Train / Test] 0.9968 / 0.9915\n",
      "[in_top_k: 3] [Train / Test] 0.9989 / 0.9969\n",
      "[top_k: 1] [Train / Test] 0.9851 / 0.9721\n",
      "[top_k: 2] [Train / Test] 0.9968 / 0.9915\n",
      "[top_k: 3] [Train / Test] 0.9989 / 0.9969\n",
      "[Test accuracy1]  0.9721\n",
      " \n",
      "(epoch 8)\n",
      "[Loss / Tranin / Test] 0.0391 / 0.9891 / 0.9772\n",
      "[in_top_k: 1] [Train / Test] 0.9891 / 0.9772\n",
      "[in_top_k: 2] [Train / Test] 0.9980 / 0.9927\n",
      "[in_top_k: 3] [Train / Test] 0.9994 / 0.9972\n",
      "[top_k: 1] [Train / Test] 0.9891 / 0.9772\n",
      "[top_k: 2] [Train / Test] 0.9980 / 0.9927\n",
      "[top_k: 3] [Train / Test] 0.9994 / 0.9972\n",
      "[Test accuracy1]  0.9772\n",
      " \n",
      "(epoch 9)\n",
      "[Loss / Tranin / Test] 0.0337 / 0.9900 / 0.9763\n",
      "[in_top_k: 1] [Train / Test] 0.9900 / 0.9763\n",
      "[in_top_k: 2] [Train / Test] 0.9986 / 0.9940\n",
      "[in_top_k: 3] [Train / Test] 0.9995 / 0.9977\n",
      "[top_k: 1] [Train / Test] 0.9900 / 0.9763\n",
      "[top_k: 2] [Train / Test] 0.9986 / 0.9940\n",
      "[top_k: 3] [Train / Test] 0.9995 / 0.9977\n",
      "[Test accuracy1]  0.9763\n",
      " \n",
      "(epoch 10)\n",
      "[Loss / Tranin / Test] 0.0248 / 0.9933 / 0.9770\n",
      "[in_top_k: 1] [Train / Test] 0.9933 / 0.9770\n",
      "[in_top_k: 2] [Train / Test] 0.9989 / 0.9933\n",
      "[in_top_k: 3] [Train / Test] 0.9997 / 0.9976\n",
      "[top_k: 1] [Train / Test] 0.9933 / 0.9770\n",
      "[top_k: 2] [Train / Test] 0.9989 / 0.9933\n",
      "[top_k: 3] [Train / Test] 0.9997 / 0.9976\n",
      "[Test accuracy1]  0.977\n",
      " \n",
      "[Test accuracy1]  0.977\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.initialize_all_variables())\n",
    "\n",
    "    for epoch in range(training_epochs):\n",
    "        nBatch  = int(nTrain/batch_size)\n",
    "        myIdx =  np.random.permutation(nTrain)\n",
    "        for ii in range(nBatch):\n",
    "            X_batch = X_train[myIdx[ii*batch_size:(ii+1)*batch_size],:]\n",
    "            Y_batch = Y_train[myIdx[ii*batch_size:(ii+1)*batch_size],:]\n",
    "            #print X_batch.shape, Y_batch.shape\n",
    "            sess.run(optimizer, feed_dict={X:X_batch, Y:Y_batch})\n",
    "          \n",
    "        if (epoch+1) % display_epoch == 0:\n",
    "            # Because of the memory allocation problem in evaluation\n",
    "            loss_temp = accuracy0_train_temp = accuracy0_test_temp = 0\n",
    "            accuracy1_train_temp = accuracy1_test_temp = 0\n",
    "            accuracy2_train_temp = accuracy2_test_temp = 0\n",
    "            accuracy3_train_temp = accuracy3_test_temp = 0\n",
    "            accuracy11_train_temp = accuracy11_test_temp = 0\n",
    "            accuracy22_train_temp = accuracy22_test_temp = 0\n",
    "            accuracy33_train_temp = accuracy33_test_temp = 0\n",
    "            \n",
    "            for jj in range(divide_train):\n",
    "                myIdx1 = jj*nTrainSub\n",
    "                myIdx2 = (jj+1)*nTrainSub\n",
    "                loss_temp += sess.run(loss, feed_dict={X: X_train[myIdx1:myIdx2,:], Y:Y_train[myIdx1:myIdx2,:]})\n",
    "                accuracy0_train_temp += accuracy0.eval({X: X_train[myIdx1:myIdx2,:], Y:Y_train[myIdx1:myIdx2,:]})\n",
    "                accuracy1_train_temp += accuracy1.eval({X: X_train[myIdx1:myIdx2,:], Y:Y_train[myIdx1:myIdx2,:]})\n",
    "                accuracy2_train_temp += accuracy2.eval({X: X_train[myIdx1:myIdx2,:], Y:Y_train[myIdx1:myIdx2,:]})\n",
    "                accuracy3_train_temp += accuracy3.eval({X: X_train[myIdx1:myIdx2,:], Y:Y_train[myIdx1:myIdx2,:]})\n",
    "                accuracy11_train_temp += accuracy11.eval({X: X_train[myIdx1:myIdx2,:], Y:Y_train[myIdx1:myIdx2,:]})\n",
    "                accuracy22_train_temp += accuracy22.eval({X: X_train[myIdx1:myIdx2,:], Y:Y_train[myIdx1:myIdx2,:]})\n",
    "                accuracy33_train_temp += accuracy33.eval({X: X_train[myIdx1:myIdx2,:], Y:Y_train[myIdx1:myIdx2,:]})\n",
    "\n",
    "            for kk in range(divide_test):\n",
    "                myIdx1 = kk*nTestSub\n",
    "                myIdx2 = (kk+1)*nTestSub\n",
    "                accuracy0_test_temp += accuracy0.eval({X: X_test[myIdx1:myIdx2,:], Y: Y_test[myIdx1:myIdx2,:]}) \n",
    "                accuracy1_test_temp += accuracy1.eval({X: X_test[myIdx1:myIdx2,:], Y:Y_test[myIdx1:myIdx2,:]})\n",
    "                accuracy2_test_temp += accuracy2.eval({X: X_test[myIdx1:myIdx2,:], Y:Y_test[myIdx1:myIdx2,:]})\n",
    "                accuracy3_test_temp += accuracy3.eval({X: X_test[myIdx1:myIdx2,:], Y:Y_test[myIdx1:myIdx2,:]})\n",
    "                accuracy11_test_temp += accuracy11.eval({X: X_test[myIdx1:myIdx2,:], Y:Y_test[myIdx1:myIdx2,:]})\n",
    "                accuracy22_test_temp += accuracy22.eval({X: X_test[myIdx1:myIdx2,:], Y:Y_test[myIdx1:myIdx2,:]})\n",
    "                accuracy33_test_temp += accuracy33.eval({X: X_test[myIdx1:myIdx2,:], Y:Y_test[myIdx1:myIdx2,:]})\n",
    "            print \"(epoch {})\".format(epoch+1) \n",
    "            print \"[Loss / Tranin / Test] {:05.4f} / {:05.4f} / {:05.4f}\".format(loss_temp/divide_train, accuracy0_train_temp/divide_train, accuracy0_test_temp/divide_test)\n",
    "            print \"[in_top_k: 1] [Train / Test] {:05.4f} / {:05.4f}\".format(accuracy1_train_temp/divide_train, accuracy1_test_temp/divide_test)\n",
    "            print \"[in_top_k: 2] [Train / Test] {:05.4f} / {:05.4f}\".format(accuracy2_train_temp/divide_train, accuracy2_test_temp/divide_test)\n",
    "            print \"[in_top_k: 3] [Train / Test] {:05.4f} / {:05.4f}\".format(accuracy3_train_temp/divide_train, accuracy3_test_temp/divide_test)\n",
    "            print \"[top_k: 1] [Train / Test] {:05.4f} / {:05.4f}\".format(accuracy11_train_temp/divide_train, accuracy11_test_temp/divide_test)\n",
    "            print \"[top_k: 2] [Train / Test] {:05.4f} / {:05.4f}\".format(accuracy22_train_temp/divide_train, accuracy22_test_temp/divide_test)\n",
    "            print \"[top_k: 3] [Train / Test] {:05.4f} / {:05.4f}\".format(accuracy33_train_temp/divide_train, accuracy33_test_temp/divide_test)\n",
    "            print \"[Test accuracy1] \",  accuracy1.eval({X: X_test, Y: Y_test})   \n",
    "            print \" \"\n",
    "    \n",
    "    print \"[Test accuracy1] \",  accuracy1.eval({X: X_test, Y: Y_test})   "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}