0002_94_accuracy_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import time\n",
    "import tensorflow as tf\n",
    "\n",
    "# https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/tutorials/mnist/input_data.py\n",
    "import input_data\n",
    "def weight_bias(W_shape, b_shape, bias_init=0.1):\n",
    "    W = tf.Variable(tf.truncated_normal(W_shape, stddev=0.1))\n",
    "    b = tf.Variable(tf.constant(bias_init, shape=b_shape), name='bias')\n",
    "    return W, b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def dense_layer(x, W_shape, b_shape, activation):\n",
    "    W, b = weight_bias(W_shape, b_shape)\n",
    "    return activation(tf.matmul(x, W) + b)\n",
    "\n",
    "def conv2d_layer(x, W_shape, b_shape, strides, padding):\n",
    "    W, b = weight_bias(W_shape, b_shape)\n",
    "    return tf.nn.relu(tf.nn.conv2d(x, W, strides, padding) + b)\n",
    "\n",
    "def highway_conv2d_layer(x, W_shape, b_shape, strides, padding, carry_bias=-1.0):\n",
    "    W, b = weight_bias(W_shape, b_shape, carry_bias)\n",
    "    W_T, b_T = weight_bias(W_shape, b_shape)\n",
    "    H = tf.nn.relu(tf.nn.conv2d(x, W, strides, padding) + b, name='activation')\n",
    "    T = tf.sigmoid(tf.nn.conv2d(x, W_T, strides, padding) + b_T, name='transform_gate')\n",
    "    C = tf.sub(1.0, T, name=\"carry_gate\")\n",
    "    return tf.add(tf.mul(H, T), tf.mul(x, C), 'y') # y = (H * T) + (x * C)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = tf.placeholder(\"float\", [None, 784])\n",
    "y_ = tf.placeholder(\"float\", [None, 10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "carry_bias_init = -1.0\n",
    "\n",
    "x_image = tf.reshape(x, [-1, 28, 28, 1]) # reshape for conv\n",
    "\n",
    "keep_prob1 = tf.placeholder(\"float\", name=\"keep_prob1\")\n",
    "x_drop = tf.nn.dropout(x_image, keep_prob1)\n",
    "\n",
    "prev_y = conv2d_layer(x_drop, [5, 5, 1, 32], [32], [1, 1, 1, 1], 'SAME')\n",
    "prev_y = highway_conv2d_layer(prev_y, [3, 3, 32, 32], [32], [1, 1, 1, 1], 'SAME', carry_bias=carry_bias_init)\n",
    "prev_y = highway_conv2d_layer(prev_y, [3, 3, 32, 32], [32], [1, 1, 1, 1], 'SAME', carry_bias=carry_bias_init)\n",
    "\n",
    "prev_y = tf.nn.max_pool(prev_y, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "\n",
    "keep_prob2 = tf.placeholder(\"float\", name=\"keep_prob2\")\n",
    "prev_y = tf.nn.dropout(prev_y, keep_prob2)\n",
    "\n",
    "prev_y = highway_conv2d_layer(prev_y, [3, 3, 32, 32], [32], [1, 1, 1, 1], 'SAME', carry_bias=carry_bias_init)\n",
    "prev_y = highway_conv2d_layer(prev_y, [3, 3, 32, 32], [32], [1, 1, 1, 1], 'SAME', carry_bias=carry_bias_init)\n",
    "prev_y = highway_conv2d_layer(prev_y, [3, 3, 32, 32], [32], [1, 1, 1, 1], 'SAME', carry_bias=carry_bias_init)\n",
    "\n",
    "prev_y = tf.nn.max_pool(prev_y, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "\n",
    "keep_prob3 = tf.placeholder(\"float\", name=\"keep_prob3\")\n",
    "prev_y = tf.nn.dropout(prev_y, keep_prob3)\n",
    "\n",
    "prev_y = highway_conv2d_layer(prev_y, [3, 3, 32, 32], [32], [1, 1, 1, 1], 'SAME', carry_bias=carry_bias_init)\n",
    "prev_y = highway_conv2d_layer(prev_y, [3, 3, 32, 32], [32], [1, 1, 1, 1], 'SAME', carry_bias=carry_bias_init)\n",
    "prev_y = highway_conv2d_layer(prev_y, [3, 3, 32, 32], [32], [1, 1, 1, 1], 'SAME', carry_bias=carry_bias_init)\n",
    "\n",
    "prev_y = tf.nn.max_pool(prev_y, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "\n",
    "keep_prob4 = tf.placeholder(\"float\", name=\"keep_prob4\")\n",
    "prev_y = tf.nn.dropout(prev_y, keep_prob4)\n",
    "\n",
    "prev_y = tf.reshape(prev_y, [-1, 4 * 4 * 32])\n",
    "y = dense_layer(prev_y, [4 * 4 * 32, 10], [10], tf.nn.softmax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\") as scope:\n",
    "    cross_entropy = -tf.reduce_sum(y_ * tf.log(y))\n",
    "\n",
    "with tf.name_scope(\"train\") as scope:\n",
    "    train_step = tf.train.GradientDescentOptimizer(1e-2).minimize(cross_entropy)\n",
    "\n",
    "with tf.name_scope(\"test\") as scope:\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.initialize_all_variables().run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 0, training accuracy 0.1\n",
      "step 100, training accuracy 0.1\n",
      "step 200, training accuracy 0.16\n",
      "step 300, training accuracy 0.14\n",
      "step 400, training accuracy 0.12\n",
      "step 500, training accuracy 0.12\n",
      "step 600, training accuracy 0.12\n",
      "step 700, training accuracy 0.12\n",
      "step 800, training accuracy 0.34\n",
      "step 900, training accuracy 0.6\n",
      "step 1000, training accuracy 0.74\n",
      "step 1100, training accuracy 0.44\n",
      "step 1200, training accuracy 0.64\n",
      "step 1300, training accuracy 0.64\n",
      "step 1400, training accuracy 0.74\n",
      "step 1500, training accuracy 0.72\n",
      "step 1600, training accuracy 0.98\n",
      "step 1700, training accuracy 0.96\n",
      "step 1800, training accuracy 0.92\n",
      "step 1900, training accuracy 0.94\n"
     ]
    }
   ],
   "source": [
    "batch_size = 50\n",
    "\n",
    "for i in range(2000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "\n",
    "    if i % 100 == 0:\n",
    "        train_accuracy = accuracy.eval(feed_dict={\n",
    "            x: batch_xs,\n",
    "            y_: batch_ys,\n",
    "            keep_prob1: 1.0,\n",
    "            keep_prob2: 1.0,\n",
    "            keep_prob3: 1.0,\n",
    "            keep_prob4: 1.0,\n",
    "        })\n",
    "        print(\"step %d, training accuracy %g\" % (i, train_accuracy))\n",
    "\n",
    "    train_step.run(feed_dict={\n",
    "        x: batch_xs,\n",
    "        y_: batch_ys,\n",
    "        keep_prob1: 0.8,\n",
    "        keep_prob2: 0.7,\n",
    "        keep_prob3: 0.6,\n",
    "        keep_prob4: 0.5,\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test accuracy 0.9431\n"
     ]
    }
   ],
   "source": [
    "print(\"test accuracy %g\" % accuracy.eval(feed_dict={\n",
    "    x: mnist.test.images,\n",
    "    y_: mnist.test.labels,\n",
    "    keep_prob1: 1.0,\n",
    "    keep_prob2: 1.0,\n",
    "    keep_prob3: 1.0,\n",
    "    keep_prob4: 1.0,\n",
    "}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}