VGG_MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n",
      "/opt/conda/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.layers.base import Layer\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST-data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST-data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST-data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST-data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist = tf.contrib.learn.datasets.load_dataset(\"mnist\")\n",
    "train_imgs = mnist.train.images\n",
    "train_labels = mnist.train.labels\n",
    "test_imgs = mnist.test.images\n",
    "test_labels = mnist.test.labels\n",
    "valid_imgs = mnist.validation.images\n",
    "valid_labels = mnist.validation.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "img_width = 28\n",
    "img_height = 28\n",
    "n_channel = 1\n",
    "dropout_rate = 0.1\n",
    "output_size = 10\n",
    "\n",
    "n_epoch = 100\n",
    "n_train_iteration = len(train_imgs) // batch_size\n",
    "n_eval_train = n_train_iteration // 3\n",
    "n_eval_valid = len(valid_imgs) // batch_size\n",
    "n_eval_test = len(test_imgs) // batch_size\n",
    "print_every = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class conv_block(Layer):\n",
    "    def __init__(self, num_filter, num_conv):\n",
    "        self.num_filter = num_filter\n",
    "        self.num_conv = num_conv\n",
    "        self.batch_norm1 = tf.layers.BatchNormalization(axis=3, name='batch1')\n",
    "        self.batch_norm2 = tf.layers.BatchNormalization(axis=3, name='batch2')\n",
    "        self.batch_norm3 = tf.layers.BatchNormalization(axis=3, name='batch3')\n",
    "        self.conv1 = tf.layers.Conv2D(filters=self.num_filter, kernel_size=3, padding='same', activation=tf.nn.relu, name='conv1')\n",
    "        self.conv2 = tf.layers.Conv2D(filters=self.num_filter, kernel_size=3, padding='same', activation=tf.nn.relu, name='conv2')\n",
    "        self.conv3 = tf.layers.Conv2D(filters=self.num_filter, kernel_size=3, padding='same', activation=tf.nn.relu, name='conv3')\n",
    "        self.maxpool = tf.layers.MaxPooling2D(pool_size=2, strides=2, padding='valid', name='maxpool')\n",
    "    \n",
    "    def __call__(self, inputs, is_train):\n",
    "        conv_layer = self.batch_norm1(inputs, training=is_train)\n",
    "        conv_layer = self.conv1(conv_layer)\n",
    "        conv_layer = self.batch_norm2(conv_layer, training=is_train)\n",
    "        conv_layer = self.conv2(conv_layer)\n",
    "        if self.num_conv == 3:\n",
    "            conv_layer = self.batch_norm3(conv_layer, training=is_train)\n",
    "            conv_layer = self.conv3(conv_layer)\n",
    "        conv_layer = self.maxpool(conv_layer)\n",
    "        return conv_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VGG(Layer):\n",
    "    def __init__(self):\n",
    "        self.conv_64 = conv_block(64, 2)\n",
    "        self.conv_128 = conv_block(128, 2)\n",
    "        self.conv_256 = conv_block(256, 3)\n",
    "        self.conv_512_1 = conv_block(512, 3)\n",
    "        self.conv_512_2 = conv_block(512, 3)\n",
    "        self.dens_1 = tf.layers.Dense(units=4096, activation=tf.nn.relu, name='dens1')\n",
    "        self.dens_2 = tf.layers.Dense(units=4096, activation=tf.nn.relu, name='dens2')\n",
    "        self.drop_1 = tf.layers.Dropout(rate=dropout_rate)\n",
    "        self.drop_2 = tf.layers.Dropout(rate=dropout_rate)\n",
    "        self.logit = tf.layers.Dense(units=output_size)\n",
    "        \n",
    "    def __call__(self, inputs, is_train):\n",
    "        input_layer = tf.reshape(inputs, [batch_size, img_height, img_width, n_channel])\n",
    "        input_resize = tf.image.resize_nearest_neighbor(input_layer, size=[224,224], name='resize')\n",
    "        conv_layer1 = self.conv_64(input_resize, is_train)\n",
    "        conv_layer2 = self.conv_128(conv_layer1, is_train)\n",
    "        conv_layer3 = self.conv_256(conv_layer2, is_train)\n",
    "        conv_layer4 = self.conv_512_1(conv_layer3, is_train)\n",
    "        conv_layer5 = self.conv_512_2(conv_layer4, is_train)\n",
    "        flatten_layer = tf.layers.flatten(conv_layer5)\n",
    "        dens_layer1 = self.dens_1(flatten_layer)\n",
    "        drop_layer1 = self.drop_1(dens_layer1, training=is_train)\n",
    "        dens_layer2 = self.dens_2(drop_layer1)\n",
    "        drop_layer2 = self.drop_2(dens_layer2, training=is_train)\n",
    "        logits = self.logit(drop_layer2)\n",
    "        return logits\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "vgg_model = VGG()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = tf.placeholder(dtype=tf.float32, shape=[batch_size, 784])\n",
    "y = tf.placeholder(dtype=tf.int32, shape = [batch_size])\n",
    "is_train = tf.placeholder(dtype = tf.bool)\n",
    "\n",
    "y_onehot = tf.one_hot(indices=tf.cast(y, tf.int32), depth=10)\n",
    "logits = vgg_model(X, is_train)\n",
    "loss = tf.losses.softmax_cross_entropy(y_onehot, logits)\n",
    "\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)\n",
    "# optimizer = tf.train.RMSPropOptimizer(learning_rate=1e-3, momentum=0.9)\n",
    "\n",
    "train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())\n",
    "predictions = tf.argmax(logits, axis=1)\n",
    "accuracy = tf.metrics.accuracy(y, predictions)\n",
    "\n",
    "extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "with tf.control_dependencies(extra_update_ops):\n",
    "    train_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train():\n",
    "#     n_iteration = 2\n",
    "    for ite in range(n_train_iteration):\n",
    "        images,labels = mnist.train.next_batch(batch_size)\n",
    "        _,_,loss_value,_ = sess.run([y_onehot, logits, loss, train_op],feed_dict={X:images, y:labels, is_train:True})\n",
    "        if (ite+1)%print_every == 0:\n",
    "            print('[ite %d]loss=%f'%(ite+1, loss_value))\n",
    "\n",
    "def evaluation():\n",
    "    train_loss,valid_loss,test_loss = 0.0,0.0,0.0\n",
    "    train_acc,valid_acc,test_acc = 0.0,0.0,0.0\n",
    "    for ite in range(n_eval_train):\n",
    "        images,labels = mnist.train.next_batch(batch_size)\n",
    "        _,_,loss_value,_,acc = sess.run([y_onehot, logits, loss, predictions, accuracy], feed_dict={X:images, y:labels, is_train:False})\n",
    "        train_loss += loss_value\n",
    "        train_acc += acc[0]\n",
    "\n",
    "    train_loss /= n_eval_train\n",
    "    train_acc /= n_eval_train\n",
    "    \n",
    "    for ite in range(n_eval_valid):\n",
    "        images,labels = mnist.validation.next_batch(batch_size)\n",
    "        _,_,loss_value,_,acc = sess.run([y_onehot, logits, loss, predictions, accuracy], feed_dict={X:images, y:labels, is_train:False})\n",
    "        valid_loss += loss_value\n",
    "        valid_acc += acc[0]\n",
    "    valid_loss /= n_eval_valid\n",
    "    valid_acc /= n_eval_valid\n",
    "    \n",
    "    for ite in range(n_eval_test):\n",
    "        images,labels = mnist.test.next_batch(batch_size)\n",
    "        _,_,loss_value,_,acc = sess.run([y_onehot, logits, loss, predictions, accuracy], feed_dict={X:images, y:labels, is_train:False})\n",
    "        test_loss += loss_value\n",
    "        test_acc += acc[0]\n",
    "    test_loss /= n_eval_test\n",
    "    test_acc /= n_eval_test\n",
    "    \n",
    "    return train_loss,valid_loss,test_loss,train_acc,valid_acc,test_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start:14:40:42\n",
      "[ite 100]loss=0.594229\n",
      "[ite 200]loss=0.251106\n",
      "[ite 300]loss=0.361489\n",
      "[ite 400]loss=0.446248\n",
      "[ite 500]loss=0.243903\n",
      "[ite 600]loss=0.099808\n",
      "[ite 700]loss=0.074153\n",
      "[ite 800]loss=0.156157\n",
      "[ite 900]loss=0.051852\n",
      "[ite 1000]loss=0.092173\n",
      "[ite 1100]loss=0.030079\n",
      "[ite 1200]loss=0.133780\n",
      "[ite 1300]loss=0.059157\n",
      "[ite 1400]loss=0.087693\n",
      "[ite 1500]loss=0.071187\n",
      "[ite 1600]loss=0.086596\n",
      "[ite 1700]loss=0.177253\n",
      "[Epoch 1]14:53:54\n",
      "train_loss=0.082762, train_acc=0.977972, valid_loss=0.085654, valid_acc=0.979558, test_loss=0.079694, test_acc=0.979049\n",
      "[ite 100]loss=0.044159\n",
      "[ite 200]loss=0.014432\n",
      "[ite 300]loss=0.047107\n",
      "[ite 400]loss=0.013024\n",
      "[ite 500]loss=0.006498\n",
      "[ite 600]loss=0.073465\n",
      "[ite 700]loss=0.075118\n",
      "[ite 800]loss=0.048996\n",
      "[ite 900]loss=0.052494\n",
      "[ite 1000]loss=0.166511\n",
      "[ite 1100]loss=0.027743\n",
      "[ite 1200]loss=0.004289\n",
      "[ite 1300]loss=0.017507\n",
      "[ite 1400]loss=0.034557\n",
      "[ite 1500]loss=0.140650\n",
      "[ite 1600]loss=0.009237\n",
      "[ite 1700]loss=0.046320\n",
      "[Epoch 2]15:07:02\n",
      "train_loss=0.034615, train_acc=0.981201, valid_loss=0.046270, valid_acc=0.983023, test_loss=0.038508, test_acc=0.983521\n",
      "[ite 100]loss=0.011808\n",
      "[ite 200]loss=0.003475\n",
      "[ite 300]loss=0.010810\n",
      "[ite 400]loss=0.004938\n",
      "[ite 500]loss=0.111724\n",
      "[ite 600]loss=0.005350\n",
      "[ite 700]loss=0.041221\n",
      "[ite 800]loss=0.002670\n",
      "[ite 900]loss=0.060337\n",
      "[ite 1000]loss=0.058475\n",
      "[ite 1100]loss=0.005438\n",
      "[ite 1200]loss=0.013525\n",
      "[ite 1300]loss=0.013414\n",
      "[ite 1400]loss=0.039369\n",
      "[ite 1500]loss=0.149276\n",
      "[ite 1600]loss=0.026916\n",
      "[ite 1700]loss=0.002150\n",
      "[Epoch 3]15:20:11\n",
      "train_loss=0.028339, train_acc=0.984701, valid_loss=0.044128, valid_acc=0.985580, test_loss=0.040613, test_acc=0.985725\n",
      "[ite 100]loss=0.030978\n",
      "[ite 200]loss=0.001399\n",
      "[ite 300]loss=0.009859\n",
      "[ite 400]loss=0.007516\n",
      "[ite 500]loss=0.048942\n",
      "[ite 600]loss=0.103325\n",
      "[ite 700]loss=0.001970\n",
      "[ite 800]loss=0.002707\n",
      "[ite 900]loss=0.039847\n",
      "[ite 1000]loss=0.006620\n",
      "[ite 1100]loss=0.003421\n",
      "[ite 1200]loss=0.003920\n",
      "[ite 1300]loss=0.007906\n",
      "[ite 1400]loss=0.029730\n",
      "[ite 1500]loss=0.042033\n",
      "[ite 1600]loss=0.003664\n",
      "[ite 1700]loss=0.007428\n",
      "[Epoch 4]15:33:18\n",
      "train_loss=0.010589, train_acc=0.986749, valid_loss=0.035503, valid_acc=0.987668, test_loss=0.031737, test_acc=0.987774\n",
      "[ite 100]loss=0.127592\n",
      "[ite 200]loss=0.018926\n",
      "[ite 300]loss=0.001281\n",
      "[ite 400]loss=0.009099\n",
      "[ite 500]loss=0.014013\n",
      "[ite 600]loss=0.007874\n",
      "[ite 700]loss=0.035658\n",
      "[ite 800]loss=0.006595\n",
      "[ite 900]loss=0.004393\n",
      "[ite 1000]loss=0.010697\n",
      "[ite 1100]loss=0.003169\n",
      "[ite 1200]loss=0.045073\n",
      "[ite 1300]loss=0.003416\n",
      "[ite 1400]loss=0.001488\n",
      "[ite 1500]loss=0.010844\n",
      "[ite 1600]loss=0.003681\n",
      "[ite 1700]loss=0.006271\n",
      "[Epoch 5]15:46:26\n",
      "train_loss=0.006234, train_acc=0.988562, valid_loss=0.032985, valid_acc=0.989225, test_loss=0.027054, test_acc=0.989281\n",
      "[ite 100]loss=0.018224\n",
      "[ite 200]loss=0.006311\n",
      "[ite 300]loss=0.000489\n",
      "[ite 400]loss=0.001281\n",
      "[ite 500]loss=0.000681\n",
      "[ite 600]loss=0.039360\n",
      "[ite 700]loss=0.006275\n",
      "[ite 800]loss=0.000990\n",
      "[ite 900]loss=0.005863\n",
      "[ite 1000]loss=0.004607\n",
      "[ite 1100]loss=0.022456\n",
      "[ite 1200]loss=0.010908\n",
      "[ite 1300]loss=0.006638\n",
      "[ite 1400]loss=0.006902\n",
      "[ite 1500]loss=0.007086\n",
      "[ite 1600]loss=0.020194\n",
      "[ite 1700]loss=0.002857\n",
      "[Epoch 6]15:59:32\n",
      "train_loss=0.005242, train_acc=0.989838, valid_loss=0.031767, valid_acc=0.990357, test_loss=0.026980, test_acc=0.990358\n",
      "[ite 100]loss=0.001952\n",
      "[ite 200]loss=0.001035\n",
      "[ite 300]loss=0.001509\n",
      "[ite 400]loss=0.002746\n",
      "[ite 500]loss=0.002604\n",
      "[ite 600]loss=0.003200\n",
      "[ite 700]loss=0.006655\n",
      "[ite 800]loss=0.006712\n",
      "[ite 900]loss=0.000108\n",
      "[ite 1000]loss=0.000532\n",
      "[ite 1100]loss=0.001205\n",
      "[ite 1200]loss=0.001587\n",
      "[ite 1300]loss=0.002621\n",
      "[ite 1400]loss=0.000461\n",
      "[ite 1500]loss=0.000696\n",
      "[ite 1600]loss=0.003951\n",
      "[ite 1700]loss=0.003594\n",
      "[Epoch 7]16:12:42\n",
      "train_loss=0.002647, train_acc=0.990771, valid_loss=0.032738, valid_acc=0.991163, test_loss=0.027586, test_acc=0.991153\n",
      "[ite 100]loss=0.003062\n",
      "[ite 200]loss=0.000817\n",
      "[ite 300]loss=0.000670\n",
      "[ite 400]loss=0.006018\n",
      "[ite 500]loss=0.008671\n",
      "[ite 600]loss=0.005150\n",
      "[ite 700]loss=0.001184\n",
      "[ite 800]loss=0.001428\n",
      "[ite 900]loss=0.002185\n",
      "[ite 1000]loss=0.001645\n",
      "[ite 1100]loss=0.000762\n",
      "[ite 1200]loss=0.000497\n",
      "[ite 1300]loss=0.002077\n",
      "[ite 1400]loss=0.001455\n",
      "[ite 1500]loss=0.003462\n",
      "[ite 1600]loss=0.000176\n",
      "[ite 1700]loss=0.002328\n",
      "[Epoch 8]16:25:49\n",
      "train_loss=0.001885, train_acc=0.991441, valid_loss=0.032107, valid_acc=0.991736, test_loss=0.025262, test_acc=0.991705\n",
      "[ite 100]loss=0.001670\n",
      "[ite 200]loss=0.001572\n",
      "[ite 300]loss=0.013647\n",
      "[ite 400]loss=0.004285\n",
      "[ite 500]loss=0.002480\n",
      "[ite 600]loss=0.000530\n",
      "[ite 700]loss=0.004234\n",
      "[ite 800]loss=0.002584\n",
      "[ite 900]loss=0.001778\n",
      "[ite 1000]loss=0.004372\n",
      "[ite 1100]loss=0.001131\n",
      "[ite 1200]loss=0.003070\n",
      "[ite 1300]loss=0.001056\n",
      "[ite 1400]loss=0.004074\n",
      "[ite 1500]loss=0.000609\n",
      "[ite 1600]loss=0.006552\n",
      "[ite 1700]loss=0.000999\n",
      "[Epoch 9]16:38:57\n",
      "train_loss=0.001756, train_acc=0.991957, valid_loss=0.032301, valid_acc=0.992213, test_loss=0.024588, test_acc=0.992166\n",
      "[ite 100]loss=0.002719\n",
      "[ite 200]loss=0.002349\n",
      "[ite 300]loss=0.006704\n",
      "[ite 400]loss=0.004313\n",
      "[ite 500]loss=0.001329\n",
      "[ite 600]loss=0.003969\n",
      "[ite 700]loss=0.001201\n",
      "[ite 800]loss=0.004104\n",
      "[ite 900]loss=0.008347\n",
      "[ite 1000]loss=0.001264\n",
      "[ite 1100]loss=0.000781\n",
      "[ite 1200]loss=0.017315\n",
      "[ite 1300]loss=0.000507\n",
      "[ite 1400]loss=0.001614\n",
      "[ite 1500]loss=0.005171\n",
      "[ite 1600]loss=0.032920\n",
      "[ite 1700]loss=0.009664\n",
      "[Epoch 10]16:52:04\n",
      "train_loss=0.001351, train_acc=0.992394, valid_loss=0.031950, valid_acc=0.992600, test_loss=0.023427, test_acc=0.992555\n",
      "[ite 100]loss=0.000388\n",
      "[ite 200]loss=0.000338\n",
      "[ite 300]loss=0.002282\n",
      "[ite 400]loss=0.002041\n",
      "[ite 500]loss=0.001757\n",
      "[ite 600]loss=0.001415\n",
      "[ite 700]loss=0.000304\n",
      "[ite 800]loss=0.004015\n",
      "[ite 900]loss=0.005085\n",
      "[ite 1000]loss=0.000517\n",
      "[ite 1100]loss=0.000458\n",
      "[ite 1200]loss=0.005036\n",
      "[ite 1300]loss=0.000353\n",
      "[ite 1400]loss=0.003227\n",
      "[ite 1500]loss=0.004280\n",
      "[ite 1600]loss=0.002912\n",
      "[ite 1700]loss=0.000309\n",
      "[Epoch 11]17:05:10\n",
      "train_loss=0.001055, train_acc=0.992737, valid_loss=0.031268, valid_acc=0.992917, test_loss=0.025004, test_acc=0.992881\n",
      "[ite 100]loss=0.000438\n",
      "[ite 200]loss=0.001539\n",
      "[ite 300]loss=0.002152\n",
      "[ite 400]loss=0.000986\n",
      "[ite 500]loss=0.002100\n",
      "[ite 600]loss=0.012074\n",
      "[ite 700]loss=0.000597\n",
      "[ite 800]loss=0.000327\n",
      "[ite 900]loss=0.000474\n",
      "[ite 1000]loss=0.009793\n",
      "[ite 1100]loss=0.002289\n",
      "[ite 1200]loss=0.001495\n",
      "[ite 1300]loss=0.000992\n",
      "[ite 1400]loss=0.001066\n",
      "[ite 1500]loss=0.000880\n",
      "[ite 1600]loss=0.001755\n",
      "[ite 1700]loss=0.000261\n",
      "[Epoch 12]17:18:16\n",
      "train_loss=0.000824, train_acc=0.993021, valid_loss=0.031814, valid_acc=0.993177, test_loss=0.024574, test_acc=0.993133\n",
      "[ite 100]loss=0.001063\n",
      "[ite 200]loss=0.001218\n",
      "[ite 300]loss=0.000686\n",
      "[ite 400]loss=0.000184\n",
      "[ite 500]loss=0.002396\n",
      "[ite 600]loss=0.003539\n",
      "[ite 700]loss=0.005603\n",
      "[ite 800]loss=0.000626\n",
      "[ite 900]loss=0.001008\n",
      "[ite 1000]loss=0.001302\n",
      "[ite 1100]loss=0.005349\n",
      "[ite 1200]loss=0.003097\n",
      "[ite 1300]loss=0.003681\n",
      "[ite 1400]loss=0.001773\n",
      "[ite 1500]loss=0.000423\n",
      "[ite 1600]loss=0.001957\n",
      "[ite 1700]loss=0.006858\n",
      "[Epoch 13]17:31:22\n",
      "train_loss=0.000636, train_acc=0.993264, valid_loss=0.030992, valid_acc=0.993404, test_loss=0.025069, test_acc=0.993366\n",
      "[ite 100]loss=0.000743\n",
      "[ite 200]loss=0.000112\n",
      "[ite 300]loss=0.005782\n",
      "[ite 400]loss=0.001388\n",
      "[ite 500]loss=0.004529\n",
      "[ite 600]loss=0.000720\n",
      "[ite 700]loss=0.002302\n",
      "[ite 800]loss=0.001509\n",
      "[ite 900]loss=0.001015\n",
      "[ite 1000]loss=0.001438\n",
      "[ite 1100]loss=0.001699\n",
      "[ite 1200]loss=0.002584\n",
      "[ite 1300]loss=0.003822\n",
      "[ite 1400]loss=0.000481\n",
      "[ite 1500]loss=0.000586\n",
      "[ite 1600]loss=0.000397\n",
      "[ite 1700]loss=0.000634\n",
      "[Epoch 14]17:44:28\n",
      "train_loss=0.000619, train_acc=0.993482, valid_loss=0.033115, valid_acc=0.993606, test_loss=0.024956, test_acc=0.993575\n",
      "[ite 100]loss=0.001861\n",
      "[ite 200]loss=0.000113\n",
      "[ite 300]loss=0.000712\n",
      "[ite 400]loss=0.000933\n",
      "[ite 500]loss=0.001578\n",
      "[ite 600]loss=0.000965\n",
      "[ite 700]loss=0.000404\n",
      "[ite 800]loss=0.000563\n",
      "[ite 900]loss=0.000233\n",
      "[ite 1000]loss=0.000695\n",
      "[ite 1100]loss=0.000941\n",
      "[ite 1200]loss=0.001653\n",
      "[ite 1300]loss=0.001979\n",
      "[ite 1400]loss=0.001850\n",
      "[ite 1500]loss=0.000667\n",
      "[ite 1600]loss=0.000315\n",
      "[ite 1700]loss=0.007201\n",
      "[Epoch 15]17:57:34\n",
      "train_loss=0.000532, train_acc=0.993672, valid_loss=0.032233, valid_acc=0.993777, test_loss=0.024743, test_acc=0.993748\n",
      "[ite 100]loss=0.001351\n",
      "[ite 200]loss=0.000625\n",
      "[ite 300]loss=0.000294\n",
      "[ite 400]loss=0.000367\n",
      "[ite 500]loss=0.001047\n",
      "[ite 600]loss=0.000221\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ite 700]loss=0.000455\n",
      "[ite 800]loss=0.000316\n",
      "[ite 900]loss=0.000474\n",
      "[ite 1000]loss=0.000762\n",
      "[ite 1100]loss=0.000307\n",
      "[ite 1200]loss=0.002288\n",
      "[ite 1300]loss=0.001634\n",
      "[ite 1400]loss=0.000821\n",
      "[ite 1500]loss=0.001137\n",
      "[ite 1600]loss=0.001877\n",
      "[ite 1700]loss=0.001565\n",
      "[Epoch 16]18:10:40\n",
      "train_loss=0.000433, train_acc=0.993832, valid_loss=0.031319, valid_acc=0.993939, test_loss=0.024647, test_acc=0.993909\n",
      "[ite 100]loss=0.000485\n",
      "[ite 200]loss=0.000346\n",
      "[ite 300]loss=0.000180\n",
      "[ite 400]loss=0.000560\n",
      "[ite 500]loss=0.004131\n",
      "[ite 600]loss=0.000609\n",
      "[ite 700]loss=0.002925\n",
      "[ite 800]loss=0.001448\n",
      "[ite 900]loss=0.000405\n",
      "[ite 1000]loss=0.001278\n",
      "[ite 1100]loss=0.000440\n",
      "[ite 1200]loss=0.000491\n",
      "[ite 1300]loss=0.000634\n",
      "[ite 1400]loss=0.000837\n",
      "[ite 1500]loss=0.001943\n",
      "[ite 1600]loss=0.001667\n",
      "[ite 1700]loss=0.001535\n",
      "[Epoch 17]18:23:46\n",
      "train_loss=0.000429, train_acc=0.993984, valid_loss=0.033640, valid_acc=0.994075, test_loss=0.024787, test_acc=0.994036\n",
      "[ite 100]loss=0.000983\n",
      "[ite 200]loss=0.001494\n",
      "[ite 300]loss=0.001941\n",
      "[ite 400]loss=0.001559\n",
      "[ite 500]loss=0.000258\n",
      "[ite 600]loss=0.000365\n",
      "[ite 700]loss=0.000524\n",
      "[ite 800]loss=0.000538\n",
      "[ite 900]loss=0.000100\n",
      "[ite 1000]loss=0.000431\n",
      "[ite 1100]loss=0.001416\n",
      "[ite 1200]loss=0.000787\n",
      "[ite 1300]loss=0.000680\n",
      "[ite 1400]loss=0.000534\n",
      "[ite 1500]loss=0.000166\n",
      "[ite 1600]loss=0.000960\n",
      "[ite 1700]loss=0.003312\n",
      "[Epoch 18]18:36:53\n",
      "train_loss=0.000385, train_acc=0.994113, valid_loss=0.029899, valid_acc=0.994203, test_loss=0.024146, test_acc=0.994165\n",
      "[ite 100]loss=0.000201\n",
      "[ite 200]loss=0.000173\n",
      "[ite 300]loss=0.000853\n",
      "[ite 400]loss=0.000203\n",
      "[ite 500]loss=0.000189\n",
      "[ite 600]loss=0.000967\n",
      "[ite 700]loss=0.001168\n",
      "[ite 800]loss=0.000104\n",
      "[ite 900]loss=0.000700\n",
      "[ite 1000]loss=0.000062\n",
      "[ite 1100]loss=0.000780\n",
      "[ite 1200]loss=0.001081\n",
      "[ite 1300]loss=0.001742\n",
      "[ite 1400]loss=0.000450\n",
      "[ite 1500]loss=0.001795\n",
      "[ite 1600]loss=0.000441\n",
      "[ite 1700]loss=0.000887\n",
      "[Epoch 19]18:49:59\n",
      "train_loss=0.000334, train_acc=0.994228, valid_loss=0.032224, valid_acc=0.994308, test_loss=0.025549, test_acc=0.994278\n",
      "[ite 100]loss=0.000731\n",
      "[ite 200]loss=0.000132\n",
      "[ite 300]loss=0.000673\n",
      "[ite 400]loss=0.001294\n",
      "[ite 500]loss=0.001229\n",
      "[ite 600]loss=0.001285\n",
      "[ite 700]loss=0.000420\n",
      "[ite 800]loss=0.000820\n",
      "[ite 900]loss=0.000965\n",
      "[ite 1000]loss=0.000092\n",
      "[ite 1100]loss=0.001254\n",
      "[ite 1200]loss=0.002245\n",
      "[ite 1300]loss=0.002605\n",
      "[ite 1400]loss=0.001092\n",
      "[ite 1500]loss=0.000086\n",
      "[ite 1600]loss=0.000899\n",
      "[ite 1700]loss=0.004552\n",
      "[Epoch 20]19:03:05\n",
      "train_loss=0.000320, train_acc=0.994334, valid_loss=0.033311, valid_acc=0.994405, test_loss=0.025021, test_acc=0.994377\n",
      "[ite 100]loss=0.000313\n",
      "[ite 200]loss=0.003533\n",
      "[ite 300]loss=0.000098\n",
      "[ite 400]loss=0.001769\n",
      "[ite 500]loss=0.000016\n",
      "[ite 600]loss=0.000597\n",
      "[ite 700]loss=0.001180\n",
      "[ite 800]loss=0.000787\n",
      "[ite 900]loss=0.001321\n",
      "[ite 1000]loss=0.000479\n",
      "[ite 1100]loss=0.000077\n",
      "[ite 1200]loss=0.000371\n",
      "[ite 1300]loss=0.002013\n",
      "[ite 1400]loss=0.000826\n",
      "[ite 1500]loss=0.000141\n",
      "[ite 1600]loss=0.000173\n",
      "[ite 1700]loss=0.000110\n",
      "[Epoch 21]19:16:11\n",
      "train_loss=0.000287, train_acc=0.994429, valid_loss=0.030298, valid_acc=0.994496, test_loss=0.024125, test_acc=0.994472\n",
      "[ite 100]loss=0.000980\n",
      "[ite 200]loss=0.000555\n",
      "[ite 300]loss=0.000074\n",
      "[ite 400]loss=0.000252\n",
      "[ite 500]loss=0.001356\n",
      "[ite 600]loss=0.002437\n",
      "[ite 700]loss=0.000987\n",
      "[ite 800]loss=0.000569\n",
      "[ite 900]loss=0.000121\n",
      "[ite 1000]loss=0.004403\n",
      "[ite 1100]loss=0.002087\n",
      "[ite 1200]loss=0.001407\n",
      "[ite 1300]loss=0.000274\n",
      "[ite 1400]loss=0.000265\n",
      "[ite 1500]loss=0.001043\n",
      "[ite 1600]loss=0.001025\n",
      "[ite 1700]loss=0.000147\n",
      "[Epoch 22]19:29:17\n",
      "train_loss=0.000291, train_acc=0.994522, valid_loss=0.036379, valid_acc=0.994582, test_loss=0.025663, test_acc=0.994547\n",
      "[ite 100]loss=0.000251\n",
      "[ite 200]loss=0.000426\n",
      "[ite 300]loss=0.000387\n",
      "[ite 400]loss=0.001094\n",
      "[ite 500]loss=0.001691\n",
      "[ite 600]loss=0.000283\n",
      "[ite 700]loss=0.000135\n",
      "[ite 800]loss=0.000492\n",
      "[ite 900]loss=0.000182\n",
      "[ite 1000]loss=0.002120\n",
      "[ite 1100]loss=0.000125\n",
      "[ite 1200]loss=0.000740\n",
      "[ite 1300]loss=0.000119\n",
      "[ite 1400]loss=0.000421\n",
      "[ite 1500]loss=0.000181\n",
      "[ite 1600]loss=0.003200\n",
      "[ite 1700]loss=0.000221\n",
      "[Epoch 23]19:42:24\n",
      "train_loss=0.000244, train_acc=0.994586, valid_loss=0.029980, valid_acc=0.994647, test_loss=0.024264, test_acc=0.994618\n",
      "[ite 100]loss=0.000571\n",
      "[ite 200]loss=0.000320\n",
      "[ite 300]loss=0.002074\n",
      "[ite 400]loss=0.000417\n",
      "[ite 500]loss=0.002098\n",
      "[ite 600]loss=0.000223\n",
      "[ite 700]loss=0.001142\n",
      "[ite 800]loss=0.000466\n",
      "[ite 900]loss=0.000052\n",
      "[ite 1000]loss=0.000662\n",
      "[ite 1100]loss=0.000371\n",
      "[ite 1200]loss=0.000219\n",
      "[ite 1300]loss=0.001165\n",
      "[ite 1400]loss=0.000224\n",
      "[ite 1500]loss=0.001786\n",
      "[ite 1600]loss=0.000082\n",
      "[ite 1700]loss=0.000464\n",
      "[Epoch 24]19:55:30\n",
      "train_loss=0.000241, train_acc=0.994660, valid_loss=0.033194, valid_acc=0.994712, test_loss=0.023525, test_acc=0.994683\n",
      "[ite 100]loss=0.002161\n",
      "[ite 200]loss=0.000500\n",
      "[ite 300]loss=0.000523\n",
      "[ite 400]loss=0.000264\n",
      "[ite 500]loss=0.000403\n",
      "[ite 600]loss=0.000916\n",
      "[ite 700]loss=0.000912\n",
      "[ite 800]loss=0.000643\n",
      "[ite 900]loss=0.003042\n",
      "[ite 1000]loss=0.000535\n",
      "[ite 1100]loss=0.000243\n",
      "[ite 1200]loss=0.000041\n",
      "[ite 1300]loss=0.000708\n",
      "[ite 1400]loss=0.000016\n",
      "[ite 1500]loss=0.000557\n",
      "[ite 1600]loss=0.000030\n",
      "[ite 1700]loss=0.000580\n",
      "[Epoch 25]20:08:36\n",
      "train_loss=0.000218, train_acc=0.994725, valid_loss=0.031342, valid_acc=0.994781, test_loss=0.026098, test_acc=0.994752\n",
      "[ite 100]loss=0.000539\n",
      "[ite 200]loss=0.000398\n",
      "[ite 300]loss=0.001224\n",
      "[ite 400]loss=0.000580\n",
      "[ite 500]loss=0.000339\n",
      "[ite 600]loss=0.004166\n",
      "[ite 700]loss=0.000194\n",
      "[ite 800]loss=0.000180\n",
      "[ite 900]loss=0.000561\n",
      "[ite 1000]loss=0.001038\n",
      "[ite 1100]loss=0.000097\n",
      "[ite 1200]loss=0.000170\n",
      "[ite 1300]loss=0.000495\n",
      "[ite 1400]loss=0.000296\n",
      "[ite 1500]loss=0.000077\n",
      "[ite 1600]loss=0.000106\n",
      "[ite 1700]loss=0.001428\n",
      "[Epoch 26]20:21:42\n",
      "train_loss=0.000219, train_acc=0.994785, valid_loss=0.034767, valid_acc=0.994828, test_loss=0.024464, test_acc=0.994807\n",
      "[ite 100]loss=0.000304\n",
      "[ite 200]loss=0.000149\n",
      "[ite 300]loss=0.000073\n",
      "[ite 400]loss=0.000035\n",
      "[ite 500]loss=0.000189\n"
     ]
    }
   ],
   "source": [
    "trainloss_list = []\n",
    "validloss_list = []\n",
    "testloss_list = []\n",
    "trainacc_list = []\n",
    "validacc_list = []\n",
    "testacc_list = []\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    print('Start:'+time.strftime(\"%H:%M:%S\", time.localtime()))\n",
    "    tf.global_variables_initializer().run()\n",
    "    tf.local_variables_initializer().run()\n",
    "    for epoch in range(n_epoch):\n",
    "        train()\n",
    "        trl,val,tel,tra,vaa,tea = evaluation()\n",
    "        trainloss_list.append(trl)\n",
    "        validloss_list.append(val)\n",
    "        testloss_list.append(tel)\n",
    "        trainacc_list.append(tra)\n",
    "        validacc_list.append(vaa)\n",
    "        testacc_list.append(tea)\n",
    "        print('[Epoch %d]'%(epoch+1)+time.strftime(\"%H:%M:%S\", time.localtime()))\n",
    "        print('train_loss=%f, train_acc=%f, valid_loss=%f, valid_acc=%f, test_loss=%f, test_acc=%f'%(trl,tra,val,vaa,tel,tea))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}