04_CNN_advances/cnn_mnist_modern.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 卷积神经示例: TF-SLIM高级API实现\n",
    "### 推荐尽量用TF-SLIM实现复杂结构"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "当前TensorFlow版本为 [1.3.0]\n",
      "所有包载入完毕\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "from scipy import ndimage\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import tensorflow.contrib.slim as slim\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "%matplotlib inline  \n",
    "print (\"当前TensorFlow版本为 [%s]\" % (tf.__version__))\n",
    "print (\"所有包载入完毕\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 载入 MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/train-images-idx3-ubyte.gz\n",
      "Extracting data/train-labels-idx1-ubyte.gz\n",
      "Extracting data/t10k-images-idx3-ubyte.gz\n",
      "Extracting data/t10k-labels-idx1-ubyte.gz\n",
      "MNIST ready\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets('data/', one_hot=True)\n",
    "trainimg   = mnist.train.images\n",
    "trainlabel = mnist.train.labels\n",
    "valimg     = mnist.validation.images\n",
    "vallabel   = mnist.validation.labels\n",
    "testimg    = mnist.test.images\n",
    "testlabel  = mnist.test.labels\n",
    "print (\"MNIST ready\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NETWORK READY\n"
     ]
    }
   ],
   "source": [
    "n_input = 784\n",
    "n_classes = 10\n",
    "x = tf.placeholder(\"float\", [None, n_input])\n",
    "y = tf.placeholder(\"float\", [None, n_classes])\n",
    "is_training = tf.placeholder(tf.bool)\n",
    "\n",
    "def lrelu(x, leak=0.2, name='lrelu'):\n",
    "    with tf.variable_scope(name):\n",
    "        f1 = 0.5 * (1 + leak)\n",
    "        f2 = 0.5 * (1 - leak)\n",
    "        return f1 * x + f2 * abs(x)\n",
    "\n",
    "def CNN(inputs, is_training=True):\n",
    "    x   = tf.reshape(inputs, [-1, 28, 28, 1])\n",
    "    batch_norm_params = {'is_training': is_training, 'decay': 0.9\n",
    "                         , 'updates_collections': None}\n",
    "    init_func = tf.truncated_normal_initializer(stddev=0.01)\n",
    "    net = slim.conv2d(x, 32, [5, 5], padding='SAME'\n",
    "                     , activation_fn       = lrelu\n",
    "                     , weights_initializer = init_func\n",
    "                     , normalizer_fn       = slim.batch_norm\n",
    "                     , normalizer_params   = batch_norm_params\n",
    "                     , scope='conv1')\n",
    "    net = slim.max_pool2d(net, [2, 2], scope='pool1')\n",
    "    net = slim.conv2d(x, 64, [5, 5], padding='SAME'\n",
    "                     , activation_fn       = lrelu\n",
    "                     , weights_initializer = init_func\n",
    "                     , normalizer_fn       = slim.batch_norm\n",
    "                     , normalizer_params   = batch_norm_params\n",
    "                     , scope='conv2')\n",
    "    net = slim.max_pool2d(net, [2, 2], scope='pool2')\n",
    "    net = slim.flatten(net, scope='flatten3')\n",
    "    net = slim.fully_connected(net, 1024\n",
    "                    , activation_fn       = lrelu\n",
    "                    , weights_initializer = init_func\n",
    "                    , normalizer_fn       = slim.batch_norm\n",
    "                    , normalizer_params   = batch_norm_params\n",
    "                    , scope='fc4')\n",
    "    net = slim.dropout(net, keep_prob=0.7, is_training=is_training, scope='dr')  \n",
    "    out = slim.fully_connected(net, n_classes\n",
    "                               , activation_fn=None, normalizer_fn=None, scope='fco')\n",
    "    return out\n",
    "print (\"NETWORK READY\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义图结构"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FUNCTIONS READY\n"
     ]
    }
   ],
   "source": [
    "# PREDICTION\n",
    "pred = CNN(x, is_training)\n",
    "\n",
    "# LOSS AND OPTIMIZER\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(\n",
    "        labels=y, logits=pred))\n",
    "optm = tf.train.AdamOptimizer(learning_rate=0.001).minimize(cost) \n",
    "corr = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))    \n",
    "accr = tf.reduce_mean(tf.cast(corr, \"float\"))\n",
    "\n",
    "# INITIALIZER\n",
    "init = tf.global_variables_initializer()\n",
    "sess = tf.Session()\n",
    "sess.run(init)\n",
    "print (\"FUNCTIONS READY\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 检查变量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=================== TRAINABLE VARIABLES ===================\n",
      "[0/8] [conv1/weights:0] / SAHPE IS (5, 5, 1, 32)\n",
      "[1/8] [conv1/BatchNorm/beta:0] / SAHPE IS (32,)\n",
      "[2/8] [conv2/weights:0] / SAHPE IS (5, 5, 1, 64)\n",
      "[3/8] [conv2/BatchNorm/beta:0] / SAHPE IS (64,)\n",
      "[4/8] [fc4/weights:0] / SAHPE IS (12544, 1024)\n",
      "[5/8] [fc4/BatchNorm/beta:0] / SAHPE IS (1024,)\n",
      "[6/8] [fco/weights:0] / SAHPE IS (1024, 10)\n",
      "[7/8] [fco/biases:0] / SAHPE IS (10,)\n"
     ]
    }
   ],
   "source": [
    "print (\"=================== TRAINABLE VARIABLES ===================\")\n",
    "t_weights = tf.trainable_variables()\n",
    "var_names_list = [v.name for v in tf.trainable_variables()] \n",
    "for i in range(len(t_weights)):\n",
    "    wval = sess.run(t_weights[i])\n",
    "    print (\"[%d/%d] [%s] / SAHPE IS %s\"  \n",
    "           % (i, len(t_weights), var_names_list[i], wval.shape,))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 存储"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SAVER READY\n"
     ]
    }
   ],
   "source": [
    "savedir = \"nets/cnn_mnist_modern/\"\n",
    "saver = tf.train.Saver(max_to_keep=100)\n",
    "save_step = 4\n",
    "if not os.path.exists(savedir):\n",
    "    os.makedirs(savedir)\n",
    "print (\"SAVER READY\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据增加"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def augment_img(xs):\n",
    "    out  = np.copy(xs)\n",
    "    xs_r = np.reshape(xs, [-1, 28, 28])\n",
    "    for i in range(xs_r.shape[0]):\n",
    "        xs_img = xs_r[i, :, :]\n",
    "        bg_value = 0\n",
    "        # ROTATE\n",
    "        angle = np.random.randint(-15, 15, 1).astype(float)\n",
    "        xs_img = ndimage.rotate(xs_img, angle, reshape=False, cval=bg_value)\n",
    "        # ZOOM\n",
    "        rg = 0.1\n",
    "        zoom_factor = np.random.uniform(1., 1.+rg)\n",
    "        h, w = xs_img.shape[:2]\n",
    "        zh   = int(np.round(zoom_factor * h))\n",
    "        zw   = int(np.round(zoom_factor * w))\n",
    "        top  = (zh - h) // 2\n",
    "        left = (zw - w) // 2\n",
    "        zoom_tuple = (zoom_factor,) * 2 + (1,) * (xs_img.ndim - 2)\n",
    "        temp = ndimage.zoom(xs_img[top:top+zh, left:left+zw], zoom_tuple)\n",
    "        trim_top  = ((temp.shape[0] - h) // 2)\n",
    "        trim_left = ((temp.shape[1] - w) // 2)\n",
    "        xs_img = temp[trim_top:trim_top+h, trim_left:trim_left+w]\n",
    "        # SHIFT\n",
    "        shift = np.random.randint(-3, 3, 2)\n",
    "        xs_img = ndimage.shift(xs_img, shift, cval=bg_value)\n",
    "        # RESHAPE\n",
    "        xs_v = np.reshape(xs_img, [1, -1])\n",
    "        out[i, :] = xs_v\n",
    "    return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试增加"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/xrong/.pyenv/versions/anaconda2-4.4.0/lib/python2.7/site-packages/scipy/ndimage/interpolation.py:600: UserWarning: From scipy 0.13.0, the output shape of zoom() is calculated with round() instead of int() - for these inputs the size of the returned array has changed.\n",
      "  \"the returned array has changed.\", UserWarning)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAEMCAYAAAAiW8hnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEI5JREFUeJzt3X+s3XV9x/HnWyixE7eWdXTlh+2qiIh/tKFxzJGFaTWs\n/6AbK5Qg1RhK0CGahsnYCHVhiZMfduGPLkUa2YZoN1BKwpwt6VZJJ5GSpi0/HGZrHdgflh/tbSBM\nynt/nG8/Hi/3fs+995x7vueW5yM5uaff9/fHu1/K635/fM73RGYiSQBva7oBSYPDQJBUGAiSCgNB\nUmEgSCoMBEmFgSCpMBCmuIj4VETsjIhXImJfRKyJiBlVbVVE/CIijkTEyxGxNSJ+r23ZCyPiuWHr\n+2hEbI6IoYh4ISK2R8SXIuLtbev8p7b5s9r+29qm3RIR3xi23pOrPv51hL/D7ohY3LOdogkzEKaw\niFgJ/C1wPfAbwPnAXGBjRJxUzfbtzDwZmAVsBv65Zn1/CvwL8E1gbmb+JnApcAZwZk0rpwGXdWj3\nT4DXgI9GxG93mFcNMRCmqIj4deDLwLWZ+b3M/EVm7gaWAvOAK9rnz8zXgXuB0yPit0ZYXwB3AH+d\nmXdl5ovVcj/OzGsz89madr4KfDkiTqyZZznw98CO4b1pcBgIU9eHgLcDD7RPzMwjwMPAR9unV0cM\nVwIvAC+NsL6zaR0J3D+BXh4ADgOfGqkYEXOBC2kF0r1VHxpABsLUNQs4WP3mH25vVQdYGhEvA68C\nVwGXjLLMsfn3HZsQEd+qrj28EhGfrOklgZuAm9pOVdp9EtiRmU8B3wLOjYiFdX85NcNAmLoOArNG\nOUyfU9UB1mfmDGA2sAs4b5T1vdC2LACZeVm17BPACXXNZObDwHPA1SOUr6R1ZEBmPg/8B61TCA0Y\nA2Hq+k9aF+n+uH1iRJwM/BHwSPv0zDwIrABWRcQc3uzHwPPD1zdOfwncCPxaWz8fAs4C/qK6C7IP\n+F3g8g7XHNQAA2GKysxDtC4q3hkRF0XEtIiYB6yn9Zv6H0dY5sfAvwF/PkLtDWAlcHNEXBURM6Pl\nLFpHF2Pp6d9pHYW0//ZfDmwE3g8sqF4fAKbTCq5jpkXE29tehkUDDIQpLDO/Sus38m20Luo9Bvwv\n8JHMfG2UxW4FVkTEqSOs79u07lJcUa3nIK2AWUvN7cph/go4BaAau7AUuDMz97W9/odWYLUHx8O0\nrnMce60a4/bUS5nZ9xdwEa1D1J8ANzTRQ4f+dgM7ge3A4wPQzzrgALCrbdoptH7zPlv9nDlg/a2i\ndQqyvXotabC/M2mNwXgKeBK4bpD2YU1/fd+HUW24byLiBOC/aN0Wew74EbAsW1egB0JE7AYWZeu8\nu3ER8QfAEeAfMvMD1bSvAi9m5lci4gZa/5i/NED9rQKOZOZtTfTUrrpmMiczn4iIdwLbgI/Tuk3a\n+D6s6W8pfd6HTZwyfBD4SWb+d2b+H63bUBc30MeUkZlbgBeHTb4YuKd6fw+tf0CNGKW/gZGZezPz\nier9EPA0cDoDsg9r+uu7JgLhdFrnp8c8R0N/+RoJbIqIbRGxoulmRjE7M/dW7/cxxgt/fXZtROyI\niHURMbPpZgCqC68LaV1vGbh9OKw/6PM+9KLiyC7IzAW0roJ/rjokHljZOu8btKflrgHm07qrsBe4\nvdl2yi3Z+4EvZObh9tog7MMR+uv7PmwiEJ7nVz8oc0Y1bWBka/AMmXkA+A6t05xBs//YeILq54GG\n+/kVmbk/M49m63bmXTS8DyNiGq3/2e7NzGPDvQdmH47UXxP7sIlA+BFwVkT8TjXM9TJgQwN9jCgi\n3lFd2CEi3gF8jNa99UGzgV/etlsOPNhgL28ybPDTJ2hwH1Yf3LobeDoz72grDcQ+HK2/JvZh3+8y\nAETEEmA1reGw6zLzb/rexCgiYj6towKAE4FvNt1fRNxH68NBs4D9wM3Ad2mNEXgXsAdYmtUnFAek\nvwtpHeomrdu4V7edr/e7vwuAH9C6lfxGNflGWufpje/Dmv6W0ed92EggSBpMXlSUVBgIkgoDQVJh\nIEgqDARJRaOBMMDDggH769Yg9zfIvUFz/TV9hDDQ/1Gwv24Ncn+D3Bs01F/TgSBpgHQ1MCkiLgL+\njtaIw69n5lc6zO8oKKkhmRmd5plwIEzkQScGgtScsQRCN6cMPuhEOs50EwhT4UEnksZh0h91Xd0+\nGfQrupLoLhDG9KCTzFxL6zHeXkOQBlw3pwwD/aATSeM34SOEzHw9Iv6M1jcBHXvQyZM960xS3/X1\nASmeMkjNmezbjpKOMwaCpMJAkFQYCJIKA0FSYSBIKgwESYWBIKkwECQVBoKkwkCQVBgIkgoDQVJh\nIEgqDARJhYEgqTAQJBUGgqTCQJBUGAiSCgNBUmEgSCoMBEmFgSCpMBAkFQaCpMJAkFQYCJIKA0FS\nYSBIKgwEScWJ3SwcEbuBIeAo8HpmLupFU5Ka0VUgVP4wMw/2YD2SGuYpg6Si20BIYFNEbIuIFb1o\nSFJzuj1luCAzn4+IU4GNEfFMZm5pn6EKCsNCmgIiM3uzoohVwJHMvK1mnt5sTNK4ZWZ0mmfCpwwR\n8Y6IeOex98DHgF0TXZ+k5nVzyjAb+E5EHFvPNzPzez3pSlIjenbKMKaNecogNWZSTxkkHX8MBEmF\ngSCpMBAkFQaCpMJAkFT04tOOGhCf/vSna+udbjG/8MILtfVzzjmntr5169ba+qOPPlpbV/M8QpBU\nGAiSCgNBUmEgSCoMBEmFgSCpMBAkFcfVOIRly5bV1hcuXFhb73Qff9DNmDGjq+WPHj1aWz/ppJNq\n66+++mpt/ZVXXqmt79y5s7Z+6aWX1tZ//vOf19bVmUcIkgoDQVJhIEgqDARJhYEgqTAQJBUGgqRi\nSj2G/bbbRv1SKACuu+662voJJ5zQzebVsM2bN9fWL7/88tr6/v37e9nOlONj2CWNi4EgqTAQJBUG\ngqTCQJBUGAiSCgNBUjGlxiH89Kc/ra2fccYZtfUdO3bU1jt9nn+ydfreggcffLBPnUzM4sWLa+tX\nXnllbX3evHldbb/TOIXLLrustn68P0+hJ+MQImJdRByIiF1t006JiI0R8Wz1c2a3zUpq3lhOGb4B\nXDRs2g3AI5l5FvBI9WdJU1zHQMjMLcCLwyZfDNxTvb8H+HiP+5LUgIleVJydmXur9/uA2T3qR1KD\nun7IamZm3cXCiFgBrOh2O5Im30SPEPZHxByA6ueB0WbMzLWZuSgzF01wW5L6ZKKBsAFYXr1fDgz2\n/TBJY9JxHEJE3AdcCMwC9gM3A98F1gPvAvYASzNz+IXHkdbV1TiE9773vbX1c889t7a+adOm2vrQ\n0NC4e9LYzZ8/v7b+0EMP1dbPOeecrrZ//fXX19Zvv/32rtY/6MYyDqHjNYTMHO3bTz4y7o4kDTSH\nLksqDARJhYEgqTAQJBUGgqTCQJBUTKnnIej4dskll9TW169f39X6Dx48WFs/9dRTu1r/oPN7GSSN\ni4EgqTAQJBUGgqTCQJBUGAiSCgNBUmEgSCoMBEmFgSCpMBAkFQaCpMJAkFQYCJIKA0FS0fVXuUlj\ndc0119TWFy2a3C/3mj59em39vPPOq61v27atl+0MJI8QJBUGgqTCQJBUGAiSCgNBUmEgSCoMBEmF\n38twHJkzZ05t/Yorrqitf/7zn+9lO29y2mmn1dYjOn5twKQ6fPhwbX3GjBl96mRy9OR7GSJiXUQc\niIhdbdNWRcTzEbG9ei3ptllJzRvLKcM3gItGmP61zFxQvR7ubVuSmtAxEDJzC/BiH3qR1LBuLipe\nGxE7qlOKmT3rSFJjJhoIa4D5wAJgL3D7aDNGxIqIeDwiHp/gtiT1yYQCITP3Z+bRzHwDuAv4YM28\nazNzUWZO7kfZJHVtQoEQEe33tz4B7BptXklTR8fnIUTEfcCFwKyIeA64GbgwIhYACewGrp7EHt8y\nFi9eXFvv9Hn9q666qrY+f/78cff0VrJu3bqmW2hcx0DIzGUjTL57EnqR1DCHLksqDARJhYEgqTAQ\nJBUGgqTCQJBU+L0MPfSe97yntr5mzZra+oc//OHa+mQ/L2DPnj219Zdeeqmr9d9000219ddee622\nfuedd9bWzz777HH31G7v3r1dLX888AhBUmEgSCoMBEmFgSCpMBAkFQaCpMJAkFQ4DmEcvvjFL9bW\nP/vZz9bW3/3ud9fWjxw5Uls/dOhQbX316tW19Z/97Ge19a1bt9bWO41TmGyd/v6dDA0N1dYfeuih\nrtZ/PPAIQVJhIEgqDARJhYEgqTAQJBUGgqTCQJBUOA5hHM4///zaeqdxBhs2bKit33HHHbX1LVu2\n1NanugULFtTW586d29X6Oz1v4Zlnnulq/ccDjxAkFQaCpMJAkFQYCJIKA0FSYSBIKgwESYXjEMbh\nmmuuqa3v3Lmztn7LLbf0sp3jTqfvtZg9e3ZX69+0aVNXy78VdDxCiIgzI2JzRDwVEU9GxHXV9FMi\nYmNEPFv9nDn57UqaTGM5ZXgdWJmZ7wfOBz4XEe8HbgAeycyzgEeqP0uawjoGQmbuzcwnqvdDwNPA\n6cDFwD3VbPcAH5+sJiX1x7guKkbEPGAh8BgwOzOPfRnePqC7EzxJjRvzRcWIOBm4H/hCZh5u/+LR\nzMyIyFGWWwGs6LZRSZNvTEcIETGNVhjcm5kPVJP3R8Scqj4HODDSspm5NjMXZeaiXjQsafKM5S5D\nAHcDT2dm++dzNwDLq/fLgQd7356kforMEY/0fzlDxAXAD4CdwBvV5BtpXUdYD7wL2AMszcwXO6yr\nfmN6S7v11ltr6ytXrqytv/zyy7X1JUuW1NZ/+MMf1tanusyMTvN0vIaQmY8Co63oI+NtStLgcuiy\npMJAkFQYCJIKA0FSYSBIKgwESYXPQ1Df7Nixo7b+vve9r6v1f//736+tH+/jDHrBIwRJhYEgqTAQ\nJBUGgqTCQJBUGAiSCgNBUuE4BPXNvHnzausnnlj/z/HQoUO19dWrV4+3JQ3jEYKkwkCQVBgIkgoD\nQVJhIEgqDARJhYEgqXAcgnpm2bJltfXp06fX1oeGhmrrV199dW3d5x10zyMESYWBIKkwECQVBoKk\nwkCQVBgIkgoDQVIRmdm/jUX0b2PquWnTptXWH3vssdp6p+9duO+++2rrn/nMZ2rrqpeZ0WmejkcI\nEXFmRGyOiKci4smIuK6avioino+I7dVrSS+altScsYxUfB1YmZlPRMQ7gW0RsbGqfS0zb5u89iT1\nU8dAyMy9wN7q/VBEPA2cPtmNSeq/cV1UjIh5wELg2MnitRGxIyLWRcTMUZZZERGPR8TjXXUqadKN\nORAi4mTgfuALmXkYWAPMBxbQOoK4faTlMnNtZi7KzEU96FfSJBpTIETENFphcG9mPgCQmfsz82hm\nvgHcBXxw8tqU1A9jucsQwN3A05l5R9v0OW2zfQLY1fv2JPXTWO4y/D7wSWBnRGyvpt0ILIuIBUAC\nu4H6D6tryus0ZqXTOILt27fX1jdu3Fhb1+Qby12GR4GRBjQ83Pt2JDXJocuSCgNBUmEgSCoMBEmF\ngSCpMBAkFT4PQXqL6MnzECS9dRgIkgoDQVJhIEgqDARJhYEgqTAQJBVjeR5CLx0E9rT9eVY1bVDZ\nX3cGub9B7g1639/csczU14FJb9p4xOOD/KxF++vOIPc3yL1Bc/15yiCpMBAkFU0HwtqGt9+J/XVn\nkPsb5N6gof4avYYgabA0fYQgaYAYCJIKA0FSYSBIKgwEScX/A6+jZyVrn+WmAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd7ec11e9d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAEMCAYAAAAiW8hnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFOlJREFUeJzt3XuQVvV9x/H3Vy4CcrUYQKIiienoxEgUHay00nEaLBnH\nyxSMVYM0LUxMTDITmzrGTCRtOtZ4SacdmWAlYjQ2MrnoRKNjEDVGJQhFAQ25ICqESxC5owj77R/n\n7M8n6+7v9+yeZ59zlv28ZnZ293yf5zzf58B+9lx++zvm7oiIABxRdgMiUh0KBBEJFAgiEigQRCRQ\nIIhIoEAQkUCBICKBAqEEZran5qPFzPbXfH+5md1oZu/m3+8ws2fN7Ox21nNi/vx57dTczFaZ2RE1\ny/7NzO6u+f4zZvZrM9ttZlvM7BEzG5LX7jazA216vbTmuVfl699nZpvNbJ6ZDa+pR9+DmU3Je/xx\nm75Py5c/2ea97G3Ty1favM7u/OM3ZvbfZjam8/8yokAogbsPbv0AXgcuqFl2X/6wH+T1kcASYFE7\nq/o08BZwqZkd2U79WOBT7fVgZucC/w5c5u5DgJOBH7R52M21vbr7D/Lnfhn4D+CfgWHAJOAE4HEz\n61/z/NR7+CNwtpn9Wc2ymcBv2mn5tDa93NzmdYYARwMXA6OB5QqFzlMgVJy7HwTuA8aa2TGty83M\nyALhBuBd4IJ2nn4zMNfM+rZTOxN4zt3/L3+d7e6+0N13x/oxs6HAXOAad3/U3d919/XADGAccEW9\n7wE4APyEPLTMrA9waf7YTst7WZOv44/Al7uynt5MgVBx+W/cTwNvku0NtJoMfBD4X+ABst+sbf0I\n2AVc1U5tKTDVzOaa2Tkd7GG05y+AAfm6A3ffAzwC/E0n3gPAPXkNYCqwGvhDnb20y90PAQ8Cf1lk\nPb2RAqG6ZpjZDmA/8E/A3+W/aVvNBH7m7m8B3wfON7MPtFmHA18DvtZmVx53/wVwCXA68DDwppnd\nlv+WbnVtfvy/w8y25ctGAtva9NJqU16v9z3g7s8CR5vZn5MFwz0dbI8VNb3sMLOpHTyu1R/IDiGk\nExQI1fWAuw8HRpH91jyjtWBmA4Hp5LvW7v4c2bmIv2+7End/BNgAzGmn9jN3v4DsB+dCsj2Jf6x5\nyC3uPjz/aP1B3waM7OAwZExeT76HNr4HfB74a+DHHTzm9Jpehrv7Yx08rtVYYHviMdKGAqHi3H0b\nMBu4seYk2cXAUOCO/Az/ZrIfgPYOGwC+ClwPDOrgNVrcfTHwBPDRREvPAe+Q7V0EZjYY+FtgcZ3v\nodb3gKuBR9x9X+L1k/IrKxcAvyi6rt5GgdADuPta4DHgK/mimcAC4FRgQv5xDnCamZ3azvOfJPsN\nHQLDzC40s0+Z2QjLnAWcCzyf6GUn2UnF/zKz882sn5mNIzuPsYHsh7ue91BbezV/7a/GXjvFzPqa\n2cnA/WRXGm4rsr7eSIHQc3wLmG1mJwDnAd929801H8uBR+l4L+EG/vSY+i2y4/rfkp14vBf4Vs1l\nzw7ll/yuB27Jn7sUeAM4z93fqeM9tD3Xgbs/4+6xk4kvthmH8O2a2qVmtgfYCTxEdvLyjMT6pD3u\n3vQP4HxgLfA74Loyekj0tx5YBawEXqhAPwuArcDqmmVHA4+T/UA/DoyoWH83AhvzbbgSmFZif8eR\njYN4GVgDfLFK2zDSX9O3oeUv3DT5WezfkF2e2gAsIxsc83JTG4kws/XARM+OfUtnZn8F7AHucfeP\n5stuBra7+01mdh3Zf+Z/qVB/NwJ73P2WMnqqlZ+3GOPuK/KRmMuBi8hOopa+DSP9zaDJ27CMQ4az\ngN+5+zp3P0B2Hf3CEvroMdz9ad5/xvxCYGH+9UKy/0Cl6KC/ynD3Te6+Iv96N/AK2UnYSmzDSH9N\nV0YgjCU73my1gZLefIQDPzez5WY2u+xmOjDK3TflX28mu7RXNdeY2UtmtsDMRpTdDEB+AvTjZOc9\nKrcN2/QHTd6GOqnYvsnuPoHsMtrn8l3iyvLsuK9qs+XOA8aTXQHZBNxabjvh0ugPgS+5+67aWhW2\nYTv9NX0blhEIG8lOorT6YL6sMtx9Y/55K9lAmbPK7ahdW1qv6eeft5bcz59w9y3ufsjdW4A7KXkb\nmlk/sh+2+9y9ddh1ZbZhe/2VsQ3LCIRlwEmW/eluf7I/bHmohD7aZWZH2Xt/AnwU8Amya/hV8xDv\nXWKcSTZ2vzLaDEC6mBK3Yf6HYHcBr7h77diESmzDjvorYxs2/SoDgJlNA74N9AEWuPs3m95EB8xs\nPO8Nn+0LfL/s/szsfmAK2d8JbAG+TvZXgg8AxwOvATPcvZQTex30N4VsV9fJLuPOqTleb3Z/k8lG\nLa4CWvLF15Mdp5e+DSP9XUaTt2EpgSAi1aSTiiISKBBEJFAgiEigQBCRQIEgIkGpgVDhYcGA+iuq\nyv1VuTcor7+y9xAq/Y+C+iuqyv1VuTcoqb+yA0FEKqTQwCQzOx/4T7IRh//j7jclHq9RUCIlcXdL\nPabLgdCViU4UCOU64oj4DmF3j1pNrT8b0t996+/t6gmEIocMmuhE5DBTJBB6wkQnItIJ7d1so6Hy\nyydVP6MrIhQLhLomOnH3+cB80DkEkaorcshQ6YlORKTzuryH4O4HzezzZHfjaZ3oZE3DOusGqbPs\nffr0idZTZ8GL1lNSz3/nndg9UqClpSVaL1t3XyVIbT9dpWjyBCllHzIoEKodCN2ttwdCd192FJHD\njAJBRAIFgogECgQRCRQIIhIoEEQk6FWXHXu7oUOHRuupy5J9+8aHrQwbNixa3717d7S+fXu5N5A+\n3C9L6rKjiHSKAkFEAgWCiAQKBBEJFAgiEigQRCRQIIhI0O1TqMl7UtfxU/VDhw5F66eddlq0PnXq\n1Gj97bffjtYHDhwYrZ944onR+ubNm6P1tWvXRuv79++P1l9+ucMJvwFYsyY+XUdPH2fQCNpDEJFA\ngSAigQJBRAIFgogECgQRCRQIIhIoEEQkOKzGIRx77LHR+oc//OFo/YwzzojWjzvuuGh9wIAB0frI\nkSOj9dR8Aqn5DMaOjd9aM1VPTVNfdYsWLYrWv/GNb0Trq1evbmQ7PVLP/h8gIg2lQBCRQIEgIoEC\nQUQCBYKIBAoEEQkUCCIS9Kj7Mpx++unR+qxZs6L1yZMnR+sTJkzodE+9ycGDB6P11HwFqfkc+vfv\nH60PGjQoWk9ZsmRJtD537txo/amnnir0+mXf96Ge+zIUGphkZuuB3cAh4KC7TyyyPhEpVyNGKv61\nu29rwHpEpGQ6hyAiQdFAcODnZrbczGY3oiERKU/RQ4bJ7r7RzD4APG5mv3b3p2sfkAeFwkKkByi0\nh+DuG/PPW4EfA2e185j57j5RJxxFqq/LgWBmR5nZkNavgU8A+vtRkR6sy+MQzGw82V4BZIce33f3\nbyaeU+hCa+o68pQpU4qsnnfffTdaT11HL3od+c0334zWn3zyyWj92WefjdZT913Yt29ftL5r165o\nfe/evdH6uHHjovVp06ZF6+eee260Pnr06Gg9ZenSpdH6nDlzovUXX3wxWj+sxyG4+zogfmcQEelR\ndNlRRAIFgogECgQRCRQIIhIoEEQkUCCISNCj5kNYt25dtH7MMcdE648++mi0vmHDhmh99+7d0frO\nnTuj9T179kTrW7ZsidZT9w144403ovXUv3WqnhqHkZJa/8CBA6P16dOnR+s33XRTtJ4ap5B6f/fe\ne2+0/oUvfCFaT43j6NevX7SeGieTUs84BO0hiEigQBCRQIEgIoECQUQCBYKIBAoEEQkUCCIS9Khx\nCJMmTYrWx4wZE63/6le/itY3btzY6Z6keYYNGxatX3vttdH6DTfcUOj1U+NMzjnnnGh9zZo10fqA\nAQOi9dR8FikahyAinaJAEJFAgSAigQJBRAIFgogECgQRCRQIIhI04u7PTfP888+X3cJhLXXfgNTf\n6xd14MCBaH3//v3R+tatWxvZzvsMHTo0Wi86X0QVaA9BRAIFgogECgQRCRQIIhIoEEQkUCCISKBA\nEJGgR41D6NOnT7Seug6cen7fvvHN0dLSEq2nruOnnl/0Onbq9VPvPzXvf2qcQFGp7f/JT34yWv/Y\nxz7WyHbeJzV3yKxZs6L1W2+9NVrv7nEU9UjuIZjZAjPbamara5YdbWaPm9lv888jurdNEWmGeg4Z\n7gbOb7PsOmCxu58ELM6/F5EeLhkI7v40sL3N4guBhfnXC4GLGtyXiJSgqycVR7n7pvzrzcCoBvUj\nIiUqfFLR3T02eaqZzQZmF30dEel+Xd1D2GJmYwDyzx2eHnX3+e4+0d0ndvG1RKRJuhoIDwEz869n\nAg82ph0RKVPyvgxmdj8wBRgJbAG+DvwEeAA4HngNmOHubU88treubr0JROo6fDPvQdEVRcdZFDVo\n0KBo/eyzz47WTzrppGg9NQ7j5JNPjtanTp0arQ8fPjxaT923IyW1/VP1K664IlpftGhRtH7EEfHf\n36ntW899GZLnENz9sg5K56WeKyI9i4Yui0igQBCRQIEgIoECQUQCBYKIBAoEEQl61HwIKUXHGRQd\nx5AaR9Dd8yH0798/Wh8/fny0nppv4LOf/Wy0/qEPfShar7qi/77bt8eH4hw8eLDTPdVK/f9pBO0h\niEigQBCRQIEgIoECQUQCBYKIBAoEEQkUCCISHFbjEIoqOo6hu+crmDRpUrSeui/BlVdeGa1Pnjy5\n0z3VSt3XIfX3/Lt27YrWV6xYEa3v2LEjWj/zzDOj9dR8Cf369YvWV65cGa2//vrr0XoVaA9BRAIF\ngogECgQRCRQIIhIoEEQkUCCISKBAEJFA4xCaaOTIkdF6ar6Byy+/PFofO3ZstD548OBoPWXbtm3R\n+qpVq6L1vXv3Ruvr16+P1m+//fZoffPmzdH6vHnzovXp06dH66lxCG+99Va0vnPnzmg9pRn3HdEe\ngogECgQRCRQIIhIoEEQkUCCISKBAEJFAgSAiQa8ah1D0Ou6RRx4ZrZ966qnRemocwVVXXRWtDx8+\nPFpPeeyxx6L1JUuWROsvvfRStL5s2bJoPTWOobvt27cvWi96HT81jmLdunWF1t+3b/zHNTUfRT2S\newhmtsDMtprZ6pplN5rZRjNbmX9MK9yJiJSunkOGu4Hz21l+u7tPyD8eaWxbIlKGZCC4+9NA/B5V\nInJYKHJS8Rozeyk/pBjRsI5EpDRdDYR5wHhgArAJuLWjB5rZbDN7wcxe6OJriUiTdCkQ3H2Lux9y\n9xbgTuCsyGPnu/tEd5/Y1SZFpDm6FAhmVjtf9cXA6o4eKyI9R3IcgpndD0wBRprZBuDrwBQzmwA4\nsB6Y0409Nkzq79kPHDgQrafGAVx99dXR+qxZs6L11HXy7373u9H6gw8+GK2n7muwfXv83PHbb78d\nrRe9L0Xqvg0tLS3R+rhx46L1448/PlofNGhQtJ6yf//+aD3VfxUkA8HdL2tn8V3d0IuIlExDl0Uk\nUCCISKBAEJFAgSAigQJBRAIFgogEvWo+hKLXgVPz6q9duzZaT40DePjhh6P1O+64I1pP3ZegbH36\n9InWi45jOOqoo6L1IUOGFFp/ar6DNWvWFFp/StHtUw/tIYhIoEAQkUCBICKBAkFEAgWCiAQKBBEJ\nFAgiEvSqcQgHDx6M1lN/j5+aD+DOO++M1n/5y19G66+++mq0XnScQdH7UhR9fnfPBzB+/PhofcSI\nYlN/Ll++PFp/7rnnCq2/u8dp1EN7CCISKBBEJFAgiEigQBCRQIEgIoECQUQCBYKIBL1qHEJK6jp6\nSuq+Bs8880yh9acUHSeQknp+d19HnzlzZrQ+d+7caH3s2LHReuq+GKlxJBs2bIjWU1L/fs2gPQQR\nCRQIIhIoEEQkUCCISKBAEJFAgSAigQJBRAKNQ6hRdD6AVL3ofACp+Rq6e76BlO4eh3DKKadE6yec\ncEKh9S9btixaf+KJJwqtPyU1X0czJPcQzOw4M1tiZi+b2Roz+2K+/Ggze9zMfpt/Ljb7hIiUrp5D\nhoPAl939FGAS8DkzOwW4Dljs7icBi/PvRaQHSwaCu29y9xX517uBV4CxwIXAwvxhC4GLuqtJEWmO\nTp1UNLNxwMeBpcAod9+UlzYDoxramYg0Xd0nFc1sMPBD4Evuvqv2BJq7u5m1e0bOzGYDs4s2KiLd\nr649BDPrRxYG97n7j/LFW8xsTF4fA2xt77nuPt/dJ7r7xEY0LCLdp56rDAbcBbzi7rfVlB4CWv8e\ndSbwYOPbE5FmqueQ4RzgSmCVma3Ml10P3AQ8YGafAV4DZnRPi9WRGqfQ0+9rUFTRcQZDhgyJ1vv1\n61do/b///e+j9e985zvR+qpVqwq9ftXHkUAdgeDuzwAd/U8+r7HtiEiZNHRZRAIFgogECgQRCRQI\nIhIoEEQkUCCISKD5EJqo6H0RypYaR1F0HMLkyZOj9UsuuaTQ+hcvXhyt33PPPdF60XEmVRhnkKI9\nBBEJFAgiEigQRCRQIIhIoEAQkUCBICKBAkFEAo1DkMoYPXp0tJ6678P8+fOj9QULFkTrPX2cSCNo\nD0FEAgWCiAQKBBEJFAgiEigQRCRQIIhIoEAQkcCaee21o9u9iQB85CMfKVRfsmRJtL53795oveh9\nM6rO3eNvEO0hiEgNBYKIBAoEEQkUCCISKBBEJFAgiEigQBCRQOMQRHqJhoxDMLPjzGyJmb1sZmvM\n7Iv58hvNbKOZrcw/pjWiaREpT3IPwczGAGPcfYWZDQGWAxcBM4A97n5L3S+mPQSR0tSzh5CcQs3d\nNwGb8q93m9krwNji7YlI1XTqpKKZjQM+DizNF11jZi+Z2QIzG9HBc2ab2Qtm9kKhTkWk29V9UtHM\nBgNPAd909x+Z2ShgG+DAv5IdVvxDYh06ZBApST2HDHUFgpn1A34KPObut7VTHwf81N0/mliPAkGk\nJI26ymDAXcArtWGQn2xsdTGwuitNikh11HOVYTLwC2AV0HqD++uBy4AJZIcM64E5+QnI2Lq0hyBS\nkoYdMjSKAkGkPJogRUQ6RYEgIoECQUQCBYKIBAoEEQkUCCISKBBEJFAgiEigQBCRQIEgIoECQUQC\nBYKIBAoEEQkUCCISJCdZbbBtwGs134/Ml1WV+iumyv1VuTdofH8n1POgps6H8L4XN3vB3SeW1kCC\n+iumyv1VuTcorz8dMohIoEAQkaDsQJhf8uunqL9iqtxflXuDkvor9RyCiFRL2XsIIlIhCgQRCRQI\nIhIoEEQkUCCISPD/anXIZMuKWFoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd7ebde6f50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAEMCAYAAAAiW8hnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAERNJREFUeJzt3X+MZXV5x/H3o9DYLrbCzrpsEXZLQiBiUrCLtWYlECsC\n/7DbH1tBEdPKEmOJiGnll4KFRWuFAv4BrkKkLah02a3EQA0SutC0IbAL4ccOFGMXgSwsC7VCaKy4\nT/+4Z7+9rjPfMzP3zpw7y/uV3Myd85x77rOH4TPnfM93zo3MRJIA3tB1A5JGh4EgqTAQJBUGgqTC\nQJBUGAiSCgNBUmEgzHMR8dGIeCQiXo2I5yLi2oh4S1O7JCJ+FhGvRMSPI+LfIuL3+l57XEQ8s8f2\n3h8Rd0fEyxHxYkQ8FBGfiYg39W3zH/rWz+b939C37LKI+MYe292v6eOOCf4N2yLi94e2UzRjBsI8\nFhGfBv4a+AvgN4B3A0uBOyPiV5rVvp2Z+wFjwN3AP1a298fAeuBmYGlmLgT+BHgbcHClld8EPtjS\n7h8CPwXeHxEHtqyrjhgI81RE/DrweeDszPznzPxZZm4DVgPLgA/3r5+ZrwE3AQdFxKIJthfAlcBf\nZebXMvOl5nVPZObZmflkpZ0vAZ+PiH0q65wBXAc8vGdvGh0Gwvz1HuBNwIb+hZn5CnA78P7+5c0R\nw0eAF4H/mmB7h9M7Erh1Br1sAH4CfHSiYkQsBY6jF0g3NX1oBBkI89cYsLP5zb+n7U0dYHVE/Bj4\nH+BM4I8mec3u9Z/bvSAivtWMPbwaEadXekngs8Bn+05V+p0OPJyZW4FvAUdGxNG1f5y6YSDMXzuB\nsUkO05c0dYBbMvMtwGLgUeB3Jtnei32vBSAzP9i8dgvwxlozmXk78Axw1gTlj9A7MiAznwU20TuF\n0IgxEOavf6c3SPcH/QsjYj/gJOCu/uWZuRNYA1wSEUv4ZU8Az+65vWm6ELgA+LW+ft4DHAac31wF\neQ74XeC0ljEHdcBAmKcy87/pDSp+JSJOjIh9I2IZcAu939R/P8FrngC+B/zlBLVdwKeBiyPizIjY\nP3oOo3d0MZWe/oXeUUj/b/8zgDuBtwNHNY93AL9KL7h22zci3tT3MCw6YCDMY5n5JXq/kb9Mb1Dv\nPuBp4H2Z+dNJXvY3wJqIeOsE2/s2vasUH262s5NewKyjcrlyDxcBBwA0cxdWA1/JzOf6Hv9JL7D6\ng+N2euMcux+XTPH9NEyZOecP4ER6h6g/AM7rooeW/rYBjwAPAQ+MQD83ADuAR/uWHUDvN++Tzdf9\nR6y/S+idgjzUPE7usL+D6c3B2Ao8BnxylPZhpb8534fRvPGciYg3Av9B77LYM8D9wKnZG4EeCRGx\nDVievfPuzkXEscArwN9l5juaZV8CXsrML0bEefR+mD8zQv1dArySmV/uoqd+zZjJkszcEhFvBjYD\nK+ldJu18H1b6W80c78MuThneBfwgM3+Ymf9L7zLUKR30MW9k5j3AS3ssPgW4sXl+I70foE5M0t/I\nyMztmbmlef4yMA4cxIjsw0p/c66LQDiI3vnpbs/Q0T++IoHvR8TmiFjTdTOTWJyZ25vnzzHFgb85\ndnZEPBwRN0TE/l03A9AMvB5Nb7xl5PbhHv3BHO9DBxUntiIzj6I3Cv6J5pB4ZGXvvG/U7pZ7LXAo\nvasK24Erum2nXJK9FTgnM3/SXxuFfThBf3O+D7sIhGf5xT+UeVuzbGRkb/IMmbkD2EjvNGfUPL97\nPkHzdUfH/fyCzHw+M3+evcuZX6PjfRgR+9L7n+2mzNw93Xtk9uFE/XWxD7sIhPuBwyLit5pprh8E\nbuugjwlFxIJmYIeIWACcQO/a+qi5jf+/bHcG8J0Oe/kle0x+WkWH+7D5w63rgfHMvLKvNBL7cLL+\nutiHc36VASAiTgauojcd9obMXDvnTUwiIg6ld1QAsA9wc9f9RcQ36f1x0BjwPHAx8E/05ggcAjwF\nrM7mLxRHpL/j6B3qJr3LuGf1na/PdX8rgHvpXUre1Sy+gN55euf7sNLfqczxPuwkECSNJgcVJRUG\ngqTCQJBUGAiSCgNBUtFpIIzwtGDA/gY1yv2Ncm/QXX9dHyGM9H8U7G9Qo9zfKPcGHfXXdSBIGiED\nTUyKiBOBq+nNOPx6Zn6xZX1nQUkdycxoW2fGgTCTG50YCFJ3phIIg5wyeKMTaS8zSCDMhxudSJqG\nWb/VdXP5ZNRHdCUxWCBM6UYnmbmO3m28HUOQRtwgpwwjfaMTSdM34yOEzHwtIv6c3icB7b7RyWND\n60zSnJvTG6R4yiB1Z7YvO0rayxgIkgoDQVJhIEgqDARJhYEgqTAQJBUGgqTCQJBUGAiSCgNBUmEg\nSCoMBEmFgSCpMBAkFQaCpMJAkFQYCJIKA0FSYSBIKgwESYWBIKkwECQVBoKkwkCQVBgIkgoDQVJh\nIEgqDARJhYEgqdin6wY0OpYuXVqtf+xjH6vWL7zwwmo9M6v1iPqnlY+Pj1frF110UbW+cePGal0D\nBkJEbANeBn4OvJaZy4fRlKRuDOMI4fjM3DmE7UjqmGMIkopBAyGB70fE5ohYM4yGJHVn0FOGFZn5\nbES8FbgzIh7PzHv6V2iCwrCQ5oGBjhAy89nm6w5gI/CuCdZZl5nLHXCURt+MAyEiFkTEm3c/B04A\nHh1WY5LmXrRdG570hRGH0jsqgN6px82ZubblNTN7M03JokWLqvXzzz+/Wv/Qhz5UrS9cuLBab5tH\nMOg8hLbXP/3009X6McccU63v3Ll3XyzLzPoOZoAxhMz8IfDbM329pNHjZUdJhYEgqTAQJBUGgqTC\nQJBUGAiSihnPQ5jRmzkPYSBt9xu49NJLq/Wu5wG88MIL1XqbsbGxan3ZsmXV+tatW6v1I488crot\nzStTmYfgEYKkwkCQVBgIkgoDQVJhIEgqDARJhYEgqXAewjxy//33V+vvfOc7q/VB5yG0Xcc//vjj\nq/VB7zewYsWKan3Tpk3Vetu/f5999u6PKXEegqRpMRAkFQaCpMJAkFQYCJIKA0FSYSBIKpyHMEKO\nOOKIar1tHsKLL75Yrbfdj6BtnsCnPvWpav2cc86p1i+//PJq/Uc/+lG13qbtZ3nXrl3V+sc//vFq\nfd26ddPuaZQ4D0HStBgIkgoDQVJhIEgqDARJhYEgqTAQJBXOQ5hH2uYptM0jGPR+BGvWrKnWr732\n2mr9mGOOqda3bNlSra9atapaX79+fbXe9rN+4IEHVuuD7r+uDWUeQkTcEBE7IuLRvmUHRMSdEfFk\n83X/QZuV1L2pnDJ8Azhxj2XnAXdl5mHAXc33kua51kDIzHuAl/ZYfApwY/P8RmDlkPuS1IGZDiou\nzsztzfPngMVD6kdShwa+q2RmZm2wMCLWAPXRKEkjYaZHCM9HxBKA5uuOyVbMzHWZuTwzl8/wvSTN\nkZkGwm3AGc3zM4DvDKcdSV1qPWWIiG8CxwFjEfEMcDHwReCWiPgz4Clg9Ww2qZ7HH3+80/dvu5/C\nE088Ua233a+h7X4L551Xv5jV9rkSsz1PY2/QGgiZeeokpfcNuRdJHXPqsqTCQJBUGAiSCgNBUmEg\nSCoMBEnFwFOXNTqOPfbYar3tfgpt8wzGx8er9cMPP7xav++++6r1RYsWVett9zNo6/+kk06q1uUR\ngqQ+BoKkwkCQVBgIkgoDQVJhIEgqDARJhfMQ9iKnnXZatX7mmWdW6233E2ibB9D2+rZ5BoPez+Ca\na66p1ts+90EeIUjqYyBIKgwESYWBIKkwECQVBoKkwkCQVDgP4XWkbR5B16+/9957q/Vzzz23Wnee\nweA8QpBUGAiSCgNBUmEgSCoMBEmFgSCpMBAkFc5D2IvcfPPN1frSpUur9bGxsWq97XMdFixYUK23\n+dznPletO89g9rUeIUTEDRGxIyIe7Vt2SUQ8GxEPNY+TZ7dNSXNhKqcM3wBOnGD532bmUc3j9uG2\nJakLrYGQmfcAL81BL5I6Nsig4tkR8XBzSrH/0DqS1JmZBsK1wKHAUcB24IrJVoyINRHxQEQ8MMP3\nkjRHZhQImfl8Zv48M3cBXwPeVVl3XWYuz8zlM21S0tyYUSBExJK+b1cBj062rqT5I6Zwr/1vAscB\nY8DzwMXN90cBCWwDzsrM7a1vFjHYH9SrU23zEC677LJqfeXKldX6gw8+WK2fdNJJ1Xrb5za83mVm\n/YMvmMLEpMw8dYLF18+oI0kjzanLkgoDQVJhIEgqDARJhYEgqTAQJBWt8xCG+mbzfB7CokWLqvUX\nXnhhjjqZn+64445q/QMf+EC13va5DFddddW0e3o9mco8BI8QJBUGgqTCQJBUGAiSCgNBUmEgSCoM\nBEmFn8vQ59hjj63Wr7hi0jvFAfD4449X66effvq0e9qbrF27tlo/4YQTqvXDDz98mO1oAh4hSCoM\nBEmFgSCpMBAkFQaCpMJAkFQYCJKK19U8hLb7GVx33XXV+o4dO6r11/s8gwULFlTrX/3qV6v1iNY/\n19cs8whBUmEgSCoMBEmFgSCpMBAkFQaCpMJAkFS8ruYhrFq1qlpv+3v7TZs2DbOdeeeII46o1m+9\n9dZqvW3/tn1GSNv9JjS41iOEiDg4Iu6OiK0R8VhEfLJZfkBE3BkRTzZf95/9diXNpqmcMrwGfDoz\n3w68G/hERLwdOA+4KzMPA+5qvpc0j7UGQmZuz8wtzfOXgXHgIOAU4MZmtRuBlbPVpKS5Ma1BxYhY\nBhwN3AcszsztTek5YPFQO5M056Y8qBgR+wG3Audk5k/6/xAlM3OyD3KNiDXAmkEblTT7pnSEEBH7\n0guDmzJzQ7P4+YhY0tSXABP+KWBmrsvM5Zm5fBgNS5o9U7nKEMD1wHhmXtlXug04o3l+BvCd4bcn\naS5F27XfiFgB3As8AuxqFl9AbxzhFuAQ4ClgdWa+1LKt+pvNsrbr6OPj49X61q1bq/UvfOELA21/\n8+bN1XqbpUuXVuvvfe97q/W2eRorV9bHjdvuZ9D2s3b11VdX6+eee261rrrMbL3hROsYQmb+KzDZ\nht433aYkjS6nLksqDARJhYEgqTAQJBUGgqTCQJBUtM5DGOqbdTwPoc369eur9dm+Dv/ggw9W620O\nOeSQan3hwoXV+qD9t71+7dq11fo111xTre/cubNaV91U5iF4hCCpMBAkFQaCpMJAkFQYCJIKA0FS\nYSBIKpyH0GfRokXV+u23316tL19evynUrl27qvXZngfQ9vpXX321Wm/7XITLL7+8Wt+4cWO1rtnl\nPARJ02IgSCoMBEmFgSCpMBAkFQaCpMJAkFQ4D2EaxsbGqvVLL710oO2vWVP/xLsNGzZU64PeL6Dt\ncxHa5iFotDkPQdK0GAiSCgNBUmEgSCoMBEmFgSCpMBAkFc5DkF4nhjIPISIOjoi7I2JrRDwWEZ9s\nll8SEc9GxEPN4+RhNC2pO61HCBGxBFiSmVsi4s3AZmAlsBp4JTO/POU38whB6sxUjhD2mcJGtgPb\nm+cvR8Q4cNDg7UkaNdMaVIyIZcDRwH3NorMj4uGIuCEi9p/kNWsi4oGIeGCgTiXNuikPKkbEfsAm\nYG1mboiIxcBOIIFL6Z1W/GnLNjxlkDoylVOGKQVCROwLfBf4XmZeOUF9GfDdzHxHy3YMBKkjw7rK\nEMD1wHh/GDSDjbutAh6dSZOSRsdUrjKsAO4FHgF2f7DABcCpwFH0Thm2AWc1A5C1bXmEIHVkaKcM\nw2IgSN3xBimSpsVAkFQYCJIKA0FSYSBIKgwESYWBIKkwECQVBoKkwkCQVBgIkgoDQVJhIEgqDARJ\nRetNVodsJ/BU3/djzbJRZX+DGeX+Rrk3GH5/S6ey0pzeD+GX3jzigcxc3lkDLexvMKPc3yj3Bt31\n5ymDpMJAkFR0HQjrOn7/NvY3mFHub5R7g47663QMQdJo6foIQdIIMRAkFQaCpMJAkFQYCJKK/wOA\nmvlpkaUjtQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd7f01c9790>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAEMCAYAAAAiW8hnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFO5JREFUeJzt3X+QVeV9x/H3l+WHIiA/VERjtZlkTBiMQJBJqlWiQdHR\nQRrFaK0kTYXE6IQZ29QQM+JYHEuIxtGUGVKJmBoKjVidqZoQGjFOrIkojYgGnRSUH/JLMawSF5Zv\n/zhnn1w3d59nd8/de+7C5zWzs7vne/fc7z2wnz0/nvscc3dERAD6lN2AiDQOBYKIBAoEEQkUCCIS\nKBBEJFAgiEigQBCRQIFQAjNrrvg4aGb7Kr7/azOba2b78+/3mNkvzezTVdbz5/nPL6xSczN70cz6\nVCz7JzO7v+L7L5nZK2a218y2m9ljZjY4r91vZi3ter2i4me/kK//PTN708wWmtnQinr0NZjZpLzH\nh9v1fXq+/Ml2r+Xddr18vd3z7M0/NpjZvWY2quv/MqJAKIG7D2r7AF4HLqlY9mD+sGV5/Rjg58B/\nVFnVNcDbwBVmNqBK/QTg89V6MLNzgNuBK919MPBxYFm7h82v7NXdl+U/eyPwz8A/AEcDnwJOBlaa\nWf+Kn0+9hp3Ap81sRMWyGcCGKi2f3q6X+e2eZzAwHJgGHA+sUSh0nQKhwbn7AeBB4EQzO7ZtuZkZ\nWSDcDOwHLqny4/OBW82sb5XaGcAz7v5C/jxvufsSd98b68fMhgC3Aje4+xPuvt/dNwLTgVOAqzv7\nGoAW4D/JQ8vMmoAr8sd2Wd7LS/k6dgI3dmc9hzMFQoPL/+JeA+wm2xtocxbwIeDfgeVkf1nbWwH8\nHvhCldqzwAVmdquZndnBHkY1fwEcka87cPdm4DFgchdeA8ADeQ3gAmAdsLWTvVTl7q3AI8BfFlnP\n4UiB0Limm9keYB9wLXBZ/pe2zQzgcXd/G/gRMMXMjmu3Dge+BXyr3a487v4L4K+A8cB/AbvN7M78\nr3Sbv8+P//eY2a582THArna9tNmW1zv7GnD3XwLDzexUsmB4oIPt8XxFL3vM7IIOHtdmK9khhHSB\nAqFxLXf3ocBIsr+an2wrmNmRwOXku9bu/gzZuYir2q/E3R8DNgOzqtQed/dLyH5xppLtSfxdxUMW\nuPvQ/KPtF30XcEwHhyGj8nryNbTzQ+B64DPAwx08ZnxFL0Pd/ScdPK7NicBbicdIOwqEBufuu4CZ\nwNyKk2TTgCHAv+Rn+N8k+wWodtgA8E1gDjCwg+c46O6rgP8GxiRaegZ4n2zvIjCzQcCFwKpOvoZK\nPwSuAx5z9/cSz5+UX1m5BPhF0XUdbhQIvYC7/xb4CfD1fNEMYDFwGjA2/zgTON3MTqvy80+S/YUO\ngWFmU83s82Y2zDITgXOA/0n08g7ZScV7zGyKmfUzs1PIzmNsJvvl7sxrqKz9X/7c34w9d4qZ9TWz\njwNLya403FlkfYcjBULv8W1gppmdDJwHfNfd36z4WAM8Qcd7CTfzwWPqt8mO618lO/H4b8C3Ky57\ndii/5DcHWJD/7LPAG8B57v5+J15D+3MduPvT7h47mfi/7cYhfLeidoWZNQPvAI+Snbz8ZGJ9Uo27\n1/0DmAL8FngNuKmMHhL9bQReBNYCzzVAP4uBHcC6imXDgZVkv9ArgWEN1t9cYEu+DdcCF5XY30lk\n4yDWAy8BX2ukbRjpr+7b0PInrpv8LPYGsstTm4Ffkw2OWV/XRiLMbCMwwbNj39KZ2dlAM/CAu4/J\nl80H3nL3O8zsJrL/zP/YQP3NBZrdfUEZPVXKz1uMcvfn85GYa4BLyU6ilr4NI/1Np87bsIxDhonA\na+7+O3dvIbuOPrWEPnoNd3+KPz1jPhVYkn+9hOw/UCk66K9huPs2d38+/3ov8DLZSdiG2IaR/uqu\njEA4kex4s81mSnrxEQ78zMzWmNnMspvpwEh335Z//SbZpb1Gc4OZ/cbMFpvZsLKbAchPgI4jO+/R\ncNuwXX9Q522ok4rVneXuY8kuo3013yVuWJ4d9zXabLkLgQ+TXQHZBnyn3HbCpdGHgNnu/vvKWiNs\nwyr91X0blhEIW8hOorT5UL6sYbj7lvzzDrKBMhPL7aiq7W3X9PPPO0ru5wPcfbu7t7r7QeD7lLwN\nzawf2S/bg+7eNuy6YbZhtf7K2IZlBMKvgY9a9tbd/mRvbHm0hD6qMrOj7I9vAT4KOJ/sGn6jeZQ/\nXmKcQTZ2v2G0G4A0jRK3Yf5GsPuAl929cmxCQ2zDjvorYxvW/SoDgJldBHwXaAIWu/u8ujfRATP7\nMH8cPtsX+FHZ/ZnZUmAS2fsEtgO3kL1LcDnwZ8AmYLq7l3Jir4P+JpHt6jrZZdxZFcfr9e7vLLJR\niy8CB/PFc8iO00vfhpH+rqTO27CUQBCRxqSTiiISKBBEJFAgiEigQBCRQIEgIkGpgdDAw4IB9VdU\nI/fXyL1Bef2VvYfQ0P8oqL+iGrm/Ru4NSuqv7EAQkQZSaGCSmU0B7iYbcfiv7n5H4vEaBdWLZSNs\ne44GyfUsd0/+A3Y7ELoz0YkCobGlfuH79q020XLn662trdF6S0tLtC7FdCYQihwyaKITkUNMkUDo\nDROdiEgXxPfxaiC/fNLoZ3RFhGKB0KmJTtx9EbAIdA5BpNEVOWRo6IlORKTrur2H4O4HzOx6srvx\ntE108lLNOpOaa2pqitZTVwH2799fqF5Unz7Fhs0cPHgw/aDDXF0nSNEhQ7mKBkLZFAjF9PRlRxE5\nxCgQRCRQIIhIoEAQkUCBICKBAkFEgh4fuiy1k7rslqofOHCg0PN/5CMfidY/+9nPRuvjxo2L1l99\n9dVo/Qc/+EG0vnv37mg99W7MotvnUKA9BBEJFAgiEigQRCRQIIhIoEAQkUCBICKBAkFEAo1D6EVS\nb99N1UePHh2t33XXXdH6+eefH60X1dzcHK2nxjHMmTMnWt+8eXOXe6qUmpX6UJhGXnsIIhIoEEQk\nUCCISKBAEJFAgSAigQJBRAIFgogEmoa9jlLzFaTGEaSmUZ88eXK0Pnv27Gj9ggsuiNZ7Wuq+Dv36\n9YvWn3766Wg9tX3+8Ic/ROtHHHFEoZ8vm6ZhF5EuUSCISKBAEJFAgSAigQJBRAIFgogECgQRCTQO\noReZMmVKtD5v3rxoffz48dF66r4E69evj9afeeaZaH3t2rXReuq+CXPnzo3WR4wYEa1fd9110foD\nDzwQrb/77rvRetFxJj2tM+MQCk2QYmYbgb1AK3DA3ScUWZ+IlKsWMyZ9xt131WA9IlIynUMQkaBo\nIDjwMzNbY2Yza9GQiJSn6CHDWe6+xcyOA1aa2Svu/lTlA/KgUFiI9AKF9hDcfUv+eQfwMDCxymMW\nufsEnXAUaXzdDgQzO8rMBrd9DZwPrKtVYyJSf0UOGUYCD+dz1fcFfuTuT9Skq0PUwIEDo/VZs2ZF\n6zfffHO0Pnz48C73VGnPnj3R+u233x6tL1u2rNDzp4wZMyZaT22/CRPiO6lLly7tck+Vyh5nUAvd\nDgR3/x1weg17EZGS6bKjiAQKBBEJFAgiEigQRCRQIIhIoEAQkaAW73Y8bBR9v3tLS0u0fvbZZ0fr\nRccZpHzve9+L1lOv75prronWU/MNpGzcuLHQz48ePTpaT43DyMfcdKiec4v0FO0hiEigQBCRQIEg\nIoECQUQCBYKIBAoEEQkUCCISaBxCF/T0debjjz8+Wk+NA3jiifh0FGvWrInWV6xYEa3feuut0frQ\noUOj9dQ4hEmTJkXrl19+ebSe8vbbbxf6+UNhnEGK9hBEJFAgiEigQBCRQIEgIoECQUQCBYKIBAoE\nEQk0DqELUtehU/MltLa2Ruv33ntvtN63b/yfa+XKldH61q1bo/V+/fpF67t2xW/yPWjQoGh9xowZ\n0fqFF14YrY8fPz5a37lzZ7S+fPnyaD0ltf0PHDhQaP2NQHsIIhIoEEQkUCCISKBAEJFAgSAigQJB\nRAIFgogEVs/3eJvZIf2G8kaftz81ziA1TuKMM86I1ufPnx+tn3nmmdH6vn37ovXUOIfVq1dH6zNn\nzozWN2zYEK03NTVF66ntVzZ3j/8HpRN7CGa22Mx2mNm6imXDzWylmb2afx5WtFkRKV9nDhnuB6a0\nW3YTsMrdPwqsyr8XkV4uGQju/hTwVrvFU4El+ddLgEtr3JeIlKC7JxVHuvu2/Os3gZE16kdESlT4\nzU3u7rGThWY2E4ifzRGRhtDdPYTtZjYKIP+8o6MHuvsid5/g7hO6+VwiUifdDYRHgbb3ss4AHqlN\nOyJSpuQhg5ktBSYBx5jZZuAW4A5guZl9CdgETO/JJnuLnh5nkHo//pgxY6L1qVOnRuvHHXdctJ66\n78InPvGJaD11HT81zuCFF16I1m+77bZo/bXXXovWU+NIGn2cQS0kA8Hdr+ygdF6NexGRkmnosogE\nCgQRCRQIIhIoEEQkUCCISKBAEJFA92XoRVLz/n/sYx+L1r/4xS9G6yeffHKXe+qK1DiNN954I1qf\nN29etL5q1aou9yQfpD0EEQkUCCISKBBEJFAgiEigQBCRQIEgIoECQUQCjUNoIKn7Juzfvz9a3759\ne7S+d+/eLvdU6eDBg9F6ar6A1OtLzYcwYsSIaP3YY4+N1nfu3Bmt9+kT//uYev2HAu0hiEigQBCR\nQIEgIoECQUQCBYKIBAoEEQkUCCISWE/fS+ADTxa55Zuk71uQus4/ePDgaP2yyy6L1seNGxetv/PO\nO9F6ar6G0aNHR+sXX3xxtJ4ax3DLLbdE6/fcc0+03tzcHK339nEK7h6/8QTaQxCRCgoEEQkUCCIS\nKBBEJFAgiEigQBCRQIEgIoHGIdSQWfwyb9++8eknUuMMil7n7t+/f6H1p8YZpKTmK5g2bVq0/o1v\nfCNaT40T+PKXvxytP/7449F66t+v6PbpaTUZh2Bmi81sh5mtq1g218y2mNna/OOios2KSPk6c8hw\nPzClyvK73H1s/vFYbdsSkTIkA8HdnwLeqkMvIlKyIicVbzCz3+SHFMNq1pGIlKa7gbAQ+DAwFtgG\nfKejB5rZTDN7zsye6+ZziUiddCsQ3H27u7e6+0Hg+8DEyGMXufsEd5/Q3SZFpD66FQhmNqri22nA\nuo4eKyK9R/K+DGa2FJgEHGNmm4FbgElmNhZwYCMwqwd77DVSYzpS91UoKjUOoqWlpUfXnxrnkLov\nwqJFi6L10047LVq//vrro/UxY8ZE66lxCKlxBqntU88xP92VDAR3v7LK4vt6oBcRKZmGLotIoEAQ\nkUCBICKBAkFEAgWCiAQKBBEJkpcdpfNS16EHDBhQ6Of37dsXrRe9zl30OnrR+QDGjh0brU+ePLnQ\n+k844YRo/cgjj4zWU9v/UKA9BBEJFAgiEigQRCRQIIhIoEAQkUCBICKBAkFEAo1DqJCa1z9134Jz\nzz03Wv/c5z4XrS9btixaX716dbSekhpnUFTqvhJDhgyJ1q+++upo/dRTT43W9+7dG61v2LAhWk+N\nM2hqaorWU6+/N9AegogECgQRCRQIIhIoEEQkUCCISKBAEJFAgSAigcYhVEiNMxg6dGi0Pnv27Gj9\nqKOOitY3bdoUraf069cvWu/p+0KMGjUqWr/22muj9a985SuFnn/VqlXR+kMPPVRo/YcD7SGISKBA\nEJFAgSAigQJBRAIFgogECgQRCRQIIhIcVuMQ+vaNv9zUfQUmTpwYrZ9zzjnR+pIlS6L1jRs3Rusp\nRccZ9O/fP1qfNGlStJ4aR3DppZd2taUPWL9+fbS+cOHCaH3Hjh3RetnjOBpBcg/BzE4ys5+b2Xoz\ne8nMvpYvH25mK83s1fzzsJ5vV0R6UmcOGQ4AN7r7aOBTwFfNbDRwE7DK3T8KrMq/F5FeLBkI7r7N\n3Z/Pv94LvAycCEwF2vaBlwDF9gdFpHRdOqloZqcA44BngZHuvi0vvQmMrGlnIlJ3nT6paGaDgIeA\n2e7++8oJO93dzazqnUDNbCYws2ijItLzOrWHYGb9yMLgQXdfkS/ebmaj8voooOopXHdf5O4T3H1C\nLRoWkZ7TmasMBtwHvOzud1aUHgVm5F/PAB6pfXsiUk+dOWQ4E/gb4EUzW5svmwPcASw3sy8Bm4Dp\nPdNi7aTGGaQMGxa/spqaTyF1HT91X4LNmzdH66n7Chx99NHR+sUXXxytp8YRnHTSSdF6yiuvvBKt\nL1iwIFr/6U9/Wuj5D4dxBinJQHD3p4GO7vBxXm3bEZEyaeiyiAQKBBEJFAgiEigQRCRQIIhIoEAQ\nkcDcq4447pkn62B4c6OoHI7dnXpq3v/UdfytW7dG66n7QgwYMCBaf//996P1gQMHRuspqfX/6le/\nitZT8xk88kh87Nt7770XrR/u3D3+HxjtIYhIBQWCiAQKBBEJFAgiEigQRCRQIIhIoEAQkeCwui9D\nSuq+BKnr7HPnzo3Wm5ubo/WrrroqWu/TJ57fLS0t0XpqnMHrr78era9bty5a37BhQ7R+9913R+tF\n70shxWkPQUQCBYKIBAoEEQkUCCISKBBEJFAgiEigQBCRQPMhdEFTU1O03traGq2n7oswfXr81hbn\nnntutJ4ahzBo0KBo/cknn4zWf/zjH0fru3fvjtZT/aXmc0jdVyO1/Q93mg9BRLpEgSAigQJBRAIF\ngogECgQRCRQIIhIoEEQk0DiEGkrdt6Hoth4yZEi0vn///mh93759hZ5fereajEMws5PM7Odmtt7M\nXjKzr+XL55rZFjNbm39cVIumRaQ8nZkx6QBwo7s/b2aDgTVmtjKv3eXuC3quPRGpp2QguPs2YFv+\n9V4zexk4sacbE5H669JJRTM7BRgHPJsvusHMfmNmi81sWAc/M9PMnjOz5wp1KiI9rtMnFc1sELAa\nmOfuK8xsJLALcOA2YJS7/21iHTqpWIBOKkoRNXtzk5n1Ax4CHnT3FfnKt7t7q7sfBL4PTCzSrIiU\nrzNXGQy4D3jZ3e+sWD6q4mHTgPgc3SLS8JKHDGZ2FvAL4EXgYL54DnAlMJbskGEjMCs/ARlb1yF9\nyCDSyDpzyKCBSSKHCU2QIiJdokAQkUCBICKBAkFEAgWCiAQKBBEJFAgiEigQRCRQIIhIoEAQkUCB\nICKBAkFEAgWCiAQKBBEJOjPrci3tAjZVfH9MvqxRqb9iGrm/Ru4Nat/fyZ15UF3nQ/iTJzd7zt0n\nlNZAgvorppH7a+TeoLz+dMggIoECQUSCsgNhUcnPn6L+imnk/hq5Nyipv1LPIYhIYyl7D0FEGogC\nQUQCBYKIBAoEEQkUCCIS/D/wgIGIT1t+eQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd7d816fad0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "naug = 2\n",
    "batch_xs = trainimg[:naug, :]\n",
    "xs2 = augment_img(batch_xs)\n",
    "for i in range(naug):\n",
    "    x1 = batch_xs[i, :].reshape([28, 28])\n",
    "    x2 = xs2[i, :].reshape([28, 28])\n",
    "    plt.matshow(x1, vmin=0, vmax=1, cmap=plt.cm.gray)\n",
    "    plt.title(\"ORIGINAL\")\n",
    "    plt.show()\n",
    "    plt.matshow(x2, vmin=0, vmax=1, cmap=plt.cm.gray)\n",
    "    plt.title(\"TRANSFORMED\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 运行"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# PARAMETERS\n",
    "training_epochs = 100\n",
    "batch_size      = 50\n",
    "display_step    = 4\n",
    "val_acc         = 0\n",
    "val_acc_max     = 0\n",
    "# OPTIMIZE\n",
    "for epoch in range(training_epochs):\n",
    "    avg_cost = 0.\n",
    "    total_batch = int(mnist.train.num_examples/batch_size)\n",
    "    # ITERATION\n",
    "    for i in range(total_batch):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "        # AUGMENT DATA\n",
    "        batch_xs = augment_img(batch_xs)\n",
    "        feeds = {x: batch_xs, y: batch_ys, is_training: True}\n",
    "        sess.run(optm, feed_dict=feeds)\n",
    "        avg_cost += sess.run(cost, feed_dict=feeds)\n",
    "    avg_cost = avg_cost / total_batch\n",
    "    # DISPLAY\n",
    "    if (epoch+1) % display_step == 0:\n",
    "        print (\"Epoch: %03d/%03d cost: %.9f\" % (epoch+1, training_epochs, avg_cost))\n",
    "        randidx = np.random.permutation(trainimg.shape[0])[:500]\n",
    "        feeds = {x: trainimg[randidx], y: trainlabel[randidx], is_training: False}\n",
    "        train_acc = sess.run(accr, feed_dict=feeds)\n",
    "        print (\" TRAIN ACCURACY: %.5f\" % (train_acc))\n",
    "        feeds = {x: valimg, y: vallabel, is_training: False}\n",
    "        val_acc = sess.run(accr, feed_dict=feeds)\n",
    "        print (\" VALIDATION ACCURACY: %.5f\" % (val_acc))\n",
    "    # SAVE\n",
    "    if (epoch+1) % save_step == 0:\n",
    "        savename = savedir + \"net-\" + str(epoch) + \".ckpt\"\n",
    "        saver.save(sess=sess, save_path=savename)\n",
    "        print (\" [%s] SAVED.\" % (savename))\n",
    "    # MAXIMUM VALIDATION ACCURACY\n",
    "    if val_acc > val_acc_max:\n",
    "        val_acc_max = val_acc\n",
    "        best_epoch = epoch\n",
    "        print (\"\\x1b[31m BEST EPOCH UPDATED!! [%d] \\x1b[0m\" % (best_epoch))\n",
    "print (\"OPTIMIZATION FINISHED\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 计算测试的精确度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "best_epoch = 55\n",
    "restorename = savedir + \"net-\" + str(best_epoch) + \".ckpt\"\n",
    "print (\"LOADING [%s]\" % (restorename))\n",
    "saver.restore(sess, restorename)\n",
    "feeds = {x: testimg, y: testlabel, is_training: False}\n",
    "test_acc = sess.run(accr, feed_dict=feeds)\n",
    "print (\"TEST ACCURACY: %.5f\" % (test_acc))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}