tensorflow1_ipynb/cnn/cnn-vgg16.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
    "- Author: Sebastian Raschka\n",
    "- GitHub Repository: https://github.com/rasbt/deeplearning-models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sebastian Raschka \n",
      "\n",
      "CPython 3.6.1\n",
      "IPython 6.0.0\n",
      "\n",
      "tensorflow 1.2.0\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -a 'Sebastian Raschka' -v -p tensorflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Zoo -- Convolutional Neural Network (VGG16)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The VGG-16 Convolutional Neural Network Architecture [1] implemented in TensorFlow and trained on Cifar-10 [2, 3] images.\n",
    "\n",
    "References:\n",
    "\n",
    "- [1] Simonyan, K., & Zisserman, A. (2015). [Very Deep Convolutional Networks for Large-Scale Image Recognition](https://arxiv.org/abs/1409.1556). International Conference on Learning Representations (ICRL), 1–14. https://doi.org/10.1016/j.infsof.2008.09.005\n",
    "- [2] Krizhevsky, A. (2009). [Learning Multiple Layers of Features from Tiny Images](https://doi.org/10.1.1.222.9220 http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.222.9220&rep=rep1&type=pdf). Science Department, University of Toronto.\n",
    "- [3] https://www.cs.toronto.edu/~kriz/cifar.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "insert() takes exactly 2 arguments (1 given)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-0beb05071f51>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minsert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'..'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mhelper\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdownload_and_extract_cifar\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: insert() takes exactly 2 arguments (1 given)"
     ]
    }
   ],
   "source": [
    "##########################\n",
    "### DATASET\n",
    "##########################\n",
    "\n",
    "# load utilities from ../helper.py\n",
    "import sys\n",
    "sys.path.insert(0, '..')  \n",
    "from helper import download_and_extract_cifar\n",
    "from helper import Cifar10Loader\n",
    "\n",
    "dest = download_and_extract_cifar('./cifar-10')\n",
    "cifar = Cifar10Loader(dest, normalize=True, \n",
    "                      zero_center=True,\n",
    "                      channel_mean_center=False)\n",
    "cifar.num_train\n",
    "\n",
    "X, y = cifar.load_test()\n",
    "half = cifar.num_test // 2\n",
    "X_test, X_valid = X[:half], X[half:]\n",
    "y_test, y_valid = y[:half], y[half:]\n",
    "\n",
    "del X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/gpu:0'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "tf.test.gpu_device_name()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Minibatch: 0200 | Cost: 9.004\n",
      "Minibatch: 0400 | Cost: 3.877\n",
      "Minibatch: 0600 | Cost: 3.213\n",
      "Minibatch: 0800 | Cost: 2.410\n",
      "Minibatch: 1000 | Cost: 2.384\n",
      "Minibatch: 1200 | Cost: 2.250\n",
      "Minibatch: 1400 | Cost: 2.352\n",
      "Epoch: 001 | AvgCost: 1897.994 | Train/Valid ACC: 0.152/0.145\n",
      "Minibatch: 0200 | Cost: 2.433\n",
      "Minibatch: 0400 | Cost: 2.426\n",
      "Minibatch: 0600 | Cost: 2.202\n",
      "Minibatch: 0800 | Cost: 2.221\n",
      "Minibatch: 1000 | Cost: 2.112\n",
      "Minibatch: 1200 | Cost: 2.182\n",
      "Minibatch: 1400 | Cost: 2.382\n",
      "Epoch: 002 | AvgCost: 2.219 | Train/Valid ACC: 0.176/0.168\n",
      "Minibatch: 0200 | Cost: 2.145\n",
      "Minibatch: 0400 | Cost: 2.089\n",
      "Minibatch: 0600 | Cost: 2.221\n",
      "Minibatch: 0800 | Cost: 1.903\n",
      "Minibatch: 1000 | Cost: 2.296\n",
      "Minibatch: 1200 | Cost: 2.169\n",
      "Minibatch: 1400 | Cost: 2.164\n",
      "Epoch: 003 | AvgCost: 2.168 | Train/Valid ACC: 0.166/0.173\n",
      "Minibatch: 0200 | Cost: 2.094\n",
      "Minibatch: 0400 | Cost: 2.015\n",
      "Minibatch: 0600 | Cost: 2.217\n",
      "Minibatch: 0800 | Cost: 2.028\n",
      "Minibatch: 1000 | Cost: 2.265\n",
      "Minibatch: 1200 | Cost: 2.103\n",
      "Minibatch: 1400 | Cost: 2.060\n",
      "Epoch: 004 | AvgCost: 2.144 | Train/Valid ACC: 0.189/0.188\n",
      "Minibatch: 0200 | Cost: 2.310\n",
      "Minibatch: 0400 | Cost: 1.986\n",
      "Minibatch: 0600 | Cost: 1.844\n",
      "Minibatch: 0800 | Cost: 2.183\n",
      "Minibatch: 1000 | Cost: 2.040\n",
      "Minibatch: 1200 | Cost: 2.004\n",
      "Minibatch: 1400 | Cost: 1.987\n",
      "Epoch: 005 | AvgCost: 2.075 | Train/Valid ACC: 0.180/0.181\n",
      "Minibatch: 0200 | Cost: 1.849\n",
      "Minibatch: 0400 | Cost: 1.898\n",
      "Minibatch: 0600 | Cost: 1.996\n",
      "Minibatch: 0800 | Cost: 1.968\n",
      "Minibatch: 1000 | Cost: 1.987\n",
      "Minibatch: 1200 | Cost: 2.003\n",
      "Minibatch: 1400 | Cost: 2.041\n",
      "Epoch: 006 | AvgCost: 2.010 | Train/Valid ACC: 0.176/0.176\n",
      "Minibatch: 0200 | Cost: 2.419\n",
      "Minibatch: 0400 | Cost: 2.109\n",
      "Minibatch: 0600 | Cost: 1.902\n",
      "Minibatch: 0800 | Cost: 1.937\n",
      "Minibatch: 1000 | Cost: 1.813\n",
      "Minibatch: 1200 | Cost: 1.990\n",
      "Minibatch: 1400 | Cost: 1.662\n",
      "Epoch: 007 | AvgCost: 1.913 | Train/Valid ACC: 0.256/0.251\n",
      "Minibatch: 0200 | Cost: 2.036\n",
      "Minibatch: 0400 | Cost: 1.690\n",
      "Minibatch: 0600 | Cost: 1.866\n",
      "Minibatch: 0800 | Cost: 1.574\n",
      "Minibatch: 1000 | Cost: 1.886\n",
      "Minibatch: 1200 | Cost: 1.720\n",
      "Minibatch: 1400 | Cost: 1.937\n",
      "Epoch: 008 | AvgCost: 1.769 | Train/Valid ACC: 0.270/0.267\n",
      "Minibatch: 0200 | Cost: 1.569\n",
      "Minibatch: 0400 | Cost: 1.800\n",
      "Minibatch: 0600 | Cost: 1.541\n",
      "Minibatch: 0800 | Cost: 1.499\n",
      "Minibatch: 1000 | Cost: 1.596\n",
      "Minibatch: 1200 | Cost: 1.994\n",
      "Minibatch: 1400 | Cost: 1.283\n",
      "Epoch: 009 | AvgCost: 1.653 | Train/Valid ACC: 0.343/0.337\n",
      "Minibatch: 0200 | Cost: 1.415\n",
      "Minibatch: 0400 | Cost: 1.648\n",
      "Minibatch: 0600 | Cost: 1.487\n",
      "Minibatch: 0800 | Cost: 1.454\n",
      "Minibatch: 1000 | Cost: 1.266\n",
      "Minibatch: 1200 | Cost: 1.267\n",
      "Minibatch: 1400 | Cost: 1.589\n",
      "Epoch: 010 | AvgCost: 1.526 | Train/Valid ACC: 0.469/0.459\n",
      "Minibatch: 0200 | Cost: 1.585\n",
      "Minibatch: 0400 | Cost: 1.449\n",
      "Minibatch: 0600 | Cost: 1.340\n",
      "Minibatch: 0800 | Cost: 1.297\n",
      "Minibatch: 1000 | Cost: 1.767\n",
      "Minibatch: 1200 | Cost: 1.088\n",
      "Minibatch: 1400 | Cost: 1.159\n",
      "Epoch: 011 | AvgCost: 1.342 | Train/Valid ACC: 0.536/0.517\n",
      "Minibatch: 0200 | Cost: 1.408\n",
      "Minibatch: 0400 | Cost: 1.510\n",
      "Minibatch: 0600 | Cost: 1.424\n",
      "Minibatch: 0800 | Cost: 1.019\n",
      "Minibatch: 1000 | Cost: 1.069\n",
      "Minibatch: 1200 | Cost: 1.465\n",
      "Minibatch: 1400 | Cost: 1.289\n",
      "Epoch: 012 | AvgCost: 1.203 | Train/Valid ACC: 0.590/0.576\n",
      "Minibatch: 0200 | Cost: 0.670\n",
      "Minibatch: 0400 | Cost: 0.993\n",
      "Minibatch: 0600 | Cost: 0.998\n",
      "Minibatch: 0800 | Cost: 1.103\n",
      "Minibatch: 1000 | Cost: 1.127\n",
      "Minibatch: 1200 | Cost: 0.969\n",
      "Minibatch: 1400 | Cost: 1.216\n",
      "Epoch: 013 | AvgCost: 1.111 | Train/Valid ACC: 0.634/0.603\n",
      "Minibatch: 0200 | Cost: 1.262\n",
      "Minibatch: 0400 | Cost: 1.147\n",
      "Minibatch: 0600 | Cost: 0.660\n",
      "Minibatch: 0800 | Cost: 0.970\n",
      "Minibatch: 1000 | Cost: 0.716\n",
      "Minibatch: 1200 | Cost: 1.017\n",
      "Minibatch: 1400 | Cost: 1.264\n",
      "Epoch: 014 | AvgCost: 1.036 | Train/Valid ACC: 0.638/0.617\n",
      "Minibatch: 0200 | Cost: 0.846\n",
      "Minibatch: 0400 | Cost: 0.843\n",
      "Minibatch: 0600 | Cost: 0.792\n",
      "Minibatch: 0800 | Cost: 0.718\n",
      "Minibatch: 1000 | Cost: 1.141\n",
      "Minibatch: 1200 | Cost: 1.024\n",
      "Minibatch: 1400 | Cost: 0.877\n",
      "Epoch: 015 | AvgCost: 0.949 | Train/Valid ACC: 0.683/0.654\n",
      "Minibatch: 0200 | Cost: 0.463\n",
      "Minibatch: 0400 | Cost: 0.854\n",
      "Minibatch: 0600 | Cost: 0.820\n",
      "Minibatch: 0800 | Cost: 0.952\n",
      "Minibatch: 1000 | Cost: 0.819\n",
      "Minibatch: 1200 | Cost: 0.648\n",
      "Minibatch: 1400 | Cost: 1.030\n",
      "Epoch: 016 | AvgCost: 0.872 | Train/Valid ACC: 0.738/0.697\n",
      "Minibatch: 0200 | Cost: 0.862\n",
      "Minibatch: 0400 | Cost: 1.010\n",
      "Minibatch: 0600 | Cost: 0.991\n",
      "Minibatch: 0800 | Cost: 0.664\n",
      "Minibatch: 1000 | Cost: 0.604\n",
      "Minibatch: 1200 | Cost: 0.879\n",
      "Minibatch: 1400 | Cost: 0.791\n",
      "Epoch: 017 | AvgCost: 0.823 | Train/Valid ACC: 0.753/0.698\n",
      "Minibatch: 0200 | Cost: 0.823\n",
      "Minibatch: 0400 | Cost: 0.619\n",
      "Minibatch: 0600 | Cost: 0.764\n",
      "Minibatch: 0800 | Cost: 1.127\n",
      "Minibatch: 1000 | Cost: 0.623\n",
      "Minibatch: 1200 | Cost: 0.796\n",
      "Minibatch: 1400 | Cost: 0.694\n",
      "Epoch: 018 | AvgCost: 0.771 | Train/Valid ACC: 0.769/0.709\n",
      "Minibatch: 0200 | Cost: 0.431\n",
      "Minibatch: 0400 | Cost: 0.688\n",
      "Minibatch: 0600 | Cost: 0.406\n",
      "Minibatch: 0800 | Cost: 0.906\n",
      "Minibatch: 1000 | Cost: 0.607\n",
      "Minibatch: 1200 | Cost: 0.445\n",
      "Minibatch: 1400 | Cost: 0.660\n",
      "Epoch: 019 | AvgCost: 0.721 | Train/Valid ACC: 0.780/0.707\n",
      "Minibatch: 0200 | Cost: 0.740\n",
      "Minibatch: 0400 | Cost: 0.542\n",
      "Minibatch: 0600 | Cost: 0.435\n",
      "Minibatch: 0800 | Cost: 0.953\n",
      "Minibatch: 1000 | Cost: 0.954\n",
      "Minibatch: 1200 | Cost: 0.544\n",
      "Minibatch: 1400 | Cost: 0.855\n",
      "Epoch: 020 | AvgCost: 0.672 | Train/Valid ACC: 0.779/0.717\n",
      "Minibatch: 0200 | Cost: 0.432\n",
      "Minibatch: 0400 | Cost: 0.376\n",
      "Minibatch: 0600 | Cost: 0.477\n",
      "Minibatch: 0800 | Cost: 0.476\n",
      "Minibatch: 1000 | Cost: 0.950\n",
      "Minibatch: 1200 | Cost: 0.468\n",
      "Minibatch: 1400 | Cost: 0.595\n",
      "Epoch: 021 | AvgCost: 0.636 | Train/Valid ACC: 0.818/0.737\n",
      "Minibatch: 0200 | Cost: 1.114\n",
      "Minibatch: 0400 | Cost: 0.519\n",
      "Minibatch: 0600 | Cost: 0.623\n",
      "Minibatch: 0800 | Cost: 0.721\n",
      "Minibatch: 1000 | Cost: 0.880\n",
      "Minibatch: 1200 | Cost: 0.654\n",
      "Minibatch: 1400 | Cost: 0.378\n",
      "Epoch: 022 | AvgCost: 0.596 | Train/Valid ACC: 0.825/0.742\n",
      "Minibatch: 0200 | Cost: 0.541\n",
      "Minibatch: 0400 | Cost: 0.478\n",
      "Minibatch: 0600 | Cost: 0.642\n",
      "Minibatch: 0800 | Cost: 0.871\n",
      "Minibatch: 1000 | Cost: 0.710\n",
      "Minibatch: 1200 | Cost: 0.682\n",
      "Minibatch: 1400 | Cost: 0.671\n",
      "Epoch: 023 | AvgCost: 0.663 | Train/Valid ACC: 0.776/0.712\n",
      "Minibatch: 0200 | Cost: 0.498\n",
      "Minibatch: 0400 | Cost: 0.379\n",
      "Minibatch: 0600 | Cost: 0.535\n",
      "Minibatch: 0800 | Cost: 0.435\n",
      "Minibatch: 1000 | Cost: 0.562\n",
      "Minibatch: 1200 | Cost: 0.777\n",
      "Minibatch: 1400 | Cost: 0.635\n",
      "Epoch: 024 | AvgCost: 0.539 | Train/Valid ACC: 0.853/0.759\n",
      "Minibatch: 0200 | Cost: 0.331\n",
      "Minibatch: 0400 | Cost: 0.419\n",
      "Minibatch: 0600 | Cost: 0.497\n",
      "Minibatch: 0800 | Cost: 0.619\n",
      "Minibatch: 1000 | Cost: 0.620\n",
      "Minibatch: 1200 | Cost: 0.196\n",
      "Minibatch: 1400 | Cost: 0.648\n",
      "Epoch: 025 | AvgCost: 0.498 | Train/Valid ACC: 0.849/0.738\n",
      "Minibatch: 0200 | Cost: 0.247\n",
      "Minibatch: 0400 | Cost: 0.263\n",
      "Minibatch: 0600 | Cost: 0.183\n",
      "Minibatch: 0800 | Cost: 0.251\n",
      "Minibatch: 1000 | Cost: 0.251\n",
      "Minibatch: 1200 | Cost: 0.447\n",
      "Minibatch: 1400 | Cost: 0.443\n",
      "Epoch: 026 | AvgCost: 0.469 | Train/Valid ACC: 0.850/0.744\n",
      "Minibatch: 0200 | Cost: 0.390\n",
      "Minibatch: 0400 | Cost: 0.325\n",
      "Minibatch: 0600 | Cost: 0.404\n",
      "Minibatch: 0800 | Cost: 0.363\n",
      "Minibatch: 1000 | Cost: 0.655\n",
      "Minibatch: 1200 | Cost: 0.195\n",
      "Minibatch: 1400 | Cost: 0.647\n",
      "Epoch: 027 | AvgCost: 0.485 | Train/Valid ACC: 0.855/0.751\n",
      "Minibatch: 0200 | Cost: 0.580\n",
      "Minibatch: 0400 | Cost: 0.177\n",
      "Minibatch: 0600 | Cost: 0.317\n",
      "Minibatch: 0800 | Cost: 0.249\n",
      "Minibatch: 1000 | Cost: 0.287\n",
      "Minibatch: 1200 | Cost: 0.282\n",
      "Minibatch: 1400 | Cost: 0.429\n",
      "Epoch: 028 | AvgCost: 0.416 | Train/Valid ACC: 0.887/0.759\n",
      "Minibatch: 0200 | Cost: 0.233\n",
      "Minibatch: 0400 | Cost: 0.118\n",
      "Minibatch: 0600 | Cost: 0.218\n",
      "Minibatch: 0800 | Cost: 0.195\n",
      "Minibatch: 1000 | Cost: 0.208\n",
      "Minibatch: 1200 | Cost: 0.188\n",
      "Minibatch: 1400 | Cost: 0.509\n",
      "Epoch: 029 | AvgCost: 0.488 | Train/Valid ACC: 0.874/0.767\n",
      "Minibatch: 0200 | Cost: 0.313\n",
      "Minibatch: 0400 | Cost: 0.508\n",
      "Minibatch: 0600 | Cost: 0.553\n",
      "Minibatch: 0800 | Cost: 0.385\n",
      "Minibatch: 1000 | Cost: 0.272\n",
      "Minibatch: 1200 | Cost: 0.366\n",
      "Minibatch: 1400 | Cost: 0.156\n",
      "Epoch: 030 | AvgCost: 0.404 | Train/Valid ACC: 0.909/0.776\n"
     ]
    }
   ],
   "source": [
    "##########################\n",
    "### SETTINGS\n",
    "##########################\n",
    "\n",
    "# Hyperparameters\n",
    "learning_rate = 0.001\n",
    "training_epochs = 30\n",
    "batch_size = 32\n",
    "\n",
    "# Other\n",
    "print_interval = 200\n",
    "\n",
    "# Architecture\n",
    "image_width, image_height, image_depth = 32, 32, 3\n",
    "n_classes = 10\n",
    "\n",
    "\n",
    "##########################\n",
    "### WRAPPER FUNCTIONS\n",
    "##########################\n",
    "\n",
    "def conv_layer(input, input_channels, output_channels, \n",
    "               kernel_size, strides, scope, padding='SAME'):\n",
    "    with tf.name_scope(scope):\n",
    "        weights_shape = kernel_size + [input_channels, output_channels]\n",
    "        weights = tf.Variable(tf.truncated_normal(shape=weights_shape,\n",
    "                                                  mean=0.0,\n",
    "                                                  stddev=0.1,\n",
    "                                                  dtype=tf.float32),\n",
    "                                                  name='weights')\n",
    "        biases = tf.Variable(tf.zeros(shape=[output_channels]),\n",
    "                             name='biases')\n",
    "        conv = tf.nn.conv2d(input=input,\n",
    "                            filter=weights,\n",
    "                            strides=strides,\n",
    "                            padding=padding,\n",
    "                            name='convolution')\n",
    "        out = tf.nn.bias_add(conv, biases, name='logits')\n",
    "        out = tf.nn.relu(out, name='activation')\n",
    "        return out\n",
    "\n",
    "\n",
    "def fc_layer(input, output_nodes, scope,\n",
    "             activation=None, seed=None):\n",
    "    with tf.name_scope(scope):\n",
    "        shape = int(np.prod(input.get_shape()[1:]))\n",
    "        flat_input = tf.reshape(input, [-1, shape])\n",
    "        weights = tf.Variable(tf.truncated_normal(shape=[shape,\n",
    "                                                         output_nodes],\n",
    "                                                  mean=0.0,\n",
    "                                                  stddev=0.1,\n",
    "                                                  dtype=tf.float32,\n",
    "                                                  seed=seed),\n",
    "                                                  name='weights')\n",
    "        biases = tf.Variable(tf.zeros(shape=[output_nodes]),\n",
    "                             name='biases')\n",
    "        act = tf.nn.bias_add(tf.matmul(flat_input, weights), biases, \n",
    "                             name='logits')\n",
    "\n",
    "        if activation is not None:\n",
    "            act = activation(act, name='activation')\n",
    "\n",
    "        return act\n",
    "\n",
    "\n",
    "##########################\n",
    "### GRAPH DEFINITION\n",
    "##########################\n",
    "\n",
    "g = tf.Graph()\n",
    "with g.as_default():\n",
    "\n",
    "    # Input data\n",
    "    tf_x = tf.placeholder(tf.float32, [None, image_width, image_height, image_depth], name='features')\n",
    "    tf_y = tf.placeholder(tf.float32, [None, n_classes], name='targets')\n",
    "     \n",
    "    ##########################\n",
    "    ### VGG16 Model\n",
    "    ##########################\n",
    "\n",
    "    # =========\n",
    "    # BLOCK 1\n",
    "    # =========\n",
    "    conv_layer_1 = conv_layer(input=tf_x,\n",
    "                              input_channels=3,\n",
    "                              output_channels=64,\n",
    "                              kernel_size=[3, 3],\n",
    "                              strides=[1, 1, 1, 1],\n",
    "                              scope='conv1')\n",
    "    \n",
    "    conv_layer_2 = conv_layer(input=conv_layer_1,\n",
    "                              input_channels=64,\n",
    "                              output_channels=64,\n",
    "                              kernel_size=[3, 3],\n",
    "                              strides=[1, 1, 1, 1],\n",
    "                              scope='conv2')    \n",
    "    \n",
    "    pool_layer_1 = tf.nn.max_pool(conv_layer_2,\n",
    "                                  ksize=[1, 2, 2, 1], \n",
    "                                  strides=[1, 2, 2, 1],\n",
    "                                  padding='SAME',\n",
    "                                  name='pool1') \n",
    "    # =========\n",
    "    # BLOCK 2\n",
    "    # =========\n",
    "    conv_layer_3 = conv_layer(input=pool_layer_1,\n",
    "                              input_channels=64,\n",
    "                              output_channels=128,\n",
    "                              kernel_size=[3, 3],\n",
    "                              strides=[1, 1, 1, 1],\n",
    "                              scope='conv3')    \n",
    "    \n",
    "    conv_layer_4 = conv_layer(input=conv_layer_3,\n",
    "                              input_channels=128,\n",
    "                              output_channels=128,\n",
    "                              kernel_size=[3, 3],\n",
    "                              strides=[1, 1, 1, 1],\n",
    "                              scope='conv4')    \n",
    "    \n",
    "    pool_layer_2 = tf.nn.max_pool(conv_layer_4,\n",
    "                                  ksize=[1, 2, 2, 1], \n",
    "                                  strides=[1, 2, 2, 1],\n",
    "                                  padding='SAME',\n",
    "                                  name='pool2') \n",
    "    # =========\n",
    "    # BLOCK 3\n",
    "    # =========\n",
    "    conv_layer_5 = conv_layer(input=pool_layer_2,\n",
    "                              input_channels=128,\n",
    "                              output_channels=256,\n",
    "                              kernel_size=[3, 3],\n",
    "                              strides=[1, 1, 1, 1],\n",
    "                              scope='conv5')        \n",
    "    \n",
    "    conv_layer_6 = conv_layer(input=conv_layer_5,\n",
    "                              input_channels=256,\n",
    "                              output_channels=256,\n",
    "                              kernel_size=[3, 3],\n",
    "                              strides=[1, 1, 1, 1],\n",
    "                              scope='conv6')      \n",
    "    \n",
    "    conv_layer_7 = conv_layer(input=conv_layer_6,\n",
    "                              input_channels=256,\n",
    "                              output_channels=256,\n",
    "                              kernel_size=[3, 3],\n",
    "                              strides=[1, 1, 1, 1],\n",
    "                              scope='conv7')\n",
    "    \n",
    "    pool_layer_3 = tf.nn.max_pool(conv_layer_7,\n",
    "                                  ksize=[1, 2, 2, 1], \n",
    "                                  strides=[1, 2, 2, 1],\n",
    "                                  padding='SAME',\n",
    "                                  name='pool3') \n",
    "    # =========\n",
    "    # BLOCK 4\n",
    "    # =========\n",
    "    conv_layer_8 = conv_layer(input=pool_layer_3,\n",
    "                              input_channels=256,\n",
    "                              output_channels=512,\n",
    "                              kernel_size=[3, 3],\n",
    "                              strides=[1, 1, 1, 1],\n",
    "                              scope='conv8')      \n",
    "    \n",
    "    conv_layer_9 = conv_layer(input=conv_layer_8,\n",
    "                              input_channels=512,\n",
    "                              output_channels=512,\n",
    "                              kernel_size=[3, 3],\n",
    "                              strides=[1, 1, 1, 1],\n",
    "                              scope='conv9')     \n",
    "    \n",
    "    conv_layer_10 = conv_layer(input=conv_layer_9,\n",
    "                               input_channels=512,\n",
    "                               output_channels=512,\n",
    "                               kernel_size=[3, 3],\n",
    "                               strides=[1, 1, 1, 1],\n",
    "                               scope='conv10')   \n",
    "    \n",
    "    pool_layer_4 = tf.nn.max_pool(conv_layer_10,\n",
    "                                  ksize=[1, 2, 2, 1], \n",
    "                                  strides=[1, 2, 2, 1],\n",
    "                                  padding='SAME',\n",
    "                                  name='pool4') \n",
    "    # =========\n",
    "    # BLOCK 5\n",
    "    # =========\n",
    "    conv_layer_11 = conv_layer(input=pool_layer_4,\n",
    "                               input_channels=512,\n",
    "                               output_channels=512,\n",
    "                               kernel_size=[3, 3],\n",
    "                               strides=[1, 1, 1, 1],\n",
    "                               scope='conv11')   \n",
    "    \n",
    "    conv_layer_12 = conv_layer(input=conv_layer_11,\n",
    "                               input_channels=512,\n",
    "                               output_channels=512,\n",
    "                               kernel_size=[3, 3],\n",
    "                               strides=[1, 1, 1, 1],\n",
    "                               scope='conv12')   \n",
    "\n",
    "    conv_layer_13 = conv_layer(input=conv_layer_12,\n",
    "                               input_channels=512,\n",
    "                               output_channels=512,\n",
    "                               kernel_size=[3, 3],\n",
    "                               strides=[1, 1, 1, 1],\n",
    "                               scope='conv13') \n",
    "    \n",
    "    pool_layer_5 = tf.nn.max_pool(conv_layer_12,\n",
    "                                  ksize=[1, 2, 2, 1], \n",
    "                                  strides=[1, 2, 2, 1],\n",
    "                                  padding='SAME',\n",
    "                                  name='pool5')     \n",
    "    # ===========\n",
    "    # CLASSIFIER\n",
    "    # ===========\n",
    "    \n",
    "    fc_layer_1 = fc_layer(input=pool_layer_5, \n",
    "                          output_nodes=4096,\n",
    "                          activation=tf.nn.relu,\n",
    "                          scope='fc1')\n",
    "    \n",
    "    fc_layer_2 = fc_layer(input=fc_layer_1, \n",
    "                          output_nodes=4096,\n",
    "                          activation=tf.nn.relu,\n",
    "                          scope='fc2')\n",
    "\n",
    "    out_layer = fc_layer(input=fc_layer_2, \n",
    "                         output_nodes=n_classes,\n",
    "                         activation=None,\n",
    "                         scope='output_layer')\n",
    "    \n",
    "    # Loss and optimizer\n",
    "    loss = tf.nn.softmax_cross_entropy_with_logits(logits=out_layer, labels=tf_y)\n",
    "    cost = tf.reduce_mean(loss, name='cost')\n",
    "    \n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "    train = optimizer.minimize(cost, name='train')\n",
    "\n",
    "    # Prediction\n",
    "    correct_prediction = tf.equal(tf.argmax(tf_y, 1), tf.argmax(out_layer, 1), \n",
    "                                  name='correct_predictions')\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')\n",
    "\n",
    "    # Saver to save session for reuse\n",
    "    saver = tf.train.Saver()\n",
    "\n",
    "    \n",
    "##########################\n",
    "### TRAINING & EVALUATION\n",
    "##########################\n",
    "\n",
    "with tf.Session(graph=g) as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    for epoch in range(training_epochs):\n",
    "        \n",
    "        avg_cost = 0.\n",
    "        mbatch_cnt = 0\n",
    "        for batch_x, batch_y in cifar.load_train_epoch(shuffle=True, batch_size=batch_size):\n",
    "            \n",
    "            mbatch_cnt += 1\n",
    "            _, c = sess.run(['train', 'cost:0'], feed_dict={'features:0': batch_x,\n",
    "                                                            'targets:0': batch_y})\n",
    "            avg_cost += c\n",
    "\n",
    "            if not mbatch_cnt % print_interval:\n",
    "                print(\"Minibatch: %04d | Cost: %.3f\" % (mbatch_cnt, c))\n",
    "                \n",
    "\n",
    "        # ===================\n",
    "        # Training Accuracy\n",
    "        # ===================\n",
    "        n_predictions, n_correct = 0, 0\n",
    "        for batch_x, batch_y in cifar.load_train_epoch(batch_size=batch_size):\n",
    "        \n",
    "            p = sess.run('correct_predictions:0', \n",
    "                         feed_dict={'features:0': batch_x,\n",
    "                                    'targets:0':  batch_y})\n",
    "            n_correct += np.sum(p)\n",
    "            n_predictions += p.shape[0]\n",
    "        train_acc = n_correct / n_predictions\n",
    "        \n",
    "        \n",
    "        # ===================\n",
    "        # Validation Accuracy\n",
    "        # ===================\n",
    "        #valid_acc = sess.run('accuracy:0', feed_dict={'features:0': X_valid,\n",
    "        #                                              'targets:0': y_valid})\n",
    "        # ---------------------------------------\n",
    "        # workaround for GPUs with <= 4 Gb memory\n",
    "        n_predictions, n_correct = 0, 0\n",
    "        indices = np.arange(y_valid.shape[0])\n",
    "        chunksize = 500\n",
    "        for start_idx in range(0, indices.shape[0] - chunksize + 1, chunksize):\n",
    "            index_slice = indices[start_idx:start_idx + chunksize]\n",
    "            p = sess.run('correct_predictions:0', \n",
    "                         feed_dict={'features:0': X_valid[index_slice],\n",
    "                                    'targets:0': y_valid[index_slice]})\n",
    "            n_correct += np.sum(p)\n",
    "            n_predictions += p.shape[0]\n",
    "        valid_acc = n_correct / n_predictions\n",
    "        # ---------------------------------------\n",
    "                                                \n",
    "        print(\"Epoch: %03d | AvgCost: %.3f\" % (epoch + 1, avg_cost / mbatch_cnt), end=\"\")\n",
    "        print(\" | Train/Valid ACC: %.3f/%.3f\" % (train_acc, valid_acc))\n",
    "    \n",
    "    saver.save(sess, save_path='./convnet-vgg16.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./convnet-vgg16.ckpt\n",
      "Test ACC: 0.772\n"
     ]
    }
   ],
   "source": [
    "##########################\n",
    "### RELOAD & TEST\n",
    "##########################\n",
    "\n",
    "with tf.Session(graph=g) as sess:\n",
    "    saver.restore(sess, save_path='./convnet-vgg16.ckpt')\n",
    "    \n",
    "    # test_acc = sess.run('accuracy:0', feed_dict={'features:0': X_test,\n",
    "    #                                              'targets:0': y_test})\n",
    "    # ---------------------------------------\n",
    "    # workaround for GPUs with <= 4 Gb memory\n",
    "    n_predictions, n_correct = 0, 0\n",
    "    indices = np.arange(y_test.shape[0])\n",
    "    chunksize = 500\n",
    "    for start_idx in range(0, indices.shape[0] - chunksize + 1, chunksize):\n",
    "        index_slice = indices[start_idx:start_idx + chunksize]\n",
    "        p = sess.run('correct_predictions:0', \n",
    "                     feed_dict={'features:0': X_test[index_slice],\n",
    "                                'targets:0': y_test[index_slice]})\n",
    "        n_correct += np.sum(p)\n",
    "        n_predictions += p.shape[0]\n",
    "    test_acc = n_correct / n_predictions\n",
    "    # ---------------------------------------\n",
    "\n",
    "    print('Test ACC: %.3f' % test_acc)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}