a blog.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-15T00:35:10.135150Z",
     "start_time": "2019-01-15T00:34:52.881054Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# import Library and some random image data set\n",
    "import tensorflow as tf\n",
    "import numpy      as np\n",
    "import seaborn    as sns \n",
    "import pandas     as pd\n",
    "import os,sys\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.gridspec as gridspec\n",
    "np.random.seed(78); tf.set_random_seed(78)\n",
    "\n",
    "# get some of the STL data set\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from skimage import util \n",
    "from skimage.transform import resize\n",
    "from skimage.io import imread\n",
    "import warnings\n",
    "from numpy import inf\n",
    "\n",
    "from scipy.stats import kurtosis,skew\n",
    "\n",
    "plt.rcParams.update({'font.size': 10})\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import gc\n",
    "from IPython.display import display, clear_output\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Things to compare</h1>\n",
    "<div>\n",
    "<ol>\n",
    "  <li>Z: Baseline </li>\n",
    "  <li>A: abs(Theta)  </li>\n",
    "  <li>B: Theta ^ 2</li>\n",
    "  <li>C: sqrt(Theta ^ 2) </li>\n",
    "  <li>D: - log(1+Theta ^ 2)</li>\n",
    "  <li>E: - tanh(Theta) </li>\n",
    "  <li>F: - tanh(Theta^2)</li>\n",
    "  <li>G: - tanh(abs(Theta))</li>\n",
    "  <li>H: - tanh(abs(Theta)^2)</li>\n",
    "</ol>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-15T00:36:49.645081Z",
     "start_time": "2019-01-15T00:36:47.281937Z"
    },
    "code_folding": [
     0,
     3,
     30,
     38
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5000, 96, 96, 3) 1.0 0.0\n",
      "(5000, 10) 1.0 0.0\n",
      "(8000, 96, 96, 3) 1.0 0.0\n",
      "(8000, 10) 1.0 0.0\n"
     ]
    }
   ],
   "source": [
    "# read all of the data\n",
    "# https://github.com/mttk/STL10\n",
    "plt.rcParams.update({'font.size': 10})\n",
    "def read_all_images(path_to_data):\n",
    "    \"\"\"\n",
    "    :param path_to_data: the file containing the binary images from the STL-10 dataset\n",
    "    :return: an array containing all the images\n",
    "    \"\"\"\n",
    "\n",
    "    with open(path_to_data, 'rb') as f:\n",
    "        # read whole file in uint8 chunks\n",
    "        everything = np.fromfile(f, dtype=np.uint8)\n",
    "\n",
    "        # We force the data into 3x96x96 chunks, since the\n",
    "        # images are stored in \"column-major order\", meaning\n",
    "        # that \"the first 96*96 values are the red channel,\n",
    "        # the next 96*96 are green, and the last are blue.\"\n",
    "        # The -1 is since the size of the pictures depends\n",
    "        # on the input file, and this way numpy determines\n",
    "        # the size on its own.\n",
    "\n",
    "        images = np.reshape(everything, (-1, 3, 96, 96))\n",
    "\n",
    "        # Now transpose the images into a standard image format\n",
    "        # readable by, for example, matplotlib.imshow\n",
    "        # You might want to comment this line or reverse the shuffle\n",
    "        # if you will use a learning algorithm like CNN, since they like\n",
    "        # their channels separated.\n",
    "        images = np.transpose(images, (0, 3, 2, 1))\n",
    "        return images\n",
    "def read_labels(path_to_labels):\n",
    "    \"\"\"\n",
    "    :param path_to_labels: path to the binary file containing labels from the STL-10 dataset\n",
    "    :return: an array containing the labels\n",
    "    \"\"\"\n",
    "    with open(path_to_labels, 'rb') as f:\n",
    "        labels = np.fromfile(f, dtype=np.uint8)\n",
    "        return labels\n",
    "def show_images(data,row=1,col=1):\n",
    "    fig=plt.figure(figsize=(10,10))\n",
    "    columns = col; rows = row\n",
    "    for i in range(1, columns*rows +1):\n",
    "        fig.add_subplot(rows, columns, i)\n",
    "        plt.imshow(data[i-1])\n",
    "    plt.show()\n",
    "\n",
    "train_images = read_all_images(\"../../../DataSet/STL10/stl10_binary/train_X.bin\") / 255.0\n",
    "train_labels = read_labels    (\"../../../DataSet/STL10/stl10_binary/train_Y.bin\")\n",
    "test_images  = read_all_images(\"../../../DataSet/STL10/stl10_binary/test_X.bin\")  / 255.0\n",
    "test_labels  = read_labels    (\"../../../DataSet/STL10/stl10_binary/test_y.bin\")\n",
    "\n",
    "label_encoder= OneHotEncoder(sparse=False,categories='auto')\n",
    "train_labels = label_encoder.fit_transform(train_labels.reshape((-1,1)))\n",
    "test_labels  = label_encoder.fit_transform(test_labels.reshape((-1,1)))\n",
    "\n",
    "print(train_images.shape,train_images.max(),train_images.min())\n",
    "print(train_labels.shape,train_labels.max(),train_labels.min())\n",
    "print(test_images.shape,test_images.max(),test_images.min())\n",
    "print(test_labels.shape,test_labels.max(),test_labels.min())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-15T00:39:48.699828Z",
     "start_time": "2019-01-15T00:39:48.634774Z"
    },
    "code_folding": [
     0,
     51,
     66,
     74,
     78
    ]
   },
   "outputs": [],
   "source": [
    "# create the layers\n",
    "def tf_softmax(x): return tf.nn.softmax(x)\n",
    "\n",
    "def tf_relu(x):   return tf.nn.relu(x)\n",
    "def d_tf_relu(x): return tf.cast(tf.greater(x,0),tf.float32)\n",
    "\n",
    "class CNN():\n",
    "\n",
    "    def __init__(self,k,inc,out, stddev=0.05,which_reg=0,act=tf_relu,d_act=d_tf_relu):\n",
    "        self.w              = tf.Variable(tf.random_normal([k,k,inc,out],stddev=stddev,seed=2,dtype=tf.float32))\n",
    "        self.m,self.v       = tf.Variable(tf.zeros_like(self.w)),tf.Variable(tf.zeros_like(self.w))\n",
    "        self.act,self.d_act = act,d_act\n",
    "        self.regularizer    = which_reg\n",
    "        \n",
    "    def getw(self): return self.w\n",
    "\n",
    "    def feedforward(self,input,stride=1,padding='SAME'):\n",
    "        self.input  = input\n",
    "        self.layer  = tf.nn.conv2d(input,self.w,strides=[1,stride,stride,1],padding=padding) \n",
    "        self.layerA = self.act(self.layer)\n",
    "        return self.layer, self.layerA\n",
    "    \n",
    "    def backprop(self,gradient,stride=1,padding='SAME'):\n",
    "        grad_part_1 = gradient\n",
    "        grad_part_2 = self.d_act(self.layer)\n",
    "        grad_part_3 = self.input\n",
    "\n",
    "        grad_middle = grad_part_1 * grad_part_2\n",
    "        grad        = tf.nn.conv2d_backprop_filter(input = grad_part_3,filter_sizes = tf.shape(self.w),  out_backprop = grad_middle,strides=[1,stride,stride,1],padding=padding) / batch_size\n",
    "        grad_pass   = tf.nn.conv2d_backprop_input (input_sizes = tf.shape(self.input),filter= self.w,out_backprop = grad_middle,strides=[1,stride,stride,1],padding=padding)\n",
    "\n",
    "        if self.regularizer == 'A': grad = grad + lamda * 2.0 * self.w\n",
    "        if self.regularizer == 'B': grad = grad + lamda * 0.5 * (tf.sign(self.w) + (1.0/tf.sqrt(tf.square(tf.abs(self.w))+ 10e-8)) * tf.abs(self.w) * tf.sign(self.w))\n",
    "        if self.regularizer == 'C': grad = grad + lamda * tf.ones_like(self.w)\n",
    "        if self.regularizer == 'D': grad = grad + lamda * (1.0/tf.sqrt(tf.square(self.w)+ 10e-8)) * self.w\n",
    "        if self.regularizer == 'E': grad = grad + lamda * tf.sign(self.w)\n",
    "        if self.regularizer == 'F': grad = grad + lamda * (1.0/tf.sqrt(tf.square(tf.abs(self.w))+ 10e-8)) * tf.abs(self.w) * tf.sign(self.w)\n",
    "        if self.regularizer == 'G': grad = grad + lamda * ( self.w**2/(tf.sqrt(self.w**2)+10e-8) - tf.sqrt(self.w ** 2))/(self.w**2 + 10e-8)\n",
    "        if self.regularizer == 'H': grad = grad + lamda * -1.0 * (1.0-tf.tanh(self.w) ** 2)\n",
    "        if self.regularizer == 'I': grad = grad + lamda * -1.0 * (1.0-tf.tanh(self.w ** 2) ** 2) * 2.0 * self.w\n",
    "        if self.regularizer == 'J': grad = grad + lamda * -1.0 * (1/(1+self.w ** 2)) * 2.0 * self.w\n",
    "        \n",
    "        update_w = []\n",
    "        update_w.append(tf.assign( self.m,self.m*beta1 + (1-beta1) * (grad)   ))\n",
    "        update_w.append(tf.assign( self.v,self.v*beta2 + (1-beta2) * (grad ** 2)   ))\n",
    "        m_hat = self.m / (1-beta1) ; v_hat = self.v / (1-beta2)\n",
    "        adam_middle = m_hat * learning_rate/(tf.sqrt(v_hat) + adam_e)\n",
    "        update_w.append(tf.assign(self.w,tf.subtract(self.w,adam_middle  )))\n",
    "        \n",
    "        return grad_pass,grad,update_w\n",
    "    \n",
    "def save_to_image(data,name):\n",
    "    l1g,l2g,l3g,l4g,l5g,l6g = data\n",
    "    l1g,l2g,l3g,l4g,l5g,l6g = np.asarray(l1g),np.asarray(l2g),np.asarray(l3g),np.asarray(l4g),np.asarray(l5g),np.asarray(l6g)\n",
    "    plt.figure(figsize=(25,15))\n",
    "    plt.suptitle('Current Iter : ' + str(iter))\n",
    "    plt.subplot(231); plt.hist(l1g.ravel(),50); plt.title('layer 1')\n",
    "    plt.subplot(232); plt.hist(l2g.ravel(),50); plt.title('layer 2')\n",
    "    plt.subplot(233); plt.hist(l3g.ravel(),50); plt.title('layer 3')\n",
    "    plt.subplot(234); plt.hist(l4g.ravel(),50); plt.title('layer 4')\n",
    "    plt.subplot(235); plt.hist(l5g.ravel(),50); plt.title('layer 5')\n",
    "    plt.subplot(236); plt.hist(l6g.ravel(),50); plt.title('layer 6')\n",
    "    plt.savefig(name + str(iter)+'.png')\n",
    "    plt.tight_layout()\n",
    "    plt.close('all')     \n",
    "    \n",
    "def append_stat(current_list,data,number):\n",
    "    current_list[0].append(data[number].mean())\n",
    "    current_list[1].append(data[number].std())\n",
    "    current_list[2].append(skew    (data[number].ravel()))\n",
    "    current_list[3].append(kurtosis(data[number].ravel()))\n",
    "    current_list[4].append(np.count_nonzero(data[number]))\n",
    "    return current_list\n",
    "\n",
    "def transform_to_2d(data):\n",
    "    batch,width,height,chan = data.shape\n",
    "    return data.reshape((batch*width,height*chan))\n",
    "\n",
    "def save_to_image(main_data,one,two,three,four,five,six,experiment_name,tran_acc,test_acc,current_exp,iter):\n",
    "    plt.figure(figsize=(20,40))\n",
    "    G = gridspec.GridSpec(8, 6)\n",
    "\n",
    "    plt.figtext(0.5,1.0,\"Iter: \" + str(iter) + \" Histogram Per \" + experiment_name,ha=\"center\", va=\"top\", fontsize=35, color=\"black\")\n",
    "    plt.subplot(G[0, 0]).hist(main_data[0].ravel(),50,color='red');       plt.subplot(G[0, 0]).set_title(experiment_name+' 1')\n",
    "    plt.subplot(G[0, 1]).hist(main_data[1].ravel(),50,color='orange');    plt.subplot(G[0, 1]).set_title(experiment_name+' 2')\n",
    "    plt.subplot(G[0, 2]).hist(main_data[2].ravel(),50,color='yellow');  plt.subplot(G[0, 2]).set_title(experiment_name+' 3')\n",
    "    plt.subplot(G[0, 3]).hist(main_data[3].ravel(),50,color='green');    plt.subplot(G[0, 3]).set_title(experiment_name+' 4')\n",
    "    plt.subplot(G[0, 4]).hist(main_data[4].ravel(),50,color='blue');     plt.subplot(G[0, 4]).set_title(experiment_name+' 5')\n",
    "    plt.subplot(G[0, 5]).hist(main_data[5].ravel(),50,color='black');     plt.subplot(G[0, 5]).set_title(experiment_name+' 6')\n",
    "\n",
    "    plt.subplot(G[1, :]).set_title(\"Mean Per \"+ experiment_name)\n",
    "    plt.subplot(G[1, :]).plot(one[0]  ,c='red',alpha=0.9,label='1')\n",
    "    plt.subplot(G[1, :]).plot(two[0]  ,c='orange',alpha=0.9,label='2')\n",
    "    plt.subplot(G[1, :]).plot(three[0],c='yellow',alpha=0.9,label='3')\n",
    "    plt.subplot(G[1, :]).plot(four[0],c='green',alpha=0.9,label='4')\n",
    "    plt.subplot(G[1, :]).plot(five[0],c='blue',alpha=0.9,label='5')\n",
    "    plt.subplot(G[1, :]).plot(six[0],c='black',alpha=0.9,label='6')\n",
    "    plt.legend(bbox_to_anchor=(0., 0.95, 1., .05), loc=9,ncol=6, mode=\"expand\", borderaxespad=0.)\n",
    "\n",
    "    plt.subplot(G[2, :]).set_title(\"Standard Deviation Per \"+ experiment_name)\n",
    "    plt.subplot(G[2, :]).plot(one[1]  ,c='red',alpha=0.9,label='1')\n",
    "    plt.subplot(G[2, :]).plot(two[1]  ,c='orange',alpha=0.9,label='2')\n",
    "    plt.subplot(G[2, :]).plot(three[1],c='yellow',alpha=0.9,label='3')\n",
    "    plt.subplot(G[2, :]).plot(four[1],c='green',alpha=0.9,label='4')\n",
    "    plt.subplot(G[2, :]).plot(five[1],c='blue',alpha=0.9,label='5')\n",
    "    plt.subplot(G[2, :]).plot(six[1],c='black',alpha=0.9,label='6')\n",
    "    plt.legend(bbox_to_anchor=(0., 0.95, 1., .05), loc=9,ncol=6, mode=\"expand\", borderaxespad=0.)\n",
    "\n",
    "    plt.subplot(G[3, :]).set_title(\"Skewness Per \"+ experiment_name)\n",
    "    plt.subplot(G[3, :]).plot(one[2]  ,c='red',alpha=0.9,label='1')\n",
    "    plt.subplot(G[3, :]).plot(two[2]  ,c='orange',alpha=0.9,label='2')\n",
    "    plt.subplot(G[3, :]).plot(three[2],c='yellow',alpha=0.9,label='3')\n",
    "    plt.subplot(G[3, :]).plot(four[2],c='green',alpha=0.9,label='4')\n",
    "    plt.subplot(G[3, :]).plot(five[2],c='blue',alpha=0.9,label='5')\n",
    "    plt.subplot(G[3, :]).plot(six[2],c='black',alpha=0.9,label='6')\n",
    "    plt.legend(bbox_to_anchor=(0., 0.95, 1., .05), loc=9,ncol=6, mode=\"expand\", borderaxespad=0.)\n",
    "\n",
    "    plt.subplot(G[4, :]).set_title(\"Kurtosis Per \"+ experiment_name)\n",
    "    plt.subplot(G[4, :]).plot(one[3]  ,c='red',alpha=0.9,label='1')\n",
    "    plt.subplot(G[4, :]).plot(two[3]  ,c='orange',alpha=0.9,label='2')\n",
    "    plt.subplot(G[4, :]).plot(three[3],c='yellow',alpha=0.9,label='3')\n",
    "    plt.subplot(G[4, :]).plot(four[3],c='green',alpha=0.9,label='4')\n",
    "    plt.subplot(G[4, :]).plot(five[3],c='blue',alpha=0.9,label='5')\n",
    "    plt.subplot(G[4, :]).plot(six[3],c='black',alpha=0.9,label='6')\n",
    "    plt.legend(bbox_to_anchor=(0., 0.95, 1., .05), loc=9,ncol=6, mode=\"expand\", borderaxespad=0.)\n",
    "\n",
    "    plt.subplot(G[5, :]).set_title(\"# Non-Zero Per \"+ experiment_name)\n",
    "    plt.subplot(G[5, :]).plot(one[4]  ,c='red',alpha=0.9   ,label='1')\n",
    "    plt.subplot(G[5, :]).plot(two[4]  ,c='orange',alpha=0.9,label='2')\n",
    "    plt.subplot(G[5, :]).plot(three[4],c='yellow',alpha=0.9,label='3')\n",
    "    plt.subplot(G[5, :]).plot(four[4],c='green',alpha=0.9  ,label='4')\n",
    "    plt.subplot(G[5, :]).plot(five[4],c='blue',alpha=0.9   ,label='5')\n",
    "    plt.subplot(G[5, :]).plot(six[4],c='black',alpha=0.9   ,label='6')\n",
    "    plt.legend(bbox_to_anchor=(0., 0.95, 1., .05), loc=9,ncol=6, mode=\"expand\", borderaxespad=0.)\n",
    "\n",
    "    plt.subplot(G[6, :]).set_title(\"Train/Test accuracy\")\n",
    "    plt.subplot(G[6, :]).plot(train_acc  ,c='red',alpha=0.9, label='Train')\n",
    "    plt.subplot(G[6, :]).plot(test_acc   ,c='blue',alpha=0.9,label='Test')\n",
    "    plt.legend()\n",
    "\n",
    "    plt.figtext(0.5,0,\"Correlation Matrix Per \"+ experiment_name,ha=\"center\", va=\"bottom\", fontsize=30, color=\"black\")\n",
    "    plt.subplot(G[7, 0]).imshow(np.corrcoef(transform_to_2d(main_data[0])),cmap='gray')\n",
    "    plt.subplot(G[7, 1]).imshow(np.corrcoef(transform_to_2d(main_data[1])),cmap='gray')\n",
    "    plt.subplot(G[7, 2]).imshow(np.corrcoef(transform_to_2d(main_data[2])),cmap='gray')\n",
    "    plt.subplot(G[7, 3]).imshow(np.corrcoef(transform_to_2d(main_data[3])),cmap='gray')\n",
    "    plt.subplot(G[7, 4]).imshow(np.corrcoef(transform_to_2d(main_data[4])),cmap='gray')\n",
    "    plt.subplot(G[7, 5]).imshow(np.corrcoef(transform_to_2d(main_data[5])),cmap='gray')\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(current_exp + '/' + experiment_name + '/' + str(iter) + '.png')\n",
    "    plt.close('all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-15T01:01:44.730538Z",
     "start_time": "2019-01-15T01:01:44.725577Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# set hyper parameter\n",
    "plt.rcParams.update({'font.size': 25})\n",
    "num_eps   = 10;\n",
    "num_epoch = 15; learning_rate = 0.0008; batch_size = 20; beta1,beta2,adam_e = 0.9,0.999,1e-9; lamda = 0.00001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-15T01:19:07.658418Z",
     "start_time": "2019-01-15T01:05:55.455176Z"
    },
    "code_folding": [],
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "    ================================================\n",
      "                Starting Episode: 0\n",
      "    ================================================\n",
      "\n",
      "Current : 0\t Train Acc : 0.12\t Test Acc : 0.163\t\n",
      "Current : 2\t Train Acc : 0.215\t Test Acc : 0.22\t\n",
      "Current : 4\t Train Acc : 0.299\t Test Acc : 0.317\t\n",
      "Current : 6\t Train Acc : 0.323\t Test Acc : 0.323\t\n",
      "Current : 8\t Train Acc : 0.337\t Test Acc : 0.336\t\n",
      "Current : 10\t Train Acc : 0.348\t Test Acc : 0.346\t\n",
      "Current : 12\t Train Acc : 0.357\t Test Acc : 0.353\t\n",
      "Current : 14\t Train Acc : 0.369\t Test Acc : 0.356\t\n",
      "\n",
      "    ================================================\n",
      "                Starting Episode: 1\n",
      "    ================================================\n",
      "\n",
      "Current : 0\t Train Acc : 0.128\t Test Acc : 0.152\t\n",
      "Current : 2\t Train Acc : 0.261\t Test Acc : 0.289\t\n",
      "Current : 4\t Train Acc : 0.33\t Test Acc : 0.33\t5\n",
      "Current : 6\t Train Acc : 0.343\t Test Acc : 0.344\t\n",
      "Current : 8\t Train Acc : 0.363\t Test Acc : 0.355\t\n",
      "Current : 10\t Train Acc : 0.374\t Test Acc : 0.365\t\n",
      "Current : 12\t Train Acc : 0.385\t Test Acc : 0.371\t\n",
      "Current : 14\t Train Acc : 0.396\t Test Acc : 0.379\t\n",
      "\n",
      "    ================================================\n",
      "                Starting Episode: 2\n",
      "    ================================================\n",
      "\n",
      "Current : 0\t Train Acc : 0.115\t Test Acc : 0.142\t\n",
      "Current : 2\t Train Acc : 0.183\t Test Acc : 0.182\t\n",
      "Current : 4\t Train Acc : 0.208\t Test Acc : 0.22\t\n",
      "Current : 6\t Train Acc : 0.227\t Test Acc : 0.243\t\n",
      "Current : 8\t Train Acc : 0.252\t Test Acc : 0.266\t\n",
      "Current : 10\t Train Acc : 0.278\t Test Acc : 0.285\t\n",
      "Current : 12\t Train Acc : 0.296\t Test Acc : 0.306\t\n",
      "Current : 14\t Train Acc : 0.318\t Test Acc : 0.324\t\n",
      "\n",
      "    ================================================\n",
      "                Starting Episode: 3\n",
      "    ================================================\n",
      "\n",
      "Current : 0\t Train Acc : 0.109\t Test Acc : 0.147\t\n",
      "Current : 2\t Train Acc : 0.192\t Test Acc : 0.22\t\n",
      "Current : 4\t Train Acc : 0.245\t Test Acc : 0.26\t\n",
      "Current : 6\t Train Acc : 0.277\t Test Acc : 0.288\t\n",
      "Current : 8\t Train Acc : 0.3\t Test Acc : 0.305\t5\n",
      "Current : 10\t Train Acc : 0.311\t Test Acc : 0.321\t\n",
      "Current : 12\t Train Acc : 0.324\t Test Acc : 0.327\t\n",
      "Current : 14\t Train Acc : 0.334\t Test Acc : 0.338\t\n",
      "\n",
      "    ================================================\n",
      "                Starting Episode: 4\n",
      "    ================================================\n",
      "\n",
      "Current : 0\t Train Acc : 0.116\t Test Acc : 0.16\t\n",
      "Current : 2\t Train Acc : 0.221\t Test Acc : 0.255\t\n",
      "Current : 4\t Train Acc : 0.274\t Test Acc : 0.306\t\n",
      "Current : 6\t Train Acc : 0.303\t Test Acc : 0.317\t\n",
      "Current : 8\t Train Acc : 0.315\t Test Acc : 0.327\t\n",
      "Current : 10\t Train Acc : 0.333\t Test Acc : 0.335\t\n",
      "Current : 12\t Train Acc : 0.341\t Test Acc : 0.342\t\n",
      "Current : 14\t Train Acc : 0.348\t Test Acc : 0.345\t\n",
      "\n",
      "    ================================================\n",
      "                Starting Episode: 5\n",
      "    ================================================\n",
      "\n",
      "Current : 0\t Train Acc : 0.138\t Test Acc : 0.169\t\n",
      "Current : 2\t Train Acc : 0.188\t Test Acc : 0.155\t\n",
      "Current : 4\t Train Acc : 0.224\t Test Acc : 0.245\t\n",
      "Current : 6\t Train Acc : 0.269\t Test Acc : 0.264\t\n",
      "Current : 8\t Train Acc : 0.289\t Test Acc : 0.28\t\n",
      "Current : 10\t Train Acc : 0.296\t Test Acc : 0.291\t\n",
      "Current : 12\t Train Acc : 0.31\t Test Acc : 0.297\t\n",
      "Current : 14\t Train Acc : 0.327\t Test Acc : 0.309\t\n",
      "\n",
      "    ================================================\n",
      "                Starting Episode: 6\n",
      "    ================================================\n",
      "\n",
      "Current : 0\t Train Acc : 0.124\t Test Acc : 0.196\t\n",
      "Current : 2\t Train Acc : 0.268\t Test Acc : 0.293\t\n",
      "Current : 4\t Train Acc : 0.327\t Test Acc : 0.337\t\n",
      "Current : 6\t Train Acc : 0.359\t Test Acc : 0.352\t\n",
      "Current : 8\t Train Acc : 0.37\t Test Acc : 0.364\t\n",
      "Current : 10\t Train Acc : 0.381\t Test Acc : 0.376\t\n",
      "Current : 12\t Train Acc : 0.394\t Test Acc : 0.386\t\n",
      "Current : 14\t Train Acc : 0.401\t Test Acc : 0.397\t\n",
      "\n",
      "    ================================================\n",
      "                Starting Episode: 7\n",
      "    ================================================\n",
      "\n",
      "Current : 0\t Train Acc : 0.115\t Test Acc : 0.149\t\n",
      "Current : 2\t Train Acc : 0.228\t Test Acc : 0.247\t\n",
      "Current : 4\t Train Acc : 0.257\t Test Acc : 0.266\t\n",
      "Current : 6\t Train Acc : 0.282\t Test Acc : 0.281\t\n",
      "Current : 8\t Train Acc : 0.31\t Test Acc : 0.311\t\n",
      "Current : 10\t Train Acc : 0.333\t Test Acc : 0.327\t\n",
      "Current : 12\t Train Acc : 0.342\t Test Acc : 0.333\t\n",
      "Current : 14\t Train Acc : 0.35\t Test Acc : 0.339\t\n",
      "\n",
      "    ================================================\n",
      "                Starting Episode: 8\n",
      "    ================================================\n",
      "\n",
      "Current : 0\t Train Acc : 0.122\t Test Acc : 0.165\t\n",
      "Current : 2\t Train Acc : 0.232\t Test Acc : 0.279\t\n",
      "Current : 4\t Train Acc : 0.3\t Test Acc : 0.315\t5\n",
      "Current : 6\t Train Acc : 0.336\t Test Acc : 0.344\t\n",
      "Current : 8\t Train Acc : 0.351\t Test Acc : 0.35\t\n",
      "Current : 10\t Train Acc : 0.367\t Test Acc : 0.36\t\n",
      "Current : 12\t Train Acc : 0.382\t Test Acc : 0.363\t\n",
      "Current : 14\t Train Acc : 0.391\t Test Acc : 0.379\t\n",
      "\n",
      "    ================================================\n",
      "                Starting Episode: 9\n",
      "    ================================================\n",
      "\n",
      "Current : 0\t Train Acc : 0.125\t Test Acc : 0.166\t\n",
      "Current : 2\t Train Acc : 0.213\t Test Acc : 0.253\t\n",
      "Current : 4\t Train Acc : 0.295\t Test Acc : 0.315\t\n",
      "Current : 6\t Train Acc : 0.32\t Test Acc : 0.338\t\n",
      "Current : 8\t Train Acc : 0.34\t Test Acc : 0.354\t\n",
      "Current : 10\t Train Acc : 0.359\t Test Acc : 0.355\t\n",
      "Current : 12\t Train Acc : 0.373\t Test Acc : 0.362\t\n",
      "Current : 14\t Train Acc : 0.385\t Test Acc : 0.369\t\n"
     ]
    }
   ],
   "source": [
    "# Z\n",
    "from IPython.display import display, clear_output\n",
    "current_exp_name = 'Z';\n",
    "sess = tf.InteractiveSession()\n",
    "current_exp_train_accuracy = np.zeros((num_eps,num_epoch))\n",
    "current_exp_test_accuracy  = np.zeros((num_eps,num_epoch))\n",
    "\n",
    "for episode in range(num_eps):\n",
    "    sys.stdout.write(\"\"\"\n",
    "    ================================================\n",
    "                Starting Episode: \"\"\" + str(episode) +  \"\"\"\n",
    "    ================================================\\n\n",
    "    \"\"\");sys.stdout.flush();\n",
    "    \n",
    "    # create layers\n",
    "    l1 = CNN(3,3, 16,which_reg=current_exp_name); \n",
    "    l2 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "    l3 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "\n",
    "    l4 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "    l5 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "    l6 = CNN(3,16,10,which_reg=current_exp_name); \n",
    "\n",
    "    # 2. graph \n",
    "    x = tf.placeholder(tf.float32,(batch_size,96,96,3))\n",
    "    y = tf.placeholder(tf.float32,(batch_size,10))\n",
    "\n",
    "    layer1, layer1a = l1. feedforward(x,stride=2)\n",
    "    layer2, layer2a = l2. feedforward(layer1a,stride=2)\n",
    "    layer3, layer3a = l3. feedforward(layer2a,stride=2)\n",
    "    layer4, layer4a = l4. feedforward(layer3a,stride=2)\n",
    "    layer5, layer5a = l5. feedforward(layer4a)\n",
    "    layer6, layer6a = l6. feedforward(layer5a)\n",
    "\n",
    "    final_layer   = tf.reduce_mean(layer6a,(1,2))\n",
    "    final_softmax = tf_softmax(final_layer)\n",
    "    cost          = -tf.reduce_mean(y * tf.log(final_softmax + 1e-8))\n",
    "    correct_prediction = tf.equal(tf.argmax(final_softmax, 1), tf.argmax(y, 1))\n",
    "    accuracy           = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "    gradient = tf.tile((final_softmax-y)[:,None,None,:],[1,6,6,1])/batch_size\n",
    "    grad6p,grad6w,grad6_up = l6.backprop(gradient)\n",
    "    grad5p,grad5w,grad5_up = l5.backprop(grad6p)\n",
    "    grad4p,grad4w,grad4_up = l4.backprop(grad5p,stride=2)\n",
    "    grad3p,grad3w,grad3_up = l3.backprop(grad4p,stride=2)\n",
    "    grad2p,grad2w,grad2_up = l2.backprop(grad3p,stride=2)\n",
    "    grad1p,grad1w,grad1_up = l1.backprop(grad2p,stride=2)\n",
    "\n",
    "    gradient_update = grad6_up + grad5_up + grad4_up + grad3_up + grad2_up + grad1_up \n",
    "\n",
    "    # train\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    avg_acc_train = 0; avg_acc_test  = 0; train_acc = [];test_acc = []\n",
    "\n",
    "    for iter in range(num_epoch):\n",
    "\n",
    "        # Training Accuracy    \n",
    "        for current_batch_index in range(0,len(train_images),batch_size):\n",
    "            current_data  = train_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "            current_label = train_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "            sess_results  = sess.run([accuracy,gradient_update],feed_dict={x:current_data,y:current_label})\n",
    "            sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(train_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "            sys.stdout.flush(); avg_acc_train = avg_acc_train + sess_results[0]\n",
    "\n",
    "        # Test Accuracy    \n",
    "        for current_batch_index in range(0,len(test_images), batch_size):\n",
    "            current_data  = test_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "            current_label = test_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "            sess_results  = sess.run([accuracy],feed_dict={x:current_data,y:current_label})\n",
    "            sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(test_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "            sys.stdout.flush(); avg_acc_test = avg_acc_test + sess_results[0]   \n",
    "\n",
    "        # ======================== print reset ========================\n",
    "        train_acc.append(avg_acc_train/(len(train_images)/batch_size))\n",
    "        test_acc .append(avg_acc_test / (len(test_images)/batch_size))\n",
    "        if iter%2 == 0 :\n",
    "            sys.stdout.write(\"Current : \"+ str(iter) + \"\\t\" +\n",
    "                  \" Train Acc : \" + str(np.around(avg_acc_train/(len(train_images)/batch_size),3)) + \"\\t\" +\n",
    "                  \" Test Acc : \"  + str(np.around(avg_acc_test/(len(test_images)/batch_size),3)) + \"\\t\\n\")\n",
    "            sys.stdout.flush();\n",
    "        avg_acc_train = 0 ; avg_acc_test  = 0\n",
    "        # ======================== print reset ========================\n",
    "\n",
    "    # save to episode\n",
    "    current_exp_train_accuracy[episode,:] = train_acc\n",
    "    current_exp_train_accuracy[episode,:] = test_acc\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-15T01:25:14.056912Z",
     "start_time": "2019-01-15T01:25:13.949161Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAEKCAYAAADq59mMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3Xl4nXWd9/H3N2kTmjaWtukObVoqUFm0Q4SyDTjIqggClQojiMMg1gXnAhUdlgLyDM84wyLiyFbAUaRI8QJ8BkWwRRFEimUvlKUtS1MkXdMlzfZ7/vieMzlJk5NzJ+ec+yyf13Wd6yz3nfv82uTkk99uIQRERESiqIi7ACIiUnwUHiIiEpnCQ0REIlN4iIhIZAoPERGJTOEhIiKRKTxERCQyhYeIiESm8BARkciGRP0CM6sFLgROBaYBHcAK4B7gxhBCa7YKZ2Y/Ab6ceLo6hFCf6dfW1dWF+vqMTxcREeDZZ59tCiGM7e+8SOFhZlOBJUB94qVtQDXQkLidaWZHhRA2RCpt7+91JHDeQL++vr6epUuXDrYYIiJlxcxWZ3Jexs1WZlYJPIQHRyNwdAhhOFADzAWagVnAz6MWtpf3qgFuA9oBJYCISIGJ0ufxRWC/xONTQwiPAoQQOkMIC+lqXjrezI4aZLmuBvYA/h14eZDXEhGRLIsSHmcn7heHEJ7q5fg9wMrE47MGWiAzmw18A+9H+f5AryMiIrmTUXgkmpEOTTx9uLdzgq/t/pvE02MGUhgzqwYWAAZ8OYTQMpDriIhIbmVa85iZcu5Lac5LHptgZqMHUJ7LEu91ewhhyQC+XkRE8iDT8JiU8vi9NOelHpvU51m9MLNZwLeB9xP3IiIyQI2NMG8ezJqVm+tnOlS3NuXxtjTnpR6r7fOsHsxsCN5cNQT4xkCH+prZeSSG906ZMmUglxARKWqNjXDVVXDHHdDZCa1Zm3nXXaHMML8Y+Bjw6xDCvQO9SAjhlhBCQwihYezYfue4iIiUjGRNY/p0uP12aGnJXXBA5jWP5pTHNWnOSz3W3OdZKczsI8ClwBZgXoblERGRhM5OmDMHnnwSQsjPe2Za81iT8nhymvNSj63p86zubgKq8LkdG8xsROqNroCzlNeHZnhtEZGS09EBmzfDe+/Ba6/Bc8/BpZfCqafCLrtAVVXuy5BpeCwHOhOP901zXvLY2hDC+gyvPS1x/294baXn7czE8Skpr301w2uLiBS0TDq229th40Z45x1YvtzD4vXXYe1a2LLFaxt1dXDxxbBkCZx7LgwbltsQyajZKoSwzcz+BBwOHAf8oOc5ZmbAsYmnj2SthCIiJShdx3Zrq4dCc7Pft0SY8TZ2LNx0k9dErrrKm7JyIcrCiHfh4fEJMzsohPB0j+NzgOmJxz/N9KL9rZRrZnfis9sjraorIlKI0oXGqlUeGNno6J4wwUMkV6KMtroLeBGf/b0ouX6VmVWY2Rzg1sR5D4cQHkv9QjObb2YhcasffLFFRIrT5z4HN9/c+2iodetyO0IqmzIOjxBCO/AZYBXeMf6omW0FtgL3Ah8CltHVRyEiUjKiTLoLAXbs8H6KtWth5Urvq1i2DC65BE45BaqrYWgRD/2JtJ9HCGGVme0PXAScgnd2t+Er3/6CLG8GJSISt3TNTMmQaGmB7du73/c1ZDbZsX3uuXDbbfDQQ37dtrb8/HuyxUK+BgXnWUNDQ9BmUCIyUOlC4803PSDShUSmmpo8RF54Ae6+e3DXAhg1yicKDpSZPRtCaOjvvMjb0IqIlLIdO7z2cMop8PTTvYfDhkHvldolWRMpNgoPESkpyRrDU095H0Nf2tq6mplSb52JGW3z5xd3s1KuKTxEpCT01czU0dF7SLS3p79eqfRN5IrCQ0SKWrq+iRdfHPzQ154h8sILg7teqVB4iEhsMm1iSpWsSSRrE6ecAkuX9t43kc05E8XaN5ErCg8RybtM9pxI9kmkBkVLy87NRlddpWalOCg8RCRv0oXG++93D4mOjsyuqb6JeCg8RCQvWlvhtNO8iaq3JqZ33x3c9dU3kV8KDxHpV5S+idTRTdu2dY1u6uiAyy7Lfe1AfRP5ofAQkT5lsjRHzyGwO3b0fT01MZUOhYeI7KS/ZcOT/RLJCXVRqYmp+Ck8RGQnn/ucbyLUWzisW5e991ETU/GKsp+HiJSo1lYPhZUrvRZQKsuGS+6o5iFShtrbfce65K3nNqfqm5D+KDxESkhfo6I6O7v2xN682UdBZUJ9E9IXhYdICeitg3vrVg+K5mYPjsHsO6G+CelJ4SFSxNKNinr11fjKJaVP4SFSZNrbu+ZUnHwyPPPM4HezE4lK4SESg0xmbIfgHdmps7S3b+/eaX3llerQlngoPETyqK9mptbWnWdqZ7I/tkZFSVwUHiJ5kK5v4rnnMl9Bti8aFSX5pvAQyaHkENlTToGnn+69JjHY4EilUVGSLwoPkSxqb/ewSN62bfPAmD9fzUpSWrQ8iUg/Ghth3jyYNWvnY21tsGEDvP02vPIKPP88vPmmb2y0dWtXTSNZI3jgATjpJC37IcVPNQ+RPvTWT7FjR9dM7S1b0i8/3hv1TUi2hdC1Ltlbb8GaNXD++XDYYbl9X4WHSA/pOrdfeik776G+CYkqBK/RJkMi9b65ueu82lr45CcVHiJ5FQLMmePLkWvincSho8P/gOkZEqtWdV+TbNQomDYNjjnG76dP99uMGbDHHrkvp8JDyl4I3j+xYYPfLr1UnduSe+3tvm97z5BYvbp7c2hdnYfCiSd2hcS0aR4evTHLT/kVHlK2tmzpCozUgNDEO8mmEGDtWh9I8cYbfv/mmx4WqT9TEyZ4MHz8491DorY2vrKno/CQstJXYPRGndsS1YYNO4fEm296zTZp/HhvVjroIL+fNg3q62H48NiKPSAKDykZfa0XFSUweqPObelp2zZvYuoZEqlb9I4c6eFwwgld/RB77FG4NYmoFB5S9HobHTXYwBAJATZu9J+vt9/uqlG89Ra8917Xebvs4k1Mhxzi4TBjht/GjMlf/0McFB5StNINqX3ttfjKJcWhowOamvznaO1anx+RfJy8T92et7ISpk6Fj3zEO6+TITFpElSU4XRrhYcUpY4OH1L71FMeHCI9tbX5vIjGxq5bMiTWrvVbz3XFdt0VJk70fohDDvHHEyfCbrvBlClQVRXPv6UQKTyk4HV0eBtz6q2lRUNqxX8O3n7b50CsWuWPkzWIpqbuc3XMYOxYH9W0775w9NH+eNIkD4gJE2DYsLj+JcVH4SF5l24jpPb2nYOiryVANKS2PCT7Hlat8uGtyaBYvdqDIhkQZh4CkybB7NkeBsnnEyb4KCetJ5Y9Cg/Jm976KDZt6h4Uqf0WmdKQ2tLQ0eFhkAyHZFisXu0/J0nV1d73sO++8OlP+zDX+nrYfXfvvJb8UHhIzqXr2H7jjey9j4bUFof29q5Jcqk1ibff7l5rHD3aQ+Goo/w+OR9i/Pjy7KAuNAoPyakQ4LTTvIlKa0WVn44OD4jly33J+ldegddf7/oDoqLCO6OnTvUO6mRATJ3q8ySkcCk8JCdaW73DsqkJLrtMfRLloLPTaw+pQfHaa13DXYcPh733htNP9/sZM7ypSSOYipPCQ7ImBNi8GT74oHsbtTq2S08I3j+RDInly/2WXIajutoD4rOfhZkzfW7ElClqbiolCg8ZtPZ2r2F88EH6Dm91bBenEOBvf/NwePnlrqBI/oEwdCh8+MO+DEcyKOrrYYh+u5Q0fXtlwLZs8cDYsCFaf4Y6tgtXcsOhV1/tui1f3rVmU2WlL8HxD//QFRR77KEhsOVI4SFp9ZyT0dEB69d7aGzfHnfpZDBC8O/v8uXdw2LDBj9eUeEd2LNne0jss4/3U2g4rIDCQ/rQ2/Da1as9OLQcSPEJwTceStYkXn3VO7OTTU/JGsXhh3tfxcyZ3hSloJC+KDykm3RzMpqa4iuXZC456im12em117yZEbyJacYMb3rae++ukU/V1fGWW4qLwkO6mTsXnnhCtYtisnEjLF0Kzz/fVaNI7nVdVeU1iOOO6woK9VFINig8pJuFC+F734O779Zw2kLV0uJB8Ze/+O3VV71ZapddYM89fcmOmTM9KKZN06gnyQ39WEk3NTXwta/5RC7NySgMHR0eEH/5CzzzDDz3nDcnDhkC++8PX/4yHHigd2orKCRf9KMm/6ulxXdJC0FzMuIUArzzjofF0097k1Rzsx/78Id9H5MDD4RZszzsReKg8BDAaxavv77z5jiak5Ef69Z5reLppz003n/fX58wwTu2DzwQPv5xXyxQpBAoPITOTl/ldCDLocvAbN3q82aS/RbJ1YU/9CFoaIBzzoGDDvJFA0t5H2wpXgoPYdWqrjWJJDc6O73f4skn4c9/hhdf9FpeVRV87GPez3TggbDXXj7nQqTQKTzK3Hvvdc0oluxqavKgSN42bvTX994bvvAFD4v999dEPClOCo8y1tQEa9fGXYrS0dbmAwueespvr73mr48eDQcf7PtVHHSQ+i2kNCg8ylRzs89ClsF5772usHjmGZ+cV1kJH/0ozJvngbHnnlqKXEqPwqMMtbR4B7l29ouupcWHziYDIxnAEyf6LO6DD/ZRUSNGxFtOkVxTeJSZvobkSu9C8LkvTz3lnd3JCXrV1XDAAT7n4uCDfdtUjYqSOJ1xhvehXXghTJ+e+/eLHB5mVgtcCJwKTAM6gBXAPcCNIYTIAz7N7AjgGKABmA7UASOADcDLwK+AW0MIWgR8EDQkNzM7dvjw2ccf98D429/89enTfT/2Qw7xEVLq6JZCsmKF7xf/0EPwpS/BpZd6jThXLERouzCzqcASoD7x0jagEkiux7kMOCqEEGn8jpn9GvhUykvJgaPDU15bCRwXQliRyTUbGhrC0qVLoxSj5L31lkZW9WXzZl8QcskSr2Vs3+57bh90kIfF7Nk+YU+kUDU0dD2uqvJ+tnPOiR4iZvZsCKGhv/MyrnmYWSXwEB4cjcBZIYRHzawCmAPcCswCfg6ckHlRAXgU+C3wBPBGCKE58Z5jgDOA/4vXcn5lZvuFELTma0QakruztWu9dvH44/Dss96UV1fn26keeaQ3S1VVxV1KKTXJ5qVzz/Wft4GorIRhw/y2yy5+nyrZunDzzb518OOPD67MvYnSbPVFYL/E41NDCE8BJH6RL0yEyN3A8WZ2VAjhsUwvHEK4vo/X1wE3mtkO4GbgI8DBwJ8ilLvsaUiuC8Gb7ZYs8Q/T8uX+en29z7s48khfXFAjoySXUpuXTjwxfYhUVnaFQ2pQ9LekflWVf22y5pELUcLj7MT94mRw9HAPcDVeQzgLyDg8MvDnlMe7ZfG6Ja/ch+R2dPhs7iVL/Pbuu/76/vvD178ORxzh4SGST8lVqh94wEPkM5/xVQamTOkeFFFrvj1DI5dNrRmFh5nVAIcmnj7c2zkhhGBmvwG+gnd+Z9PhKY/fzPK1S1a5DsltafE5F0uWwB/+4M11Q4f6ENqzzoK///uBNxeIZFMyRBYt8sUwB9O89LGPef9crkMjKdOax0wgWZl/Kc15yWMTzGx0CGH9QAtmZsPwWsYc4LLEy38IIagXPAPlNiR306auDu8//7mrw/uww7w56uCDNfdC4mHmS+fX1votVTabl5YtG9zXR5VpeExKefxemvNSj00CIoWHmU3AO+N78xDe7yL9KJchuZs2weLF8MgjXR3eY8fCpz7V1eGt7VYlDsOHd4XFiBE796Pls3kpVzINj9S83JbmvNRjtX2e1bcOILGTASOB5Ej6XwKX9VeTMbPzgPMApkyZMoC3Lw2lvEruli1etX/kEa9hdHTA7rurw1uyL8qoqNSaxYgR6VdGznfzUq4U1AzzEMIHwAQAMzNgMnA+PinxZDP7WgjhljRffwtwC/g8j9yXuPCU4pDc7dvhj3+E3/0O/vQnr1FNmOAf7mOO8VVqNbtbsi3dqKhhw7qHRZTtf/PdvJQrmf6Tm1Mep9v4MvVYc59nZSD47MV3gUvM7K/AIuC/zOzpEMLzg7l2qSqlIbk7dvhkvd/+1oOjpcU/uKeeCkcfDfvtp8CQ3Os5KurMM2H+fK/tlrtMw2NNyuPJQF87Wk/u42sGJYRwv5mtBqYC/wR8I1vXLhWvvw7f+54vCX733XGXZmDa2nwb1t/9zju+t26FXXf1PoxjjvHqvjZKkjgkQ+TOO33Xx1xMuis2mYbHcqATH3G1L30M100cA1g7mJFWfViDh8eMLF+3qDU2wuWXw09/6h3lyR/yYtHeDn/9q/dhLF7sneC1tXDUUR4YDQ3RmgREBqOyEkaNgjFjur+ej0l3xSajj2UIYZuZ/Qmfb3Ec8IOe5yT6KI5NPH0kayXsuva0xNNBNYeVisZGuOoquOMO7zQuptDo7ITnn/fAeOwxWL/eOxyPOMIDY/ZsjZKS/DGDkSN9k66RI7sPuCiFUVG5EuVvurvw8PiEmR0UQni6x/E5+Iq4AD/N9KJmNiSE0N7PaeeQ6EjHF2Yse3Pn+ryGziJa5euNN+DBB+HRR32l2upqOPxwD4xDDtEqtZJfI0Z4YIwa1XvttlRGReVK1PC4AF/fapGZnR1CeCyxptWp+MKIAA/3XNfKzOYDlyeeTgshrEo5fJiZXZn4+sUhhHdTvu7DeB/HhYmX3gTujFDmkvWLX8BFF8H99xd2c1VHh3d4L1zos76HDvUP5De+4TO9a9INvxDJsupqb5IaPdofp1Mqo6JyJePwCCG0m9lngMX4yrqPmtk2vB8k+TfjMuDMAZTj8MQNM2sBtuDLsaeuFfk8cLL29PDlRrZu9U1fzj4bbrvNR4IUUog0N/sIlXvvhTVrYPx4X0vqpJO8E1wkW/qbjzFkiIfF6NE+eU+yI1JXZAhhlZntD1wEnIL3Q7ThGzb9goFtBvUsvpDikcABePPUGGAHXtNIDtO9L4RQJott9C0EH3u+ebM/r6uDiy/2D85tt/loqzitXOm1jF//2ofX/t3fwQUXeH+GOr4lG8uR99TbfIxx47z/YswY+NCHNKw7FyJtBlVMSnUzqFWrYN26uEvRXWenT9675x4faltV5ft5n3467LVX3KWTQtLQ4E2XFRX9L0ce5ZpJQ4d6B/fZZ/soxFzupFeqsr4ZlMTvnXcKKzi2bPEO8Hvv9aXOx42DefPglFPUNCV96znx7sQT4Z//2TulhwzxX/7J+9THfb3W89ptbXDrrb5fi+Zj5I7Co0isWdO1l3bcVq3ywPj1r2HbNvjoR+GrX4VPfEJNU5K5ZIjcf7//bGfjF73mY+SPPupF4P33fV5HnDo7fbmQhQvhySe9eeDYY71paubMeMsmhc/Mh8SmyuYves3HyD+FR4Fraura/S4OW7d6DWPhQt+RsK4Ozj8fPvvZnWfhivRUWenL5I8b1zXxM9u/6DUfIx4KjwK2YQOsXh3Pe7/zjjdNPfigB8i++8L3v+/Lhmj2t/SnutqHZ48Z033Gdi5+0Ws+RjwUHgVq82YffhiHJ5+Ef/kXb2o4+mhvmtp33/6/TmTECA+NkSN7Hx6rX/SlQ+FRgLZsiW/v8Q0b4IorYOpU+PGPtde39C/ZnzFunCbhlROFR4HZts3XgIpjzaoQ4OqrvdZz440KDkmvstJ/RsaN834MKS8KjwLS0uL7cnTENI/+wQd9H40LLoA994ynDFIY0s0Er672wBgzRvurlDOFR4FobfVlFtr7W184R959F/7zP+GAA3y3NClvvS35UV+fvj9DyktF/6dIrrW1+Yc1rkUN29vhsst8VMwVV3QfHSPlq63NtwN+4AE4+WS44QbfT17BIaDwiF17uzdV7dgRXxnuvNMXVLz4Yo2TL3fV1b76bKq2Nm9Svflm30dGBNRsFauODu8c3x7jIvMvv+zrAB17rC9mKOVjyBAfHZW81dT0vryMlvyQ3ig8YtLZ6cNxt26Nrwzbt/svg7o6+M534iuHDFymS5ybeTikhkV/myFpyQ9JR+ERg+SeHM0x78Z+/fU+k/y//sv3PJDi01vHdl2db+mbWqOoqYnWV6ElP6Q/6vPIs8ZG+MIX4IQT4i3HE0/AokU+sqqh35X7pZAlO7YffNA7tm+91fst6ut9Xanhw6N3ci9bBjfdpOCQvqnmkSeNjXDVVbBgQfzbxa5fD1deCTNm+P4bUnyGDNm5ttia2MNTe1lIPig8ciwZGnfc4aHRGnWT3iwLwRc4bG725Uc0M7h4DBvmcyxGjuy9NqGObcknhUeOzZ3rTURxLDfSmwcegD/8wRc+nDEj7tJIOhUVUFvbFRh9Bb06tiUOCo8cW7iwq7mqoyPe5qp33vFZ5AceCJ//fHzlKEeZjoqqquoKi9ra/idsqmNb4mIhjqVb86ChoSEsXbo07mIAHhq//z3ccouPikn2eeSzeO3t/otr9Wq45x5fZkLyp6HB90GpqOg+KsrMm6CSgTFsWNwllXJnZs+GEPodRqPRVnnQ1OSjXy6+2JuNTjop/wsPLlgAL70E3/2ugiMuqct9nHSSj2YaNw722strDQoOKSZqtsqDDz7oelxX5yGSTy+9BLffDscfD8cck9/3lp0lmy7vussnimpUlBQj1TxybPPmeNet2rbN28PHjoVvfzu+cpSznv0WVVVeyzj/fO8TEylGqnnk2N/+Fu/7X3edL7f+k594B6zkj5nveTFpkj/XqCgpJQqPHNqxAzZtiu/9H38cfvUrOOss36dD8mfUKA+NXXbx5xoVJaVG4ZFDTU3xvfe6dT4ZcM89vXlE8qO2FiZP3nkv72XL4imPSK4oPHKkszO+8AjB55Zs3erNVZpFnns1NR4aWmBSyoXCI0c2bIhvS9n77/dZ7RdeCHvsEU8ZykV1tTdP9dxASaTUKTxyJHV4bj6tXu2d5AcdBKefHk8ZysHQoTBxYtdEP5Fyo/DIga1b49nkqb3dO2SrquDyy7UXeS5UVvoky/Hj9f8r5U3hkQNx1Tpuvx1eeQWuucZnLsvA9LYOlZn/n06Y0PtWrSLlRn87ZVl7u++XkW8vvODh8alPwSc/mf/3LyUrVnQtIXLNNb422b77wm67KThEkhQeWdbU5KOd8mnrVrjsMv+r+Fvfyu97l6rU3fmOOAK++U3fm0VEnP6OyqIQ4mmyuvZaeO89X7V3xIj8v38p6fn/l9y86+ab4eWXtQ6VSJLCI4s2b87/ToFLlngTyznnwKxZ+X3vUlFV5UNtx4zpmhGeeky784nsTOGRRflex6qpyWeR7703nHdeft+72FVUwK67emDU1va/pauWFBHpTuGRJTt2eM0jXzo74YorYPt2n00+dGj+3ruYjRjhgTFqlIdDb7QOlUj/FB5Zku++jnvvhaeegu98B6ZNy+97F5uqKg+MMWN8Rnh/tA6VSP8UHlmQ73Ws3ngDfvhDOOwwOO20/L1voUm3L3iyWaquTkvRi+SCwiML1q/3uQD50NrqzSkjRvjw3HJeGmPFCli50veFT+4LXl/ff7OUiAye5nlkQT6brG66CV5/3YNDi/F1n49x8slwww3+moJDJLcUHoO0datv9ZoPf/4z/PznMGeON1lJl9ZWaGnx+Rhz58ZdGpHSp/AYpHwNz9240UdXTZsGF1yQn/csVDU1MH1699e0L7hIfqnPYxDa233fjlwLAa6+2t/r+ut3nshWLmprfehs6oZLmo8hEg+FxyDkax2rBx+ExYvhG9+AvfbK/fsVmlGjfAn0nlu7aj6GSHwUHgOUr3Ws3n4b/uM/oKEB/vEfc/9+hcLMR02NH993TUvzMUTio/AYoE2bcr+OVXJzp6FDYf788th8qKICxo710NCseZHCpfAYoHzUOm67zVdyveaa0m+WGTLEA2PsWA2zFSkGCo8BaGnJ/TpWzz0HCxbApz9d/Js7pZsJXl3toTFmTHnUrERKhcJjAHJd69iyxScBTpwIF12U2/fKh95mgu++u9emRo0q71nyIsVKf+tF1NkJ69bl9j3+/d/h/fd9tdxS2dwpORP8gQd8JviPfuTPFRwixUnhEdG6dbldx+q3v4X/+R/40pe8qafUtLVpJrhIKVB4RJTLJqu1a+Hf/g322w/+6Z9y9z751HPElGaCi5QG9XlEsGWLb76UCx0d3s/R2enNVUOK/DtTUeF9GuPH+3PNBBcpLUX+Kyq/crmO1X//N/z1r3D55bDbbrl7n3yoq4NJk7pqHZoJLlJ6FB4ZamvzxQlzYfly+MlP4KijfGhusRo5EiZP9mapVJoJLlJ6FB4ZytU6Vi0tcMklvjfH975XnKOPamq8tqQd+0TKh8IjA7lcx+q663z9qh//2P9yLyZVVV7T0KZUIuUn8mgrM6s1s/lm9qKZbTGzTWb2jJldaGZVAymEmU02s3lm9ksze8PMtiduK83sF2b2DwO5brZs3OjNVtn2hz/AokW+4OHHP5796+dKZaWHxj77KDhEylWkmoeZTQWWAPWJl7YB1UBD4nammR0VQsh4lwsz2x1YDaQ22GxLPK9P3Oaa2QLgvBBCnnYL75KLWkdTk4+q2nNP+MpXsn/9XDDztacmTiz+0WAiMjgZ1zzMrBJ4CP9l3ggcHUIYDtQAc4FmYBbw84hlqMSD4jHgbGBy4rojgH2ABxLnfQmYH/Hag7Z9OzQ3Z/eaIcCVV/r2td//vjf/FIozzvCFGJuaur++665e09h9dwWHiERrtvoisF/i8akhhEcBQgidIYSFwJcTx443s6MiXHcDcEAI4ZMhhJ+GENakXPcV4LPAbxLnftPM8rqPXi5qHffeC08+6dvJ9txONW4rVvgSIied5CGydatvQLXHHr6IoYgIRAuPsxP3i0MIT/Vy/B5gZeLxWZleNISwKYTw1zTHA7Ag8XQEMDPTaw9WR0f217F66y344Q/h0ENhzpzsXjtbkutQPfggHHssfPvb0NgYd6lEpJBkFB5mVgMcmnj6cG/nJH7JJ2sIxwy+aN20pDzO224P69f7jO9saW31Ybk1NT6bvNCH5ba2ah0qEeldpjWPmSnnvpTmvOSxCWaWzXE4RybuW4EVWbxuWtmeUf7jH3uz0GWX+f4VhU7rUIlIXzLt+pyU8vi9NOelHpsErI9coh7MbBpwfuLpwhD3+RR5AAALzklEQVRCjrdhcs3N/ld3tvzlL/Czn8Fpp8Hhh2fvurmgdahEpD+Zhkfq3OFtac5LPTbo+cZmNgz4JT6iax3w3X7OPw84D2DKlCmDeu+XXoJrr4UXXoC77x7Updi40desqq+Hb35zcNfKtX32gSOOUGiISHoFO+jSzIYAdwMHAG3AGSGEdLUeQgi3ALcANDQ0DGgxkcZGmD8f7rrL+zsGOzkwBLjiCg+Q66+HXfI6Viya6mp48cXC74sRkfhlGh6pMx1q0pyXemzAsyMSc0p+BpwMtOPB8chAr5eJxkaftHfHHR4ara3Zue6998If/+jbye61V3aumSu77abgEJHMZBoea1IeTwZe6OO8yX18TcZSguN0oAP4xxDCfQO5VhRz58ITT2R3dNWKFXDDDXDYYXD66dm7bi7U1vpEQBGRTGQ62mo5kPy1um+a85LH1oYQIneWJ4Lj5/iM9WRw5GWcz8KFPqpo2LCdd78biO3bfZXc2lrv7yj0v+iLfQ8REcmvjMIjhLAN+FPi6XG9nWNmBhybeBq5iSklOFJrHPdEvc5ATZgAN93kk/g+9zlv/x9MiFx7Laxe7U1ho0Zlr5y5UFfnc09ERDIVZYb5XYn7T5jZQb0cnwMkF9v4aZRCJILjbjw42oEz8xkcqSZM8A7z5BIde+4Z/RqPPgq/+hWcfTYceGDWi5hVFRW+65+ISBRRw+NFfBHDRcn1q8yswszmALcmzns4hPBY6hcmlnAPiVt9j2OVwH8Dn6Orczz2KWl1dXDxxdGH6TY2+mKH++zjzWCFbuLE7DTTiUh5yXiobgih3cw+AyzGV9Z91My24QGUHIC6DDgzYhkOBT6ffBvgRjO7Mc35FxRCuPSmvR3+9V99eO7VVxf+6rNVVTBuXNylEJFiFOnXWwhhlZntD1wEnAJMw+dgvAz8ArgxhBB1kGtq7WcoML6f84f1czw2t93mkwq///3i6IDebTdvthIRiSry38YhhGbg8sQt06+ZTx97cYQQltB9I6iitHQp3H47nHgiHNfrkILCMmJE4Xfki0jh0t+dWbBxoy92uPvu8K1vxV2azBRDzUhECleBt8oXvhB8OO769XDnncUx5HX0aBg+PO5SiEgxU81jkO67Dx5/HL7+ddh777hL07+KCpg8uf/zRETSUXgMwhtvwHXXwSGHwOc/3//5hWD8+MLaM11EipPCY4BaWrovP1IMo5aGDtUy6yKSHerzGKDrrvOlTH70o+LYFRC8uaoYQk5ECp9+lQzA738PixbBF74As2fHXZrMDB9ePCEnIoVP4RHR2rU+uuojH4F58+IuTeY0NFdEsknhEUF7u2/P2tHhy48Uy5pQo0b5pEARkWxReESwYAEsW+YLJu6+e9ylyYyZah0ikn0KjwwtW+ZrV51wgt+KhYbmikguKDwysGkTXHKJj1b6znfiLk3mNDRXRHJFQ3X7EYKvktvUBHfcUVzLekyaBJWVcZdCREqRah79uP9+WLwYvvY1H2FVLGpqNDRXRHJH4ZHGm2/6XuSzZ8OZUbe4itluu3lnuYhILig8+pBcfmT4cN/TvJhmZu+6qy+bIiKSK+rz6MMNN3jN44YbfD/zYqGhuSKSD0X093T+PPoo/PKX3lR16KFxlyaaceOgujruUohIqVN49PDuu/Dd7/reHF/9atyliWbIEJg4Me5SiEg5UHj0sGABtLb68iPZmFx3xhlwzTU+1DfXNDRXRPJFfR49XHqpj67K1jDXFStg5Up46CE48UQ499zc9KEMG1ZcfTMiUtxU8+jBDKZPz+4129pgxw544AE46aTs1ER61mg0NFdE8knhkUfJEFm0yGs4w4b5EiID+aW/YkVXGF17LWzdmv3yioj0Rc1WeVRV5X0S55zj4ZG67lRHhy/5nrzv65Y8Dh5GAPfd5zPhk9dVp7mI5JrCI0eqq70DG9KHRlJl5cA7u1tb/f7mm+Hll+Hxxwd2HRGRTCk8smzoUA+NMWO8OepjH4NDDuk7NLKhZziJiOSawiNLhgzxcBg7tvtSJsuW5e49M6nRiIjkgsJjkCoqfMOl8ePzO8ciHzUaEZG+KDwGyMyXApkwwWsd+ZbLGo2ISH8UHhGZeX/GxIna3lVEypfCI4LRo70zXAsPiki50yTBhFmzYN48aGzc+djIkTBzJkybpuAQEQGFx/967jm4/XZfmuTyy33ZjxEjYK+9YMYM39ZVREScwiNFa6vvIHjffXDyyb4RVHNz3KUSESk8Co9eJEPk5pth7ty4SyMiUngUHr2oqvJFC88/HxYujLs0IiKFR6OtUmjGtohIZhQeCZqxLSKSOYVHgmZsi4hkTn0eIiISmcJDREQiU3iIiEhkCg8REYlM4SEiIpFZCCHuMuSEmX0ArB7gl9cBTVksjuSevmfFR9+zwjQ1hDC2v5NKNjwGw8yWhhAa4i6HZE7fs+Kj71lxU7OViIhEpvAQEZHIFB69uyXuAkhk+p4VH33Pipj6PEREJDLVPEREJDKFh4iIRKbwSDCzWjObb2YvmtkWM9tkZs+Y2YVmVhV3+aSLmX3RzEIGt0/GXdZyYWY1Zna8mV1iZveb2eqU78P8DK8x3sz+08xeM7PtZrbezP5oZueameX4nyARaUl2wMymAkuA+sRL24BqoCFxO9PMjgohbIilgNKXTuCDNMd35KsgwoHA/wz0i83sAOC3wJjES1uAWuCwxG2OmX0mhKDvaYEo+5qHmVUCD+HB0QgcHUIYDtQAc4FmYBbw87jKKH16J4QwIc3tj3EXsMxsAB4DfgB8HlibyReZ2Ujg13hwvAp8PIRQCwwHvga0AccA1+WgzDJAqnnAF4H9Eo9PDSE8BRBC6AQWmlkFcDdwfKL28Vg8xRQpaH8MIYxOfcHMrsnway8CJgDbgRNCCCsBQgitwE1m9iHg/wDnmdn1IYQVWSy3DFDZ1zyAsxP3i5PB0cM9wMrE47PyUySR4hJC6BjElyc/V/ckg6OHG/FmrErgzEG8j2RRWYeHmdUAhyaePtzbOcEnwvwm8fSYfJRLpFyY2V7AlMTTvj6DW4BkE6Q+gwWirMMDmEnX/8FLac5LHptgZqPTnCf5NdbMnk2MjttuZm+Z2c/M7Mi4CyYZ2zflcSafwY/ksCwSQbmHx6SUx++lOS/12KQ+z5J8qwH+DmjFf5an4c0ai81sgZmpT6/wRf0MfsjMRuSwPJKhcg+P2pTH29Kcl3qsts+zJF/WAFcAHwV2SXTUJpsgH02ccw4anVMM9BksUuUeHlKEQgiPhBDmhxBeSI77DyF0hBCeBI4FHkicOs/MPhxbQUVKWLmHR3PK45o056Uea+7zLIldYoj1RYmnFcCJMRZH+qfPYJEq9/BYk/J4cprzUo+t6fMsKQghhDfo2t50epxlkX5F/QxuToy+kpiVe3gsx5e4gO6jPnpKHlsbQlif2yKJlJXUEVaZfAZfyWFZJIKyDo8QwjbgT4mnx/V2TmJBtmMTTx/JR7lkcMxsD6Au8bS3SWdSIEIIrwFvJ5729RkcDhyeeKrPYIEo6/BIuCtx/wkzO6iX43Poavr4aX6KJH3pb3XVxPEfJJ524msmSWFLfq7mmll9L8e/CowAOtAacwVD4eHh8SJgwCIzOwrAzCrMbA5wa+K8h7WuVUGYamZ/MbMvm9n0ZJgkvl+z8VnKn02ce3PiL1vJAzMbZWZ1yRtdv19qUl/vZZ7Gf+CLKNYA/y+xwi5mVmVmXwGuSpx3i9a1KhzahhZI/LWzmO5LslcAuySeLwO0JHsBSHyvUpuiduCjb2rxZfST7gDOCyG0561wZc7MVgFTMzj1rhDCF3t8bc8l2Zvxz9/QxPNHAC3JXkBU8wBCCKuA/YEr8Q68gC8D/Sw+7HO2gqNgvA98HV/p+BVgM7Ar/v16FVgAHBZC+JKCo3iEEJ4F9sEndr6Oh8ZW4Angn4HjFRyFRTUPERGJTDUPERGJTOEhIiKRKTxERCQyhYeIiESm8BARkcgUHiIiEpnCQ0REIlN4iIhIZAoPERGJTOEhIiKRKTxERCSy/w857CCT91CksQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "current_exp_train_accuracy.max(0).shape\n",
    "current_exp_train_accuracy.min(0).shape\n",
    "\n",
    "plt.plot(current_exp_train_accuracy.max(0),  '<' ,color='blue')\n",
    "plt.plot(current_exp_train_accuracy.mean(0), '-' ,color='blue')\n",
    "plt.plot(current_exp_train_accuracy.min(0) , '>' ,color='blue')\n",
    "plt.fill_between(range(num_epoch),current_exp_train_accuracy.max(0),current_exp_train_accuracy.min(0),facecolor='blue', alpha=0.2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-05T02:14:30.085275Z",
     "start_time": "2019-01-05T01:28:38.463294Z"
    },
    "code_folding": [],
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current : 0 Train Acc : 0.11660000236332417 Test Acc : 0.159750002855435\n",
      "\n",
      "Current : 1 Train Acc : 0.16240000292658807 Test Acc : 0.20162500322796403\n",
      "\n",
      "Current : 2 Train Acc : 0.232800002977252 Test Acc : 0.27775000233203173\n",
      "\n",
      "Current : 3 Train Acc : 0.2702000027447939 Test Acc : 0.29712500181980434\n",
      "\n",
      "Current : 4 Train Acc : 0.2896000024676323 Test Acc : 0.3038750021997839\n",
      "\n",
      "Current : 5 Train Acc : 0.30340000185370447 Test Acc : 0.31387500192038714\n",
      "\n",
      "Current : 6 Train Acc : 0.3138000020235777 Test Acc : 0.3185000020544976\n",
      "\n",
      "Current : 7 Train Acc : 0.320400001257658 Test Acc : 0.3266250020265579\n",
      "\n",
      "Current : 8 Train Acc : 0.3264000009596348 Test Acc : 0.3303750018402934\n",
      "\n",
      "Current : 9 Train Acc : 0.3310000019520521 Test Acc : 0.334375001937151\n",
      "\n",
      "Current : 10 Train Acc : 0.3350000016838312 Test Acc : 0.33825000187382104\n",
      "\n",
      "Current : 11 Train Acc : 0.33620000129938127 Test Acc : 0.34137500166893003\n",
      "\n",
      "Current : 12 Train Acc : 0.34300000116229057 Test Acc : 0.34500000135041775\n",
      "\n",
      "Current : 13 Train Acc : 0.3468000015169382 Test Acc : 0.3485000017378479\n",
      "\n",
      "Current : 14 Train Acc : 0.34820000137388707 Test Acc : 0.34962500142864883\n",
      "\n",
      "Current : 15 Train Acc : 0.35120000164210796 Test Acc : 0.35387500139884653\n",
      "\n",
      "Current : 16 Train Acc : 0.35820000107586386 Test Acc : 0.3567500016931444\n",
      "\n",
      "Current : 17 Train Acc : 0.36120000110566614 Test Acc : 0.3571250015962869\n",
      "\n",
      "Current : 18 Train Acc : 0.364800000667572 Test Acc : 0.3612500016950071\n",
      "\n",
      "Current : 19 Train Acc : 0.37140000075101853 Test Acc : 0.36200000158511103\n",
      "\n",
      "Current : 20 Train Acc : 0.37540000024437903 Test Acc : 0.3633750007394701\n",
      "\n",
      "Current : 21 Train Acc : 0.37679999992251395 Test Acc : 0.3651250004488975\n",
      "\n",
      "Current : 22 Train Acc : 0.3792000004947185 Test Acc : 0.3678750005830079\n",
      "\n",
      "Current : 23 Train Acc : 0.38000000080466273 Test Acc : 0.3696250004414469\n",
      "\n",
      "Current : 24 Train Acc : 0.383000000923872 Test Acc : 0.37100000041536985\n",
      "\n",
      "Current : 25 Train Acc : 0.383800000756979 Test Acc : 0.3726250005047768\n",
      "\n",
      "Current : 26 Train Acc : 0.3842000014483929 Test Acc : 0.37450000026263297\n",
      "\n",
      "Current : 27 Train Acc : 0.3850000014305115 Test Acc : 0.37475000069476666\n",
      "\n",
      "Current : 28 Train Acc : 0.3874000018239021 Test Acc : 0.3783750008139759\n",
      "\n",
      "Current : 29 Train Acc : 0.38780000180006025 Test Acc : 0.3803750006295741\n",
      "\n",
      "Current : 30 Train Acc : 0.3914000019431114 Test Acc : 0.3796250007860362\n",
      "\n",
      "Current : 31 Train Acc : 0.3916000018119812 Test Acc : 0.3805000007338822\n",
      "\n",
      "Current : 32 Train Acc : 0.3942000014185906 Test Acc : 0.38112500083632767\n",
      "\n",
      "Current : 33 Train Acc : 0.3932000014185906 Test Acc : 0.38400000114925203\n",
      "\n",
      "Current : 34 Train Acc : 0.39620000144839285 Test Acc : 0.3861250013206154\n",
      "\n",
      "Current : 35 Train Acc : 0.400800001591444 Test Acc : 0.3876250005606562\n",
      "\n",
      "Current : 36 Train Acc : 0.3996000018417835 Test Acc : 0.38750000034458937\n",
      "\n",
      "Current : 37 Train Acc : 0.40380000206828115 Test Acc : 0.38912500071339307\n",
      "\n",
      "Current : 38 Train Acc : 0.4050000015795231 Test Acc : 0.3906250009592622\n",
      "\n",
      "Current : 39 Train Acc : 0.4064000011384487 Test Acc : 0.3911250015255064\n",
      "\n",
      "Current : 40 Train Acc : 0.40740000107884405 Test Acc : 0.39250000099651516\n",
      "\n",
      "Current : 41 Train Acc : 0.41020000067353246 Test Acc : 0.3933750013913959\n",
      "\n",
      "Current : 42 Train Acc : 0.40880000093579294 Test Acc : 0.39475000138394534\n",
      "\n",
      "Current : 43 Train Acc : 0.4128000008761883 Test Acc : 0.3946250013168901\n",
      "\n",
      "Current : 44 Train Acc : 0.4154000011384487 Test Acc : 0.39450000098906457\n",
      "\n",
      "Current : 45 Train Acc : 0.41700000086426736 Test Acc : 0.39500000077299774\n",
      "\n",
      "Current : 46 Train Acc : 0.4188000017106533 Test Acc : 0.3958750009071082\n",
      "\n",
      "Current : 47 Train Acc : 0.4188000021278858 Test Acc : 0.3962500010896474\n",
      "\n",
      "Current : 48 Train Acc : 0.4202000022828579 Test Acc : 0.3975000007171184\n",
      "\n",
      "Current : 49 Train Acc : 0.4224000024199486 Test Acc : 0.39787500069476667\n",
      "\n",
      "Current : 50 Train Acc : 0.4236000021100044 Test Acc : 0.39837500036694107\n",
      "\n",
      "Current : 51 Train Acc : 0.42660000270605086 Test Acc : 0.4002500003110617\n",
      "\n",
      "Current : 52 Train Acc : 0.42740000212192536 Test Acc : 0.3991250001359731\n",
      "\n",
      "Current : 53 Train Acc : 0.4298000026345253 Test Acc : 0.4012500003632158\n",
      "\n",
      "Current : 54 Train Acc : 0.4292000025510788 Test Acc : 0.40137500054202974\n",
      "\n",
      "Current : 55 Train Acc : 0.4322000021338463 Test Acc : 0.40262500059790907\n",
      "\n",
      "Current : 56 Train Acc : 0.430800002515316 Test Acc : 0.40300000114366413\n",
      "\n",
      "Current : 57 Train Acc : 0.43380000251531603 Test Acc : 0.4048750007897615\n",
      "\n",
      "Current : 58 Train Acc : 0.4352000025510788 Test Acc : 0.4057499997969717\n",
      "\n",
      "Current : 59 Train Acc : 0.4376000028252602 Test Acc : 0.40675000052899124\n",
      "\n",
      "Current : 60 Train Acc : 0.441000002682209 Test Acc : 0.4282499995827675\n",
      "\n",
      "Current : 61 Train Acc : 0.46840000194311143 Test Acc : 0.43012500047683716\n",
      "\n",
      "Current : 62 Train Acc : 0.4732000017762184 Test Acc : 0.43400000032037495\n",
      "\n",
      "Current : 63 Train Acc : 0.4770000013709068 Test Acc : 0.4351250001043081\n",
      "\n",
      "Current : 64 Train Acc : 0.47960000091791155 Test Acc : 0.4350000002235174\n",
      "\n",
      "Current : 65 Train Acc : 0.4778000013232231 Test Acc : 0.43600000090897084\n",
      "\n",
      "Current : 66 Train Acc : 0.47920000141859054 Test Acc : 0.4375000010430813\n",
      "\n",
      "Current : 67 Train Acc : 0.4818000019788742 Test Acc : 0.4376250011846423\n",
      "\n",
      "Current : 68 Train Acc : 0.4832000018954277 Test Acc : 0.43675000075250864\n",
      "\n",
      "Current : 69 Train Acc : 0.4852000021338463 Test Acc : 0.4383750006183982\n",
      "\n",
      "Current : 70 Train Acc : 0.4868000018000603 Test Acc : 0.4392500006407499\n",
      "\n",
      "Current : 71 Train Acc : 0.48740000182390214 Test Acc : 0.44012500047683717\n",
      "\n",
      "Current : 72 Train Acc : 0.486200001180172 Test Acc : 0.4411250007711351\n",
      "\n",
      "Current : 73 Train Acc : 0.4872000016570091 Test Acc : 0.44087500078603625\n",
      "\n",
      "Current : 74 Train Acc : 0.48840000182390214 Test Acc : 0.44200000083073976\n",
      "\n",
      "Current : 75 Train Acc : 0.4916000019907951 Test Acc : 0.44287500077858566\n",
      "\n",
      "Current : 76 Train Acc : 0.4926000021696091 Test Acc : 0.44262500109151004\n",
      "\n",
      "Current : 77 Train Acc : 0.49340000212192536 Test Acc : 0.4425000006891787\n",
      "\n",
      "Current : 78 Train Acc : 0.4926000015735626 Test Acc : 0.44362500090152024\n",
      "\n",
      "Current : 79 Train Acc : 0.49560000133514404 Test Acc : 0.44362500108778474\n",
      "\n",
      "Current : 80 Train Acc : 0.4978000019788742 Test Acc : 0.4441250004991889\n",
      "\n",
      "Current : 81 Train Acc : 0.4972000012397766 Test Acc : 0.4436250007897615\n",
      "\n",
      "Current : 82 Train Acc : 0.49980000138282776 Test Acc : 0.4453750005364418\n",
      "\n",
      "Current : 83 Train Acc : 0.5046000015735627 Test Acc : 0.4463750004395843\n",
      "\n",
      "Current : 84 Train Acc : 0.5054000009298325 Test Acc : 0.44637500058859586\n",
      "\n",
      "Current : 85 Train Acc : 0.5058000018596649 Test Acc : 0.44737500049173834\n",
      "\n",
      "Current : 86 Train Acc : 0.5080000017881393 Test Acc : 0.448500000461936\n",
      "\n",
      "Current : 87 Train Acc : 0.5068000015020371 Test Acc : 0.4483750003576279\n",
      "\n",
      "Current : 88 Train Acc : 0.5092000017166137 Test Acc : 0.4482499999180436\n",
      "\n",
      "Current : 89 Train Acc : 0.51200000166893 Test Acc : 0.44762500025331975\n",
      "\n",
      "Current : 90 Train Acc : 0.5120000020861626 Test Acc : 0.44899999994784595\n",
      "\n",
      "Current : 91 Train Acc : 0.5146000012755394 Test Acc : 0.4472499999403954\n",
      "\n",
      "Current : 92 Train Acc : 0.5156000012159347 Test Acc : 0.44837500028312205\n",
      "\n",
      "Current : 93 Train Acc : 0.517200001835823 Test Acc : 0.44950000029057263\n",
      "\n",
      "Current : 94 Train Acc : 0.5166000016927719 Test Acc : 0.45037499990314245\n",
      "\n",
      "Current : 95 Train Acc : 0.5206000016927719 Test Acc : 0.45275000043213365\n",
      "\n",
      "Current : 96 Train Acc : 0.521400001525879 Test Acc : 0.45325000006705524\n",
      "\n",
      "Current : 97 Train Acc : 0.5232000012397766 Test Acc : 0.4555000011250377\n",
      "\n",
      "Current : 98 Train Acc : 0.5244000009298324 Test Acc : 0.4542500003054738\n",
      "\n",
      "Current : 99 Train Acc : 0.5258000013828278 Test Acc : 0.45637500025331973\n",
      "\n",
      "Current : 100 Train Acc : 0.5272000007629395 Test Acc : 0.4575000004470348\n",
      "\n",
      "Current : 101 Train Acc : 0.5308000016212463 Test Acc : 0.4588750008866191\n",
      "\n",
      "Current : 102 Train Acc : 0.5314000005722046 Test Acc : 0.45887500084936617\n",
      "\n",
      "Current : 103 Train Acc : 0.5322000006437302 Test Acc : 0.4593750010058284\n",
      "\n",
      "Current : 104 Train Acc : 0.5338000015020371 Test Acc : 0.4601250008493662\n",
      "\n",
      "Current : 105 Train Acc : 0.5340000010728836 Test Acc : 0.46112500071525575\n",
      "\n",
      "Current : 106 Train Acc : 0.5356000009775161 Test Acc : 0.46325000081211326\n",
      "\n",
      "Current : 107 Train Acc : 0.5372000012397766 Test Acc : 0.4628750006854534\n",
      "\n",
      "Current : 108 Train Acc : 0.5380000007152558 Test Acc : 0.46337500028312206\n",
      "\n",
      "Current : 109 Train Acc : 0.5390000010728836 Test Acc : 0.46287500083446503\n",
      "\n",
      "Current : 110 Train Acc : 0.5396000009775161 Test Acc : 0.463875000923872\n",
      "\n",
      "Current : 111 Train Acc : 0.5418000009059906 Test Acc : 0.4652500012516975\n",
      "\n",
      "Current : 112 Train Acc : 0.5428000011444092 Test Acc : 0.4662500011175871\n",
      "\n",
      "Current : 113 Train Acc : 0.5436000019311905 Test Acc : 0.4663750010728836\n",
      "\n",
      "Current : 114 Train Acc : 0.5444000016450882 Test Acc : 0.4668750012665987\n",
      "\n",
      "Current : 115 Train Acc : 0.545800001502037 Test Acc : 0.4672500006109476\n",
      "\n",
      "Current : 116 Train Acc : 0.5464000009298324 Test Acc : 0.46800000056624413\n",
      "\n",
      "Current : 117 Train Acc : 0.5460000010728836 Test Acc : 0.468250000923872\n",
      "\n",
      "Current : 118 Train Acc : 0.5468000011444092 Test Acc : 0.4695000005140901\n",
      "\n",
      "Current : 119 Train Acc : 0.5478000006675721 Test Acc : 0.4701250006258488\n",
      "\n",
      "Current : 120 Train Acc : 0.548600000500679 Test Acc : 0.4712500000372529\n",
      "\n",
      "Current : 121 Train Acc : 0.5496000003814697 Test Acc : 0.47250000029802325\n",
      "\n",
      "Current : 122 Train Acc : 0.5503999999761582 Test Acc : 0.47275000043213367\n",
      "\n",
      "Current : 123 Train Acc : 0.5501999998092652 Test Acc : 0.47325000021606684\n",
      "\n",
      "Current : 124 Train Acc : 0.553200001001358 Test Acc : 0.47325000036507847\n",
      "\n",
      "Current : 125 Train Acc : 0.5558000005483628 Test Acc : 0.475375000461936\n",
      "\n",
      "Current : 126 Train Acc : 0.5564000002145767 Test Acc : 0.4737500002980232\n",
      "\n",
      "Current : 127 Train Acc : 0.5576000010967255 Test Acc : 0.47524999983608723\n",
      "\n",
      "Current : 128 Train Acc : 0.5606000003814697 Test Acc : 0.47399999998509884\n",
      "\n",
      "Current : 129 Train Acc : 0.5596000001430511 Test Acc : 0.4758750000223517\n",
      "\n",
      "Current : 130 Train Acc : 0.5620000001192093 Test Acc : 0.4748749998211861\n",
      "\n",
      "Current : 131 Train Acc : 0.5634000003933907 Test Acc : 0.4757499998062849\n",
      "\n",
      "Current : 132 Train Acc : 0.5650000014305114 Test Acc : 0.47637499995529653\n",
      "\n",
      "Current : 133 Train Acc : 0.5642000010609627 Test Acc : 0.47662499994039537\n",
      "\n",
      "Current : 134 Train Acc : 0.5644000014662742 Test Acc : 0.4767499999701977\n",
      "\n",
      "Current : 135 Train Acc : 0.566000001847744 Test Acc : 0.4791250001639128\n",
      "\n",
      "Current : 136 Train Acc : 0.5680000013709068 Test Acc : 0.47975000031292436\n",
      "\n",
      "Current : 137 Train Acc : 0.5682000009417534 Test Acc : 0.47887500017881396\n",
      "\n",
      "Current : 138 Train Acc : 0.570000001013279 Test Acc : 0.48025000013411046\n",
      "\n",
      "Current : 139 Train Acc : 0.5700000013709068 Test Acc : 0.480375000089407\n",
      "\n",
      "Current : 140 Train Acc : 0.5694000012278557 Test Acc : 0.4797500001639128\n",
      "\n",
      "Current : 141 Train Acc : 0.5716000016331673 Test Acc : 0.4793750002235174\n",
      "\n",
      "Current : 142 Train Acc : 0.5714000013470649 Test Acc : 0.4812500000372529\n",
      "\n",
      "Current : 143 Train Acc : 0.573600001513958 Test Acc : 0.48187500033527614\n",
      "\n",
      "Current : 144 Train Acc : 0.5740000013709068 Test Acc : 0.48212500009685755\n",
      "\n",
      "Current : 145 Train Acc : 0.5730000016689301 Test Acc : 0.4828750004246831\n",
      "\n",
      "Current : 146 Train Acc : 0.5738000018000603 Test Acc : 0.48225000012665986\n",
      "\n",
      "Current : 147 Train Acc : 0.5776000018715859 Test Acc : 0.4828750008717179\n",
      "\n",
      "Current : 148 Train Acc : 0.5778000012040139 Test Acc : 0.483000000603497\n",
      "\n",
      "Current : 149 Train Acc : 0.5780000017285347 Test Acc : 0.48187500026077035\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Z\n",
    "current_exp_name = 'Z';\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "# create layers\n",
    "l1 = CNN(3,3, 16,which_reg=current_exp_name); \n",
    "l2 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l3 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "\n",
    "l4 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l5 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l6 = CNN(3,16,10,which_reg=current_exp_name); \n",
    "\n",
    "# 2. graph \n",
    "x = tf.placeholder(tf.float32,(batch_size,96,96,3))\n",
    "y = tf.placeholder(tf.float32,(batch_size,10))\n",
    "\n",
    "layer1, layer1a = l1. feedforward(x,stride=2)\n",
    "layer2, layer2a = l2. feedforward(layer1a,stride=2)\n",
    "layer3, layer3a = l3. feedforward(layer2a,stride=2)\n",
    "layer4, layer4a = l4. feedforward(layer3a,stride=2)\n",
    "layer5, layer5a = l5. feedforward(layer4a)\n",
    "layer6, layer6a = l6. feedforward(layer5a)\n",
    "\n",
    "final_layer   = tf.reduce_mean(layer6a,(1,2))\n",
    "final_softmax = tf_softmax(final_layer)\n",
    "cost          = -tf.reduce_mean(y * tf.log(final_softmax + 1e-8))\n",
    "correct_prediction = tf.equal(tf.argmax(final_softmax, 1), tf.argmax(y, 1))\n",
    "accuracy           = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "gradient = tf.tile((final_softmax-y)[:,None,None,:],[1,6,6,1])/batch_size\n",
    "grad6p,grad6w,grad6_up = l6.backprop(gradient)\n",
    "grad5p,grad5w,grad5_up = l5.backprop(grad6p)\n",
    "grad4p,grad4w,grad4_up = l4.backprop(grad5p,stride=2)\n",
    "grad3p,grad3w,grad3_up = l3.backprop(grad4p,stride=2)\n",
    "grad2p,grad2w,grad2_up = l2.backprop(grad3p,stride=2)\n",
    "grad1p,grad1w,grad1_up = l1.backprop(grad2p,stride=2)\n",
    "\n",
    "gradient_update = grad6_up + grad5_up + grad4_up + grad3_up + grad2_up + grad1_up \n",
    "\n",
    "# train\n",
    "sess.run(tf.global_variables_initializer())\n",
    "avg_acc_train = 0; avg_acc_test  = 0; train_acc = [];test_acc = []\n",
    "\n",
    "# mean std skew kurt non-zero\n",
    "llayer1 = [[],[],[],[],[]]; llayer2 = [[],[],[],[],[]]; llayer3 = [[],[],[],[],[]]\n",
    "llayer4 = [[],[],[],[],[]]; llayer5 = [[],[],[],[],[]]; llayer6 = [[],[],[],[],[]]\n",
    "\n",
    "llayer1a = [[],[],[],[],[]]; llayer2a = [[],[],[],[],[]]; llayer3a = [[],[],[],[],[]]\n",
    "llayer4a = [[],[],[],[],[]]; llayer5a = [[],[],[],[],[]]; llayer6a = [[],[],[],[],[]]\n",
    "\n",
    "weight1 = [[],[],[],[],[]]; weight2 = [[],[],[],[],[]]; weight3 = [[],[],[],[],[]];\n",
    "weight4 = [[],[],[],[],[]]; weight5 = [[],[],[],[],[]]; weight6 = [[],[],[],[],[]];\n",
    "\n",
    "gradw1  = [[],[],[],[],[]]; gradw2  = [[],[],[],[],[]]; gradw3  = [[],[],[],[],[]];\n",
    "gradw4  = [[],[],[],[],[]]; gradw5  = [[],[],[],[],[]]; gradw6  = [[],[],[],[],[]];\n",
    "\n",
    "gradp1  = [[],[],[],[],[]]; gradp2  = [[],[],[],[],[]]; gradp3  = [[],[],[],[],[]];\n",
    "gradp4  = [[],[],[],[],[]]; gradp5  = [[],[],[],[],[]]; gradp6  = [[],[],[],[],[]];\n",
    "\n",
    "gradup1  = [[],[],[],[],[]]; gradup2  = [[],[],[],[],[]]; gradup3  = [[],[],[],[],[]];\n",
    "gradup4  = [[],[],[],[],[]]; gradup5  = [[],[],[],[],[]]; gradup6  = [[],[],[],[],[]];\n",
    "\n",
    "list_of_outputs = [\n",
    "    layer1,layer2,layer3,layer4,layer5,layer6,\n",
    "    layer1a,layer2a,layer3a,layer4a,layer5a,layer6a,\n",
    "    l1.getw(),l2.getw(),l3.getw(),l4.getw(),l5.getw(),l6.getw(),\n",
    "    grad1w,grad2w,grad3w,grad4w,grad5w,grad6w,\n",
    "    grad1p,grad2p,grad3p,grad4p,grad5p,grad6p,\n",
    "    grad1_up[0],grad2_up[0],grad3_up[0],grad4_up[0],grad5_up[0],grad6_up[0]\n",
    "]\n",
    "\n",
    "for iter in range(num_epoch):\n",
    "\n",
    "    # Training Accuracy    \n",
    "    for current_batch_index in range(0,len(train_images),batch_size):\n",
    "        current_data  = train_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = train_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy,gradient_update],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(train_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_train = avg_acc_train + sess_results[0]\n",
    "        \n",
    "    # get the results\n",
    "    mid_stat = sess.run(list_of_outputs,feed_dict={x:current_data,y:current_label})\n",
    "    \n",
    "    # Test Accuracy    \n",
    "    for current_batch_index in range(0,len(test_images), batch_size):\n",
    "        current_data  = test_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = test_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(test_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_test = avg_acc_test + sess_results[0]   \n",
    "        \n",
    "    # ======================== extract stats ========================\n",
    "    llayer1 = append_stat(llayer1,mid_stat,0);  llayer2 = append_stat(llayer2,mid_stat,1);  llayer3 = append_stat(llayer3,mid_stat,2);\n",
    "    llayer4 = append_stat(llayer4,mid_stat,3);  llayer5 = append_stat(llayer5,mid_stat,4);  llayer6 = append_stat(llayer6,mid_stat,5);\n",
    "\n",
    "    llayer1a = append_stat(llayer1a,mid_stat,6);  llayer2a = append_stat(llayer2a,mid_stat,7);  llayer3a = append_stat(llayer3a,mid_stat,8);\n",
    "    llayer4a = append_stat(llayer4a,mid_stat,9);  llayer5a = append_stat(llayer5a,mid_stat,10); llayer6a = append_stat(llayer6a,mid_stat,11);\n",
    "    \n",
    "    weight1 = append_stat(weight1,mid_stat,12);  weight2 = append_stat(weight2,mid_stat,13);  weight3 = append_stat(weight3,mid_stat,14);\n",
    "    weight4 = append_stat(weight4,mid_stat,15);  weight5 = append_stat(weight5,mid_stat,16);  weight6 = append_stat(weight6,mid_stat,17);\n",
    "    \n",
    "    gradw1 = append_stat(gradw1,mid_stat,18); gradw2 = append_stat(gradw2,mid_stat,19); gradw3 = append_stat(gradw3,mid_stat,20);\n",
    "    gradw4 = append_stat(gradw4,mid_stat,21); gradw5 = append_stat(gradw5,mid_stat,22); gradw6 = append_stat(gradw6,mid_stat,23);\n",
    "    \n",
    "    gradp1 = append_stat(gradp1,mid_stat,24); gradp2 = append_stat(gradp2,mid_stat,25); gradp3 = append_stat(gradp3,mid_stat,26);\n",
    "    gradp4 = append_stat(gradp4,mid_stat,27); gradp5 = append_stat(gradp5,mid_stat,28); gradp6 = append_stat(gradp6,mid_stat,29);\n",
    "\n",
    "    gradup1 = append_stat(gradup1,mid_stat,30); gradup2 = append_stat(gradup2,mid_stat,31); gradup3 = append_stat(gradup3,mid_stat,32);\n",
    "    gradup4 = append_stat(gradup4,mid_stat,33); gradup5 = append_stat(gradup5,mid_stat,34); gradup6 = append_stat(gradup6,mid_stat,35);\n",
    "\n",
    "    train_acc.append(avg_acc_train/(len(train_images)/batch_size))\n",
    "    test_acc .append(avg_acc_test / (len(test_images)/batch_size))\n",
    "    # ======================== extract stats ========================\n",
    "    \n",
    "    # ======================== save to image ========================\n",
    "    save_to_image(mid_stat[0:6]   ,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,\"layer\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[6:12]  ,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a,\"layera\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[12:18] ,weight1,weight2,weight3,weight4,weight5,weight6,\"weights\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[18:24] ,gradw1,gradw2,gradw3,gradw4,gradw5,gradw6,\"gradientw\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[24:30] ,gradp1,gradp2,gradp3,gradp4,gradp5,gradp6,\"gradientp\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[30:36] ,gradup1,gradup2,gradup3,gradup4,gradup5,gradup6,\"moment\",train_acc,test_acc,current_exp_name,iter)\n",
    "    # ======================== save to image ========================\n",
    "        \n",
    "    # ======================== print reset ========================\n",
    "    print(\"Current : \"+ str(iter) + \" Train Acc : \" + str(avg_acc_train/(len(train_images)/batch_size)) + \" Test Acc : \" + str(avg_acc_test/(len(test_images)/batch_size)) + '\\n')\n",
    "    avg_acc_train = 0 ; avg_acc_test  = 0\n",
    "    # ======================== print reset ========================\n",
    "\n",
    "np.save(current_exp_name+'/train_acc.npy',train_acc); np.save(current_exp_name+'/test_acc.npy', test_acc)    \n",
    "np.save(current_exp_name+'/llayer1.npy', llayer1);  np.save(current_exp_name+'/llayer2.npy', llayer2);  np.save(current_exp_name+'/llayer3.npy', llayer3); \n",
    "np.save(current_exp_name+'/llayer4.npy', llayer4);  np.save(current_exp_name+'/llayer5.npy', llayer5);  np.save(current_exp_name+'/llayer6.npy', llayer6); \n",
    "\n",
    "np.save(current_exp_name+'/llayer1a.npy', llayer1a);  np.save(current_exp_name+'/llayer2a.npy', llayer2a);  np.save(current_exp_name+'/llayer3a.npy', llayer3a); \n",
    "np.save(current_exp_name+'/llayer4a.npy', llayer4a);  np.save(current_exp_name+'/llayer5a.npy', llayer5a);  np.save(current_exp_name+'/llayer6a.npy', llayer6a); \n",
    "\n",
    "np.save(current_exp_name+'/weight1.npy', weight1);  np.save(current_exp_name+'/weight2.npy', weight2);  np.save(current_exp_name+'/weight3.npy', weight3);  \n",
    "np.save(current_exp_name+'/weight4.npy', weight4);  np.save(current_exp_name+'/weight5.npy', weight5);  np.save(current_exp_name+'/weight6.npy', weight6);  \n",
    "\n",
    "np.save(current_exp_name+'/gradw1.npy', gradw1); np.save(current_exp_name+'/gradw2.npy', gradw2); np.save(current_exp_name+'/gradw3.npy', gradw3);\n",
    "np.save(current_exp_name+'/gradw4.npy', gradw4); np.save(current_exp_name+'/gradw5.npy', gradw5); np.save(current_exp_name+'/gradw6.npy', gradw6);\n",
    "\n",
    "np.save(current_exp_name+'/gradp1.npy', gradp1); np.save(current_exp_name+'/gradp2.npy', gradp2); np.save(current_exp_name+'/gradp3.npy', gradp3);\n",
    "np.save(current_exp_name+'/gradp4.npy', gradp4); np.save(current_exp_name+'/gradp5.npy', gradp5); np.save(current_exp_name+'/gradp6.npy', gradp6);\n",
    "\n",
    "np.save(current_exp_name+'/gradup1.npy', gradup1); np.save(current_exp_name+'/gradup2.npy', gradup2); np.save(current_exp_name+'/gradup3.npy', gradup3);\n",
    "np.save(current_exp_name+'/gradup4.npy', gradup4); np.save(current_exp_name+'/gradup5.npy', gradup5); np.save(current_exp_name+'/gradup6.npy', gradup6);\n",
    "\n",
    "sess.close(); tf.reset_default_graph();\n",
    "\n",
    "%reset_selective -f l1,l2,l3,l4,l5,l6\n",
    "%reset_selective -f layer1,layer2,layer3,layer4,layer5,layer6\n",
    "%reset_selective -f layer1a,layer2a,layer3a,layer4a,layer5a,layer6a\n",
    "%reset_selective -f train_acc,test_acc,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a\n",
    "%reset_selective -f weight1,weight2,weight3,weight4,weight5,weight6\n",
    "%reset_selective -f gradw1,gradw2,gradw3,gradw4,gradw5,gradw6\n",
    "%reset_selective -f gradp1,gradp2,gradp3,gradp4,gradp5,gradp6\n",
    "%reset_selective -f gradup1,gradup2,gradup3,gradup4,gradup5,gradup6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-05T03:04:25.512552Z",
     "start_time": "2019-01-05T02:14:30.111207Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current : 0 Train Acc : 0.12520000219345093 Test Acc : 0.17137500314973295\n",
      "\n",
      "Current : 1 Train Acc : 0.16260000313818454 Test Acc : 0.16837500305846334\n",
      "\n",
      "Current : 2 Train Acc : 0.20100000366568566 Test Acc : 0.2147500032838434\n",
      "\n",
      "Current : 3 Train Acc : 0.2406000030785799 Test Acc : 0.2551250029914081\n",
      "\n",
      "Current : 4 Train Acc : 0.2748000023216009 Test Acc : 0.2888750022184104\n",
      "\n",
      "Current : 5 Train Acc : 0.29140000288188456 Test Acc : 0.3045000023022294\n",
      "\n",
      "Current : 6 Train Acc : 0.3048000024408102 Test Acc : 0.31912500218488277\n",
      "\n",
      "Current : 7 Train Acc : 0.31300000251829624 Test Acc : 0.3307500015571713\n",
      "\n",
      "Current : 8 Train Acc : 0.3224000021219254 Test Acc : 0.33987500217743216\n",
      "\n",
      "Current : 9 Train Acc : 0.33340000224113464 Test Acc : 0.34937500187195836\n",
      "\n",
      "Current : 10 Train Acc : 0.34400000230968 Test Acc : 0.36100000262260434\n",
      "\n",
      "Current : 11 Train Acc : 0.35580000130832196 Test Acc : 0.36737500195391476\n",
      "\n",
      "Current : 12 Train Acc : 0.3604000013321638 Test Acc : 0.37512500140815974\n",
      "\n",
      "Current : 13 Train Acc : 0.3678000010997057 Test Acc : 0.3797500015422702\n",
      "\n",
      "Current : 14 Train Acc : 0.3739999998062849 Test Acc : 0.37950000146403906\n",
      "\n",
      "Current : 15 Train Acc : 0.3818000003248453 Test Acc : 0.3861250020749867\n",
      "\n",
      "Current : 16 Train Acc : 0.3859999998509884 Test Acc : 0.3892500012740493\n",
      "\n",
      "Current : 17 Train Acc : 0.3934000009596348 Test Acc : 0.3905000018700957\n",
      "\n",
      "Current : 18 Train Acc : 0.397400001257658 Test Acc : 0.3926250018179417\n",
      "\n",
      "Current : 19 Train Acc : 0.40280000123381615 Test Acc : 0.3980000016093254\n",
      "\n",
      "Current : 20 Train Acc : 0.4074000011384487 Test Acc : 0.40062500115484\n",
      "\n",
      "Current : 21 Train Acc : 0.40860000094771387 Test Acc : 0.4036250016093254\n",
      "\n",
      "Current : 22 Train Acc : 0.4112000012099743 Test Acc : 0.40850000113248824\n",
      "\n",
      "Current : 23 Train Acc : 0.4154000008404255 Test Acc : 0.4100000010803342\n",
      "\n",
      "Current : 24 Train Acc : 0.4166000008881092 Test Acc : 0.4126250009983778\n",
      "\n",
      "Current : 25 Train Acc : 0.42160000121593477 Test Acc : 0.41300000093877315\n",
      "\n",
      "Current : 26 Train Acc : 0.42380000126361844 Test Acc : 0.4141250007599592\n",
      "\n",
      "Current : 27 Train Acc : 0.42840000087022784 Test Acc : 0.4151250010356307\n",
      "\n",
      "Current : 28 Train Acc : 0.4292000004053116 Test Acc : 0.4182500006631017\n",
      "\n",
      "Current : 29 Train Acc : 0.431200001001358 Test Acc : 0.4173750011622906\n",
      "\n",
      "Current : 30 Train Acc : 0.4324000009894371 Test Acc : 0.4187500010058284\n",
      "\n",
      "Current : 31 Train Acc : 0.43400000149011614 Test Acc : 0.42050000056624415\n",
      "\n",
      "Current : 32 Train Acc : 0.43460000115633013 Test Acc : 0.42387500137090683\n",
      "\n",
      "Current : 33 Train Acc : 0.4426000002324581 Test Acc : 0.4235000008717179\n",
      "\n",
      "Current : 34 Train Acc : 0.4454000000059605 Test Acc : 0.424000000320375\n",
      "\n",
      "Current : 35 Train Acc : 0.44880000039935114 Test Acc : 0.42525000132620333\n",
      "\n",
      "Current : 36 Train Acc : 0.45340000104904177 Test Acc : 0.42662500094622374\n",
      "\n",
      "Current : 37 Train Acc : 0.4548000007271767 Test Acc : 0.4288750008866191\n",
      "\n",
      "Current : 38 Train Acc : 0.45960000044107435 Test Acc : 0.4296250008046627\n",
      "\n",
      "Current : 39 Train Acc : 0.45980000001192095 Test Acc : 0.4298750006407499\n",
      "\n",
      "Current : 40 Train Acc : 0.4606000007987022 Test Acc : 0.4307500007003546\n",
      "\n",
      "Current : 41 Train Acc : 0.4626000003218651 Test Acc : 0.4333750004693866\n",
      "\n",
      "Current : 42 Train Acc : 0.4656000007390976 Test Acc : 0.4343750005587935\n",
      "\n",
      "Current : 43 Train Acc : 0.46600000113248824 Test Acc : 0.43687500070780516\n",
      "\n",
      "Current : 44 Train Acc : 0.4690000007152557 Test Acc : 0.4366250004246831\n",
      "\n",
      "Current : 45 Train Acc : 0.47000000077486037 Test Acc : 0.44037500079721215\n",
      "\n",
      "Current : 46 Train Acc : 0.47260000139474867 Test Acc : 0.4388750007376075\n",
      "\n",
      "Current : 47 Train Acc : 0.4736000016331673 Test Acc : 0.4403750012069941\n",
      "\n",
      "Current : 48 Train Acc : 0.47240000194311144 Test Acc : 0.441625001244247\n",
      "\n",
      "Current : 49 Train Acc : 0.4748000019192696 Test Acc : 0.441625001244247\n",
      "\n",
      "Current : 50 Train Acc : 0.4736000022292137 Test Acc : 0.4450000013411045\n",
      "\n",
      "Current : 51 Train Acc : 0.4740000024437904 Test Acc : 0.4471250006183982\n",
      "\n",
      "Current : 52 Train Acc : 0.4744000017642975 Test Acc : 0.44675000093877315\n",
      "\n",
      "Current : 53 Train Acc : 0.47600000143051147 Test Acc : 0.4458750006556511\n",
      "\n",
      "Current : 54 Train Acc : 0.4772000016570091 Test Acc : 0.4440000004693866\n",
      "\n",
      "Current : 55 Train Acc : 0.47520000201463697 Test Acc : 0.4451250010728836\n",
      "\n",
      "Current : 56 Train Acc : 0.4780000020265579 Test Acc : 0.4465000008419156\n",
      "\n",
      "Current : 57 Train Acc : 0.4810000017285347 Test Acc : 0.4448750011995435\n",
      "\n",
      "Current : 58 Train Acc : 0.48280000203847884 Test Acc : 0.4442500013485551\n",
      "\n",
      "Current : 59 Train Acc : 0.48360000240802764 Test Acc : 0.445000001527369\n",
      "\n",
      "Current : 60 Train Acc : 0.4860000023841858 Test Acc : 0.4463750006258488\n",
      "\n",
      "Current : 61 Train Acc : 0.4880000022053719 Test Acc : 0.44537500105798244\n",
      "\n",
      "Current : 62 Train Acc : 0.4872000025510788 Test Acc : 0.4475000012293458\n",
      "\n",
      "Current : 63 Train Acc : 0.48900000262260435 Test Acc : 0.4445000011473894\n",
      "\n",
      "Current : 64 Train Acc : 0.49120000207424164 Test Acc : 0.44687500096857546\n",
      "\n",
      "Current : 65 Train Acc : 0.4940000023841858 Test Acc : 0.44750000096857545\n",
      "\n",
      "Current : 66 Train Acc : 0.49400000166893004 Test Acc : 0.44725000083446503\n",
      "\n",
      "Current : 67 Train Acc : 0.4962000021338463 Test Acc : 0.44887500125914814\n",
      "\n",
      "Current : 68 Train Acc : 0.49700000214576723 Test Acc : 0.4488750011101365\n",
      "\n",
      "Current : 69 Train Acc : 0.4976000016927719 Test Acc : 0.4486250012740493\n",
      "\n",
      "Current : 70 Train Acc : 0.5006000017523765 Test Acc : 0.4486250012740493\n",
      "\n",
      "Current : 71 Train Acc : 0.5000000008940697 Test Acc : 0.44837500106543304\n",
      "\n",
      "Current : 72 Train Acc : 0.5012000016570092 Test Acc : 0.45062500115484\n",
      "\n",
      "Current : 73 Train Acc : 0.5026000011563301 Test Acc : 0.45062500100582836\n",
      "\n",
      "Current : 74 Train Acc : 0.5030000014901161 Test Acc : 0.4526250011473894\n",
      "\n",
      "Current : 75 Train Acc : 0.5036000013947487 Test Acc : 0.4532500009983778\n",
      "\n",
      "Current : 76 Train Acc : 0.505600001513958 Test Acc : 0.4561250016465783\n",
      "\n",
      "Current : 77 Train Acc : 0.5076000011563301 Test Acc : 0.45412500139325856\n",
      "\n",
      "Current : 78 Train Acc : 0.5074000014662743 Test Acc : 0.45762500148266555\n",
      "\n",
      "Current : 79 Train Acc : 0.5098000014424324 Test Acc : 0.4552500016614795\n",
      "\n",
      "Current : 80 Train Acc : 0.5100000020861626 Test Acc : 0.4568750014528632\n",
      "\n",
      "Current : 81 Train Acc : 0.5116000013947487 Test Acc : 0.45487500179558993\n",
      "\n",
      "Current : 82 Train Acc : 0.5104000015854836 Test Acc : 0.4545000022277236\n",
      "\n",
      "Current : 83 Train Acc : 0.5126000009179116 Test Acc : 0.4565000020340085\n",
      "\n",
      "Current : 84 Train Acc : 0.5132000014185906 Test Acc : 0.4572500018402934\n",
      "\n",
      "Current : 85 Train Acc : 0.5164000012874603 Test Acc : 0.4566250020265579\n",
      "\n",
      "Current : 86 Train Acc : 0.5178000015020371 Test Acc : 0.4570000022649765\n",
      "\n",
      "Current : 87 Train Acc : 0.5200000021457672 Test Acc : 0.45800000227987764\n",
      "\n",
      "Current : 88 Train Acc : 0.5198000020980835 Test Acc : 0.46200000174343586\n",
      "\n",
      "Current : 89 Train Acc : 0.5210000022649764 Test Acc : 0.4612500015646219\n",
      "\n",
      "Current : 90 Train Acc : 0.5246000025272369 Test Acc : 0.46125000193715093\n",
      "\n",
      "Current : 91 Train Acc : 0.5230000025033951 Test Acc : 0.4645000016689301\n",
      "\n",
      "Current : 92 Train Acc : 0.5266000014543534 Test Acc : 0.461875001937151\n",
      "\n",
      "Current : 93 Train Acc : 0.5262000019550324 Test Acc : 0.46625000160187485\n",
      "\n",
      "Current : 94 Train Acc : 0.5266000019311905 Test Acc : 0.4612500014528632\n",
      "\n",
      "Current : 95 Train Acc : 0.5298000022172927 Test Acc : 0.4620000015944242\n",
      "\n",
      "Current : 96 Train Acc : 0.5314000029563903 Test Acc : 0.46387500163167716\n",
      "\n",
      "Current : 97 Train Acc : 0.5300000021457673 Test Acc : 0.4660000015050173\n",
      "\n",
      "Current : 98 Train Acc : 0.5326000024080276 Test Acc : 0.4655000014975667\n",
      "\n",
      "Current : 99 Train Acc : 0.5328000029325485 Test Acc : 0.4666250018030405\n",
      "\n",
      "Current : 100 Train Acc : 0.5346000019311905 Test Acc : 0.4675000021606684\n",
      "\n",
      "Current : 101 Train Acc : 0.5350000026226044 Test Acc : 0.4688750022649765\n",
      "\n",
      "Current : 102 Train Acc : 0.5346000019311905 Test Acc : 0.47187500182539227\n",
      "\n",
      "Current : 103 Train Acc : 0.5376000014543534 Test Acc : 0.47137500148266553\n",
      "\n",
      "Current : 104 Train Acc : 0.5390000021457673 Test Acc : 0.4723750014975667\n",
      "\n",
      "Current : 105 Train Acc : 0.5408000018596649 Test Acc : 0.47425000175833704\n",
      "\n",
      "Current : 106 Train Acc : 0.5422000018358231 Test Acc : 0.4747500017285347\n",
      "\n",
      "Current : 107 Train Acc : 0.5436000024080276 Test Acc : 0.477125001847744\n",
      "\n",
      "Current : 108 Train Acc : 0.5464000012874604 Test Acc : 0.4765000019967556\n",
      "\n",
      "Current : 109 Train Acc : 0.5448000007867813 Test Acc : 0.4780000018328428\n",
      "\n",
      "Current : 110 Train Acc : 0.5446000013351441 Test Acc : 0.47775000117719174\n",
      "\n",
      "Current : 111 Train Acc : 0.5468000005483628 Test Acc : 0.48062500171363354\n",
      "\n",
      "Current : 112 Train Acc : 0.5470000015497207 Test Acc : 0.47950000133365395\n",
      "\n",
      "Current : 113 Train Acc : 0.5476000014543533 Test Acc : 0.4817500017210841\n",
      "\n",
      "Current : 114 Train Acc : 0.5494000004529953 Test Acc : 0.4825000014901161\n",
      "\n",
      "Current : 115 Train Acc : 0.5492000006437302 Test Acc : 0.4823750014603138\n",
      "\n",
      "Current : 116 Train Acc : 0.5542000004053116 Test Acc : 0.48287500120699406\n",
      "\n",
      "Current : 117 Train Acc : 0.5526000003814697 Test Acc : 0.48650000140070915\n",
      "\n",
      "Current : 118 Train Acc : 0.5534000018835068 Test Acc : 0.4853750010579824\n",
      "\n",
      "Current : 119 Train Acc : 0.5574000016450882 Test Acc : 0.4852500015869737\n",
      "\n",
      "Current : 120 Train Acc : 0.5572000019550324 Test Acc : 0.487125001475215\n",
      "\n",
      "Current : 121 Train Acc : 0.5586000020503998 Test Acc : 0.48512500125914815\n",
      "\n",
      "Current : 122 Train Acc : 0.55880000269413 Test Acc : 0.4877500012144446\n",
      "\n",
      "Current : 123 Train Acc : 0.5596000015735626 Test Acc : 0.48962500162422656\n",
      "\n",
      "Current : 124 Train Acc : 0.559800001502037 Test Acc : 0.4886250015348196\n",
      "\n",
      "Current : 125 Train Acc : 0.55900000166893 Test Acc : 0.49025000173598526\n",
      "\n",
      "Current : 126 Train Acc : 0.5592000017166138 Test Acc : 0.49087500154972075\n",
      "\n",
      "Current : 127 Train Acc : 0.5586000019311905 Test Acc : 0.4895000013709068\n",
      "\n",
      "Current : 128 Train Acc : 0.5604000010490418 Test Acc : 0.4905000017210841\n",
      "\n",
      "Current : 129 Train Acc : 0.5628000020980835 Test Acc : 0.49062500182539226\n",
      "\n",
      "Current : 130 Train Acc : 0.5624000015258789 Test Acc : 0.4908750017359853\n",
      "\n",
      "Current : 131 Train Acc : 0.5642000013589858 Test Acc : 0.4925000014528632\n",
      "\n",
      "Current : 132 Train Acc : 0.5648000017404556 Test Acc : 0.491500001065433\n",
      "\n",
      "Current : 133 Train Acc : 0.5662000014781952 Test Acc : 0.49100000131875277\n",
      "\n",
      "Current : 134 Train Acc : 0.567400000333786 Test Acc : 0.49175000105053185\n",
      "\n",
      "Current : 135 Train Acc : 0.5676000008583069 Test Acc : 0.4915000008419156\n",
      "\n",
      "Current : 136 Train Acc : 0.5696000002622604 Test Acc : 0.49225000113248824\n",
      "\n",
      "Current : 137 Train Acc : 0.5714000004529953 Test Acc : 0.4926250019669533\n",
      "\n",
      "Current : 138 Train Acc : 0.5704000008106231 Test Acc : 0.49275000162422655\n",
      "\n",
      "Current : 139 Train Acc : 0.5720000014305114 Test Acc : 0.4935000018030405\n",
      "\n",
      "Current : 140 Train Acc : 0.5722000014781952 Test Acc : 0.49112500160932543\n",
      "\n",
      "Current : 141 Train Acc : 0.5750000011920929 Test Acc : 0.4897500013187528\n",
      "\n",
      "Current : 142 Train Acc : 0.5752000007629394 Test Acc : 0.4922500012069941\n",
      "\n",
      "Current : 143 Train Acc : 0.5734000011682511 Test Acc : 0.49175000201910735\n",
      "\n",
      "Current : 144 Train Acc : 0.5766000010967255 Test Acc : 0.492500001527369\n",
      "\n",
      "Current : 145 Train Acc : 0.5778000020980835 Test Acc : 0.4931250013783574\n",
      "\n",
      "Current : 146 Train Acc : 0.577600001335144 Test Acc : 0.4928750018775463\n",
      "\n",
      "Current : 147 Train Acc : 0.5784000015258789 Test Acc : 0.4935000013560057\n",
      "\n",
      "Current : 148 Train Acc : 0.5810000015497208 Test Acc : 0.49387500144541263\n",
      "\n",
      "Current : 149 Train Acc : 0.5808000031709671 Test Acc : 0.4960000018030405\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# A\n",
    "current_exp_name = 'A';\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "# create layers\n",
    "l1 = CNN(3,3, 16,which_reg=current_exp_name); \n",
    "l2 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l3 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "\n",
    "l4 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l5 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l6 = CNN(3,16,10,which_reg=current_exp_name); \n",
    "\n",
    "# 2. graph \n",
    "x = tf.placeholder(tf.float32,(batch_size,96,96,3))\n",
    "y = tf.placeholder(tf.float32,(batch_size,10))\n",
    "\n",
    "layer1, layer1a = l1. feedforward(x,stride=2)\n",
    "layer2, layer2a = l2. feedforward(layer1a,stride=2)\n",
    "layer3, layer3a = l3. feedforward(layer2a,stride=2)\n",
    "layer4, layer4a = l4. feedforward(layer3a,stride=2)\n",
    "layer5, layer5a = l5. feedforward(layer4a)\n",
    "layer6, layer6a = l6. feedforward(layer5a)\n",
    "\n",
    "final_layer   = tf.reduce_mean(layer6a,(1,2))\n",
    "final_softmax = tf_softmax(final_layer)\n",
    "cost          = -tf.reduce_mean(y * tf.log(final_softmax + 1e-8))\n",
    "correct_prediction = tf.equal(tf.argmax(final_softmax, 1), tf.argmax(y, 1))\n",
    "accuracy           = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "gradient = tf.tile((final_softmax-y)[:,None,None,:],[1,6,6,1])/batch_size\n",
    "grad6p,grad6w,grad6_up = l6.backprop(gradient)\n",
    "grad5p,grad5w,grad5_up = l5.backprop(grad6p)\n",
    "grad4p,grad4w,grad4_up = l4.backprop(grad5p,stride=2)\n",
    "grad3p,grad3w,grad3_up = l3.backprop(grad4p,stride=2)\n",
    "grad2p,grad2w,grad2_up = l2.backprop(grad3p,stride=2)\n",
    "grad1p,grad1w,grad1_up = l1.backprop(grad2p,stride=2)\n",
    "\n",
    "gradient_update = grad6_up + grad5_up + grad4_up + grad3_up + grad2_up + grad1_up \n",
    "\n",
    "# train\n",
    "sess.run(tf.global_variables_initializer())\n",
    "avg_acc_train = 0; avg_acc_test  = 0; train_acc = [];test_acc = []\n",
    "\n",
    "# mean std skew kurt non-zero\n",
    "llayer1 = [[],[],[],[],[]]; llayer2 = [[],[],[],[],[]]; llayer3 = [[],[],[],[],[]]\n",
    "llayer4 = [[],[],[],[],[]]; llayer5 = [[],[],[],[],[]]; llayer6 = [[],[],[],[],[]]\n",
    "\n",
    "llayer1a = [[],[],[],[],[]]; llayer2a = [[],[],[],[],[]]; llayer3a = [[],[],[],[],[]]\n",
    "llayer4a = [[],[],[],[],[]]; llayer5a = [[],[],[],[],[]]; llayer6a = [[],[],[],[],[]]\n",
    "\n",
    "weight1 = [[],[],[],[],[]]; weight2 = [[],[],[],[],[]]; weight3 = [[],[],[],[],[]];\n",
    "weight4 = [[],[],[],[],[]]; weight5 = [[],[],[],[],[]]; weight6 = [[],[],[],[],[]];\n",
    "\n",
    "gradw1  = [[],[],[],[],[]]; gradw2  = [[],[],[],[],[]]; gradw3  = [[],[],[],[],[]];\n",
    "gradw4  = [[],[],[],[],[]]; gradw5  = [[],[],[],[],[]]; gradw6  = [[],[],[],[],[]];\n",
    "\n",
    "gradp1  = [[],[],[],[],[]]; gradp2  = [[],[],[],[],[]]; gradp3  = [[],[],[],[],[]];\n",
    "gradp4  = [[],[],[],[],[]]; gradp5  = [[],[],[],[],[]]; gradp6  = [[],[],[],[],[]];\n",
    "\n",
    "gradup1  = [[],[],[],[],[]]; gradup2  = [[],[],[],[],[]]; gradup3  = [[],[],[],[],[]];\n",
    "gradup4  = [[],[],[],[],[]]; gradup5  = [[],[],[],[],[]]; gradup6  = [[],[],[],[],[]];\n",
    "\n",
    "list_of_outputs = [\n",
    "    layer1,layer2,layer3,layer4,layer5,layer6,\n",
    "    layer1a,layer2a,layer3a,layer4a,layer5a,layer6a,\n",
    "    l1.getw(),l2.getw(),l3.getw(),l4.getw(),l5.getw(),l6.getw(),\n",
    "    grad1w,grad2w,grad3w,grad4w,grad5w,grad6w,\n",
    "    grad1p,grad2p,grad3p,grad4p,grad5p,grad6p,\n",
    "    grad1_up[0],grad2_up[0],grad3_up[0],grad4_up[0],grad5_up[0],grad6_up[0]\n",
    "]\n",
    "\n",
    "for iter in range(num_epoch):\n",
    "\n",
    "    # Training Accuracy    \n",
    "    for current_batch_index in range(0,len(train_images),batch_size):\n",
    "        current_data  = train_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = train_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy,gradient_update],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(train_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_train = avg_acc_train + sess_results[0]\n",
    "        \n",
    "    # get the results\n",
    "    mid_stat = sess.run(list_of_outputs,feed_dict={x:current_data,y:current_label})\n",
    "    \n",
    "    # Test Accuracy    \n",
    "    for current_batch_index in range(0,len(test_images), batch_size):\n",
    "        current_data  = test_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = test_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(test_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_test = avg_acc_test + sess_results[0]   \n",
    "        \n",
    "    # ======================== extract stats ========================\n",
    "    llayer1 = append_stat(llayer1,mid_stat,0);  llayer2 = append_stat(llayer2,mid_stat,1);  llayer3 = append_stat(llayer3,mid_stat,2);\n",
    "    llayer4 = append_stat(llayer4,mid_stat,3);  llayer5 = append_stat(llayer5,mid_stat,4);  llayer6 = append_stat(llayer6,mid_stat,5);\n",
    "\n",
    "    llayer1a = append_stat(llayer1a,mid_stat,6);  llayer2a = append_stat(llayer2a,mid_stat,7);  llayer3a = append_stat(llayer3a,mid_stat,8);\n",
    "    llayer4a = append_stat(llayer4a,mid_stat,9);  llayer5a = append_stat(llayer5a,mid_stat,10); llayer6a = append_stat(llayer6a,mid_stat,11);\n",
    "    \n",
    "    weight1 = append_stat(weight1,mid_stat,12);  weight2 = append_stat(weight2,mid_stat,13);  weight3 = append_stat(weight3,mid_stat,14);\n",
    "    weight4 = append_stat(weight4,mid_stat,15);  weight5 = append_stat(weight5,mid_stat,16);  weight6 = append_stat(weight6,mid_stat,17);\n",
    "    \n",
    "    gradw1 = append_stat(gradw1,mid_stat,18); gradw2 = append_stat(gradw2,mid_stat,19); gradw3 = append_stat(gradw3,mid_stat,20);\n",
    "    gradw4 = append_stat(gradw4,mid_stat,21); gradw5 = append_stat(gradw5,mid_stat,22); gradw6 = append_stat(gradw6,mid_stat,23);\n",
    "    \n",
    "    gradp1 = append_stat(gradp1,mid_stat,24); gradp2 = append_stat(gradp2,mid_stat,25); gradp3 = append_stat(gradp3,mid_stat,26);\n",
    "    gradp4 = append_stat(gradp4,mid_stat,27); gradp5 = append_stat(gradp5,mid_stat,28); gradp6 = append_stat(gradp6,mid_stat,29);\n",
    "\n",
    "    gradup1 = append_stat(gradup1,mid_stat,30); gradup2 = append_stat(gradup2,mid_stat,31); gradup3 = append_stat(gradup3,mid_stat,32);\n",
    "    gradup4 = append_stat(gradup4,mid_stat,33); gradup5 = append_stat(gradup5,mid_stat,34); gradup6 = append_stat(gradup6,mid_stat,35);\n",
    "\n",
    "    train_acc.append(avg_acc_train/(len(train_images)/batch_size))\n",
    "    test_acc .append(avg_acc_test / (len(test_images)/batch_size))\n",
    "    # ======================== extract stats ========================\n",
    "    \n",
    "    # ======================== save to image ========================\n",
    "    save_to_image(mid_stat[0:6]   ,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,\"layer\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[6:12]  ,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a,\"layera\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[12:18] ,weight1,weight2,weight3,weight4,weight5,weight6,\"weights\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[18:24] ,gradw1,gradw2,gradw3,gradw4,gradw5,gradw6,\"gradientw\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[24:30] ,gradp1,gradp2,gradp3,gradp4,gradp5,gradp6,\"gradientp\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[30:36] ,gradup1,gradup2,gradup3,gradup4,gradup5,gradup6,\"moment\",train_acc,test_acc,current_exp_name,iter)\n",
    "    # ======================== save to image ========================\n",
    "        \n",
    "    # ======================== print reset ========================\n",
    "    print(\"Current : \"+ str(iter) + \" Train Acc : \" + str(avg_acc_train/(len(train_images)/batch_size)) + \" Test Acc : \" + str(avg_acc_test/(len(test_images)/batch_size)) + '\\n')\n",
    "    avg_acc_train = 0 ; avg_acc_test  = 0\n",
    "    # ======================== print reset ========================\n",
    "\n",
    "np.save(current_exp_name+'/train_acc.npy',train_acc); np.save(current_exp_name+'/test_acc.npy', test_acc)    \n",
    "np.save(current_exp_name+'/llayer1.npy', llayer1);  np.save(current_exp_name+'/llayer2.npy', llayer2);  np.save(current_exp_name+'/llayer3.npy', llayer3); \n",
    "np.save(current_exp_name+'/llayer4.npy', llayer4);  np.save(current_exp_name+'/llayer5.npy', llayer5);  np.save(current_exp_name+'/llayer6.npy', llayer6); \n",
    "\n",
    "np.save(current_exp_name+'/llayer1a.npy', llayer1a);  np.save(current_exp_name+'/llayer2a.npy', llayer2a);  np.save(current_exp_name+'/llayer3a.npy', llayer3a); \n",
    "np.save(current_exp_name+'/llayer4a.npy', llayer4a);  np.save(current_exp_name+'/llayer5a.npy', llayer5a);  np.save(current_exp_name+'/llayer6a.npy', llayer6a); \n",
    "\n",
    "np.save(current_exp_name+'/weight1.npy', weight1);  np.save(current_exp_name+'/weight2.npy', weight2);  np.save(current_exp_name+'/weight3.npy', weight3);  \n",
    "np.save(current_exp_name+'/weight4.npy', weight4);  np.save(current_exp_name+'/weight5.npy', weight5);  np.save(current_exp_name+'/weight6.npy', weight6);  \n",
    "\n",
    "np.save(current_exp_name+'/gradw1.npy', gradw1); np.save(current_exp_name+'/gradw2.npy', gradw2); np.save(current_exp_name+'/gradw3.npy', gradw3);\n",
    "np.save(current_exp_name+'/gradw4.npy', gradw4); np.save(current_exp_name+'/gradw5.npy', gradw5); np.save(current_exp_name+'/gradw6.npy', gradw6);\n",
    "\n",
    "np.save(current_exp_name+'/gradp1.npy', gradp1); np.save(current_exp_name+'/gradp2.npy', gradp2); np.save(current_exp_name+'/gradp3.npy', gradp3);\n",
    "np.save(current_exp_name+'/gradp4.npy', gradp4); np.save(current_exp_name+'/gradp5.npy', gradp5); np.save(current_exp_name+'/gradp6.npy', gradp6);\n",
    "\n",
    "np.save(current_exp_name+'/gradup1.npy', gradup1); np.save(current_exp_name+'/gradup2.npy', gradup2); np.save(current_exp_name+'/gradup3.npy', gradup3);\n",
    "np.save(current_exp_name+'/gradup4.npy', gradup4); np.save(current_exp_name+'/gradup5.npy', gradup5); np.save(current_exp_name+'/gradup6.npy', gradup6);\n",
    "\n",
    "sess.close(); tf.reset_default_graph();\n",
    "\n",
    "%reset_selective -f l1,l2,l3,l4,l5,l6\n",
    "%reset_selective -f layer1,layer2,layer3,layer4,layer5,layer6\n",
    "%reset_selective -f layer1a,layer2a,layer3a,layer4a,layer5a,layer6a\n",
    "%reset_selective -f train_acc,test_acc,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a\n",
    "%reset_selective -f weight1,weight2,weight3,weight4,weight5,weight6\n",
    "%reset_selective -f gradw1,gradw2,gradw3,gradw4,gradw5,gradw6\n",
    "%reset_selective -f gradp1,gradp2,gradp3,gradp4,gradp5,gradp6\n",
    "%reset_selective -f gradup1,gradup2,gradup3,gradup4,gradup5,gradup6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-05T03:53:47.443406Z",
     "start_time": "2019-01-05T03:04:25.541597Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current : 0 Train Acc : 0.13880000250041485 Test Acc : 0.14112500302493572\n",
      "\n",
      "Current : 1 Train Acc : 0.16340000315010547 Test Acc : 0.1870000028796494\n",
      "\n",
      "Current : 2 Train Acc : 0.18060000301897525 Test Acc : 0.20100000320002437\n",
      "\n",
      "Current : 3 Train Acc : 0.19960000337660314 Test Acc : 0.2270000031311065\n",
      "\n",
      "Current : 4 Train Acc : 0.2302000036239624 Test Acc : 0.25800000298768283\n",
      "\n",
      "Current : 5 Train Acc : 0.2560000031143427 Test Acc : 0.27962500222027303\n",
      "\n",
      "Current : 6 Train Acc : 0.27820000299811365 Test Acc : 0.29662500210106374\n",
      "\n",
      "Current : 7 Train Acc : 0.29180000261962413 Test Acc : 0.3046250015497208\n",
      "\n",
      "Current : 8 Train Acc : 0.3062000022083521 Test Acc : 0.31175000201910735\n",
      "\n",
      "Current : 9 Train Acc : 0.3142000021189451 Test Acc : 0.32212500132620336\n",
      "\n",
      "Current : 10 Train Acc : 0.31940000122785567 Test Acc : 0.32762500174343584\n",
      "\n",
      "Current : 11 Train Acc : 0.3250000011920929 Test Acc : 0.3311250019259751\n",
      "\n",
      "Current : 12 Train Acc : 0.33360000175237653 Test Acc : 0.3353750019147992\n",
      "\n",
      "Current : 13 Train Acc : 0.3376000009775162 Test Acc : 0.3425000018067658\n",
      "\n",
      "Current : 14 Train Acc : 0.3416000011712313 Test Acc : 0.3462500011362135\n",
      "\n",
      "Current : 15 Train Acc : 0.34420000171661375 Test Acc : 0.3525000012014061\n",
      "\n",
      "Current : 16 Train Acc : 0.34660000133514407 Test Acc : 0.3558750010468066\n",
      "\n",
      "Current : 17 Train Acc : 0.3500000008940697 Test Acc : 0.35812500076368453\n",
      "\n",
      "Current : 18 Train Acc : 0.3538000007867813 Test Acc : 0.3615000013634562\n",
      "\n",
      "Current : 19 Train Acc : 0.3576000013947487 Test Acc : 0.36525000113993883\n",
      "\n",
      "Current : 20 Train Acc : 0.3606000010371208 Test Acc : 0.3676250011473894\n",
      "\n",
      "Current : 21 Train Acc : 0.362200001180172 Test Acc : 0.3680000012367964\n",
      "\n",
      "Current : 22 Train Acc : 0.36620000141859055 Test Acc : 0.36887500070035456\n",
      "\n",
      "Current : 23 Train Acc : 0.36760000079870225 Test Acc : 0.37037500079721214\n",
      "\n",
      "Current : 24 Train Acc : 0.37040000057220457 Test Acc : 0.3746250010840595\n",
      "\n",
      "Current : 25 Train Acc : 0.3726000007390976 Test Acc : 0.3758750015683472\n",
      "\n",
      "Current : 26 Train Acc : 0.37380000060796736 Test Acc : 0.37900000140070916\n",
      "\n",
      "Current : 27 Train Acc : 0.37560000145435335 Test Acc : 0.3801250018179417\n",
      "\n",
      "Current : 28 Train Acc : 0.3800000010728836 Test Acc : 0.3800000022165477\n",
      "\n",
      "Current : 29 Train Acc : 0.38440000075101854 Test Acc : 0.38075000237673523\n",
      "\n",
      "Current : 30 Train Acc : 0.3828000009059906 Test Acc : 0.3820000021159649\n",
      "\n",
      "Current : 31 Train Acc : 0.38520000165700913 Test Acc : 0.3835000020824373\n",
      "\n",
      "Current : 32 Train Acc : 0.38700000175833704 Test Acc : 0.3852500022388995\n",
      "\n",
      "Current : 33 Train Acc : 0.3898000013828278 Test Acc : 0.3852500023134053\n",
      "\n",
      "Current : 34 Train Acc : 0.39320000129938126 Test Acc : 0.3862500022351742\n",
      "\n",
      "Current : 35 Train Acc : 0.3952000012397766 Test Acc : 0.3886250017955899\n",
      "\n",
      "Current : 36 Train Acc : 0.3978000009059906 Test Acc : 0.3888750007376075\n",
      "\n",
      "Current : 37 Train Acc : 0.39960000073909757 Test Acc : 0.39187500078231097\n",
      "\n",
      "Current : 38 Train Acc : 0.4018000010848045 Test Acc : 0.392000000923872\n",
      "\n",
      "Current : 39 Train Acc : 0.4066000006794929 Test Acc : 0.39400000097230076\n",
      "\n",
      "Current : 40 Train Acc : 0.40900000101327894 Test Acc : 0.39462500099092723\n",
      "\n",
      "Current : 41 Train Acc : 0.4096000016927719 Test Acc : 0.39462500089779495\n",
      "\n",
      "Current : 42 Train Acc : 0.4114000017940998 Test Acc : 0.39537500085309146\n",
      "\n",
      "Current : 43 Train Acc : 0.4142000017464161 Test Acc : 0.39637500101700424\n",
      "\n",
      "Current : 44 Train Acc : 0.41520000126957896 Test Acc : 0.3971250016056001\n",
      "\n",
      "Current : 45 Train Acc : 0.4176000012457371 Test Acc : 0.39862500118091704\n",
      "\n",
      "Current : 46 Train Acc : 0.41660000187158586 Test Acc : 0.3996250008791685\n",
      "\n",
      "Current : 47 Train Acc : 0.4190000021159649 Test Acc : 0.40025000166147945\n",
      "\n",
      "Current : 48 Train Acc : 0.4206000017821789 Test Acc : 0.4012500012293458\n",
      "\n",
      "Current : 49 Train Acc : 0.42120000192523005 Test Acc : 0.40175000090152024\n",
      "\n",
      "Current : 50 Train Acc : 0.423200001180172 Test Acc : 0.40450000066310166\n",
      "\n",
      "Current : 51 Train Acc : 0.42320000153779985 Test Acc : 0.4058750007301569\n",
      "\n",
      "Current : 52 Train Acc : 0.4262000016570091 Test Acc : 0.40912500105798244\n",
      "\n",
      "Current : 53 Train Acc : 0.4260000016093254 Test Acc : 0.4106250009313226\n",
      "\n",
      "Current : 54 Train Acc : 0.4300000012516975 Test Acc : 0.411500000692904\n",
      "\n",
      "Current : 55 Train Acc : 0.43040000152587893 Test Acc : 0.413125000372529\n",
      "\n",
      "Current : 56 Train Acc : 0.43120000094175337 Test Acc : 0.41450000073760745\n",
      "\n",
      "Current : 57 Train Acc : 0.433200001001358 Test Acc : 0.41562500074505804\n",
      "\n",
      "Current : 58 Train Acc : 0.43660000067949295 Test Acc : 0.4173750008642674\n",
      "\n",
      "Current : 59 Train Acc : 0.4388000007271767 Test Acc : 0.41825000088661907\n",
      "\n",
      "Current : 60 Train Acc : 0.440800000667572 Test Acc : 0.4192500009387732\n",
      "\n",
      "Current : 61 Train Acc : 0.4398000010251999 Test Acc : 0.4182500013336539\n",
      "\n",
      "Current : 62 Train Acc : 0.44280000060796737 Test Acc : 0.4191250013932586\n",
      "\n",
      "Current : 63 Train Acc : 0.4452000012397766 Test Acc : 0.4207500013336539\n",
      "\n",
      "Current : 64 Train Acc : 0.4480000007748604 Test Acc : 0.42075000118464234\n",
      "\n",
      "Current : 65 Train Acc : 0.44880000084638594 Test Acc : 0.4230000015348196\n",
      "\n",
      "Current : 66 Train Acc : 0.4502000010609627 Test Acc : 0.42337500158697366\n",
      "\n",
      "Current : 67 Train Acc : 0.44960000067949296 Test Acc : 0.4248750017955899\n",
      "\n",
      "Current : 68 Train Acc : 0.45200000125169754 Test Acc : 0.4248750016838312\n",
      "\n",
      "Current : 69 Train Acc : 0.4526000009179115 Test Acc : 0.427125001847744\n",
      "\n",
      "Current : 70 Train Acc : 0.4564000012278557 Test Acc : 0.4291250020638108\n",
      "\n",
      "Current : 71 Train Acc : 0.4584000009894371 Test Acc : 0.42850000157952306\n",
      "\n",
      "Current : 72 Train Acc : 0.4608000010251999 Test Acc : 0.4313750014081597\n",
      "\n",
      "Current : 73 Train Acc : 0.4628000012040138 Test Acc : 0.43050000112503767\n",
      "\n",
      "Current : 74 Train Acc : 0.46360000103712085 Test Acc : 0.4311250016838312\n",
      "\n",
      "Current : 75 Train Acc : 0.4656000008583069 Test Acc : 0.4335000008717179\n",
      "\n",
      "Current : 76 Train Acc : 0.4670000010728836 Test Acc : 0.43300000108778475\n",
      "\n",
      "Current : 77 Train Acc : 0.4692000012397766 Test Acc : 0.4330000014603138\n",
      "\n",
      "Current : 78 Train Acc : 0.4704000011086464 Test Acc : 0.4347500013932586\n",
      "\n",
      "Current : 79 Train Acc : 0.4726000010371208 Test Acc : 0.4326250008121133\n",
      "\n",
      "Current : 80 Train Acc : 0.4736000009775162 Test Acc : 0.43350000090897084\n",
      "\n",
      "Current : 81 Train Acc : 0.4744000012278557 Test Acc : 0.43375000085681675\n",
      "\n",
      "Current : 82 Train Acc : 0.4752000009417534 Test Acc : 0.4340000007674098\n",
      "\n",
      "Current : 83 Train Acc : 0.47600000029802325 Test Acc : 0.4333750006183982\n",
      "\n",
      "Current : 84 Train Acc : 0.47720000010728836 Test Acc : 0.4338750008493662\n",
      "\n",
      "Current : 85 Train Acc : 0.4784000009894371 Test Acc : 0.43275000125169755\n",
      "\n",
      "Current : 86 Train Acc : 0.4798000009059906 Test Acc : 0.431500001065433\n",
      "\n",
      "Current : 87 Train Acc : 0.48080000120401384 Test Acc : 0.4323750013113022\n",
      "\n",
      "Current : 88 Train Acc : 0.4806000011563301 Test Acc : 0.43387500125914813\n",
      "\n",
      "Current : 89 Train Acc : 0.483600001513958 Test Acc : 0.43525000140070913\n",
      "\n",
      "Current : 90 Train Acc : 0.48540000092983243 Test Acc : 0.43450000148266554\n",
      "\n",
      "Current : 91 Train Acc : 0.4848000011444092 Test Acc : 0.4360000015050173\n",
      "\n",
      "Current : 92 Train Acc : 0.48520000141859054 Test Acc : 0.437375001385808\n",
      "\n",
      "Current : 93 Train Acc : 0.4878000007867813 Test Acc : 0.4372500014677644\n",
      "\n",
      "Current : 94 Train Acc : 0.48860000133514403 Test Acc : 0.4383750018104911\n",
      "\n",
      "Current : 95 Train Acc : 0.48920000112056733 Test Acc : 0.4387500014528632\n",
      "\n",
      "Current : 96 Train Acc : 0.48920000088214877 Test Acc : 0.44100000195205213\n",
      "\n",
      "Current : 97 Train Acc : 0.4896000007390976 Test Acc : 0.4422500015422702\n",
      "\n",
      "Current : 98 Train Acc : 0.49100000131130217 Test Acc : 0.44287500232458116\n",
      "\n",
      "Current : 99 Train Acc : 0.4930000015497208 Test Acc : 0.4442500020563602\n",
      "\n",
      "Current : 100 Train Acc : 0.4942000013589859 Test Acc : 0.4455000023543835\n",
      "\n",
      "Current : 101 Train Acc : 0.4942000005245209 Test Acc : 0.4463750017806888\n",
      "\n",
      "Current : 102 Train Acc : 0.49559999990463255 Test Acc : 0.4483750020712614\n",
      "\n",
      "Current : 103 Train Acc : 0.4939999998807907 Test Acc : 0.44950000163167714\n",
      "\n",
      "Current : 104 Train Acc : 0.49639999914169314 Test Acc : 0.4505000017955899\n",
      "\n",
      "Current : 105 Train Acc : 0.4966000002622604 Test Acc : 0.44987500172108413\n",
      "\n",
      "Current : 106 Train Acc : 0.4980000005364418 Test Acc : 0.450000001527369\n",
      "\n",
      "Current : 107 Train Acc : 0.4990000006556511 Test Acc : 0.4508750016987324\n",
      "\n",
      "Current : 108 Train Acc : 0.49960000038146973 Test Acc : 0.4516250018030405\n",
      "\n",
      "Current : 109 Train Acc : 0.5014000002741814 Test Acc : 0.45137500151991844\n",
      "\n",
      "Current : 110 Train Acc : 0.5022000005841255 Test Acc : 0.45087500136345626\n",
      "\n",
      "Current : 111 Train Acc : 0.5018000004887581 Test Acc : 0.45150000099092724\n",
      "\n",
      "Current : 112 Train Acc : 0.5032000007033348 Test Acc : 0.45175000112503766\n",
      "\n",
      "Current : 113 Train Acc : 0.5046000009179116 Test Acc : 0.4507500008493662\n",
      "\n",
      "Current : 114 Train Acc : 0.5054000008702279 Test Acc : 0.44950000125914813\n",
      "\n",
      "Current : 115 Train Acc : 0.5072000010609626 Test Acc : 0.44800000105053184\n",
      "\n",
      "Current : 116 Train Acc : 0.507600000679493 Test Acc : 0.4502500008791685\n",
      "\n",
      "Current : 117 Train Acc : 0.5102000015377999 Test Acc : 0.4508750008791685\n",
      "\n",
      "Current : 118 Train Acc : 0.5120000014901162 Test Acc : 0.4506250008568168\n",
      "\n",
      "Current : 119 Train Acc : 0.5110000008940697 Test Acc : 0.45112500082701446\n",
      "\n",
      "Current : 120 Train Acc : 0.5142000010609626 Test Acc : 0.4507500008493662\n",
      "\n",
      "Current : 121 Train Acc : 0.5154000014662743 Test Acc : 0.45287500131875275\n",
      "\n",
      "Current : 122 Train Acc : 0.5160000016093254 Test Acc : 0.4536250013485551\n",
      "\n",
      "Current : 123 Train Acc : 0.5166000009775161 Test Acc : 0.45150000110268595\n",
      "\n",
      "Current : 124 Train Acc : 0.5178000018000603 Test Acc : 0.4557500006631017\n",
      "\n",
      "Current : 125 Train Acc : 0.5162000026106834 Test Acc : 0.45437500029802325\n",
      "\n",
      "Current : 126 Train Acc : 0.5172000016570091 Test Acc : 0.45412500072270634\n",
      "\n",
      "Current : 127 Train Acc : 0.516400001347065 Test Acc : 0.45312500115484\n",
      "\n",
      "Current : 128 Train Acc : 0.5176000017523765 Test Acc : 0.45300000097602605\n",
      "\n",
      "Current : 129 Train Acc : 0.5184000014662743 Test Acc : 0.4532500008121133\n",
      "\n",
      "Current : 130 Train Acc : 0.5208000016808509 Test Acc : 0.4570000013336539\n",
      "\n",
      "Current : 131 Train Acc : 0.520600002348423 Test Acc : 0.4590000012144446\n",
      "\n",
      "Current : 132 Train Acc : 0.5222000017762184 Test Acc : 0.4575000008195639\n",
      "\n",
      "Current : 133 Train Acc : 0.5226000017523765 Test Acc : 0.45837500106543305\n",
      "\n",
      "Current : 134 Train Acc : 0.5226000010967254 Test Acc : 0.459500000551343\n",
      "\n",
      "Current : 135 Train Acc : 0.5236000016331672 Test Acc : 0.45987500105053186\n",
      "\n",
      "Current : 136 Train Acc : 0.5256000013947487 Test Acc : 0.4607500011846423\n",
      "\n",
      "Current : 137 Train Acc : 0.5276000013947487 Test Acc : 0.46100000105798244\n",
      "\n",
      "Current : 138 Train Acc : 0.527600001513958 Test Acc : 0.4656250007264316\n",
      "\n",
      "Current : 139 Train Acc : 0.5274000014662743 Test Acc : 0.4645000007376075\n",
      "\n",
      "Current : 140 Train Acc : 0.5284000014662743 Test Acc : 0.46462500052526595\n",
      "\n",
      "Current : 141 Train Acc : 0.5298000014424324 Test Acc : 0.4645000002719462\n",
      "\n",
      "Current : 142 Train Acc : 0.5296000009179115 Test Acc : 0.4650000005401671\n",
      "\n",
      "Current : 143 Train Acc : 0.530200001001358 Test Acc : 0.4658750001899898\n",
      "\n",
      "Current : 144 Train Acc : 0.5324000009298324 Test Acc : 0.46737500010058286\n",
      "\n",
      "Current : 145 Train Acc : 0.5346000014543534 Test Acc : 0.4671250003390014\n",
      "\n",
      "Current : 146 Train Acc : 0.5362000014781952 Test Acc : 0.46675000060349703\n",
      "\n",
      "Current : 147 Train Acc : 0.5382000019550324 Test Acc : 0.46725000102072956\n",
      "\n",
      "Current : 148 Train Acc : 0.5412000013589859 Test Acc : 0.469125000257045\n",
      "\n",
      "Current : 149 Train Acc : 0.5398000005483627 Test Acc : 0.46937500055879355\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# B\n",
    "current_exp_name = 'B';\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "# create layers\n",
    "l1 = CNN(3,3, 16,which_reg=current_exp_name); \n",
    "l2 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l3 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "\n",
    "l4 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l5 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l6 = CNN(3,16,10,which_reg=current_exp_name); \n",
    "\n",
    "# 2. graph \n",
    "x = tf.placeholder(tf.float32,(batch_size,96,96,3))\n",
    "y = tf.placeholder(tf.float32,(batch_size,10))\n",
    "\n",
    "layer1, layer1a = l1. feedforward(x,stride=2)\n",
    "layer2, layer2a = l2. feedforward(layer1a,stride=2)\n",
    "layer3, layer3a = l3. feedforward(layer2a,stride=2)\n",
    "layer4, layer4a = l4. feedforward(layer3a,stride=2)\n",
    "layer5, layer5a = l5. feedforward(layer4a)\n",
    "layer6, layer6a = l6. feedforward(layer5a)\n",
    "\n",
    "final_layer   = tf.reduce_mean(layer6a,(1,2))\n",
    "final_softmax = tf_softmax(final_layer)\n",
    "cost          = -tf.reduce_mean(y * tf.log(final_softmax + 1e-8))\n",
    "correct_prediction = tf.equal(tf.argmax(final_softmax, 1), tf.argmax(y, 1))\n",
    "accuracy           = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "gradient = tf.tile((final_softmax-y)[:,None,None,:],[1,6,6,1])/batch_size\n",
    "grad6p,grad6w,grad6_up = l6.backprop(gradient)\n",
    "grad5p,grad5w,grad5_up = l5.backprop(grad6p)\n",
    "grad4p,grad4w,grad4_up = l4.backprop(grad5p,stride=2)\n",
    "grad3p,grad3w,grad3_up = l3.backprop(grad4p,stride=2)\n",
    "grad2p,grad2w,grad2_up = l2.backprop(grad3p,stride=2)\n",
    "grad1p,grad1w,grad1_up = l1.backprop(grad2p,stride=2)\n",
    "\n",
    "gradient_update = grad6_up + grad5_up + grad4_up + grad3_up + grad2_up + grad1_up \n",
    "\n",
    "# train\n",
    "sess.run(tf.global_variables_initializer())\n",
    "avg_acc_train = 0; avg_acc_test  = 0; train_acc = [];test_acc = []\n",
    "\n",
    "# mean std skew kurt non-zero\n",
    "llayer1 = [[],[],[],[],[]]; llayer2 = [[],[],[],[],[]]; llayer3 = [[],[],[],[],[]]\n",
    "llayer4 = [[],[],[],[],[]]; llayer5 = [[],[],[],[],[]]; llayer6 = [[],[],[],[],[]]\n",
    "\n",
    "llayer1a = [[],[],[],[],[]]; llayer2a = [[],[],[],[],[]]; llayer3a = [[],[],[],[],[]]\n",
    "llayer4a = [[],[],[],[],[]]; llayer5a = [[],[],[],[],[]]; llayer6a = [[],[],[],[],[]]\n",
    "\n",
    "weight1 = [[],[],[],[],[]]; weight2 = [[],[],[],[],[]]; weight3 = [[],[],[],[],[]];\n",
    "weight4 = [[],[],[],[],[]]; weight5 = [[],[],[],[],[]]; weight6 = [[],[],[],[],[]];\n",
    "\n",
    "gradw1  = [[],[],[],[],[]]; gradw2  = [[],[],[],[],[]]; gradw3  = [[],[],[],[],[]];\n",
    "gradw4  = [[],[],[],[],[]]; gradw5  = [[],[],[],[],[]]; gradw6  = [[],[],[],[],[]];\n",
    "\n",
    "gradp1  = [[],[],[],[],[]]; gradp2  = [[],[],[],[],[]]; gradp3  = [[],[],[],[],[]];\n",
    "gradp4  = [[],[],[],[],[]]; gradp5  = [[],[],[],[],[]]; gradp6  = [[],[],[],[],[]];\n",
    "\n",
    "gradup1  = [[],[],[],[],[]]; gradup2  = [[],[],[],[],[]]; gradup3  = [[],[],[],[],[]];\n",
    "gradup4  = [[],[],[],[],[]]; gradup5  = [[],[],[],[],[]]; gradup6  = [[],[],[],[],[]];\n",
    "\n",
    "list_of_outputs = [\n",
    "    layer1,layer2,layer3,layer4,layer5,layer6,\n",
    "    layer1a,layer2a,layer3a,layer4a,layer5a,layer6a,\n",
    "    l1.getw(),l2.getw(),l3.getw(),l4.getw(),l5.getw(),l6.getw(),\n",
    "    grad1w,grad2w,grad3w,grad4w,grad5w,grad6w,\n",
    "    grad1p,grad2p,grad3p,grad4p,grad5p,grad6p,\n",
    "    grad1_up[0],grad2_up[0],grad3_up[0],grad4_up[0],grad5_up[0],grad6_up[0]\n",
    "]\n",
    "\n",
    "for iter in range(num_epoch):\n",
    "\n",
    "    # Training Accuracy    \n",
    "    for current_batch_index in range(0,len(train_images),batch_size):\n",
    "        current_data  = train_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = train_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy,gradient_update],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(train_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_train = avg_acc_train + sess_results[0]\n",
    "        \n",
    "    # get the results\n",
    "    mid_stat = sess.run(list_of_outputs,feed_dict={x:current_data,y:current_label})\n",
    "    \n",
    "    # Test Accuracy    \n",
    "    for current_batch_index in range(0,len(test_images), batch_size):\n",
    "        current_data  = test_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = test_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(test_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_test = avg_acc_test + sess_results[0]   \n",
    "        \n",
    "    # ======================== extract stats ========================\n",
    "    llayer1 = append_stat(llayer1,mid_stat,0);  llayer2 = append_stat(llayer2,mid_stat,1);  llayer3 = append_stat(llayer3,mid_stat,2);\n",
    "    llayer4 = append_stat(llayer4,mid_stat,3);  llayer5 = append_stat(llayer5,mid_stat,4);  llayer6 = append_stat(llayer6,mid_stat,5);\n",
    "\n",
    "    llayer1a = append_stat(llayer1a,mid_stat,6);  llayer2a = append_stat(llayer2a,mid_stat,7);  llayer3a = append_stat(llayer3a,mid_stat,8);\n",
    "    llayer4a = append_stat(llayer4a,mid_stat,9);  llayer5a = append_stat(llayer5a,mid_stat,10); llayer6a = append_stat(llayer6a,mid_stat,11);\n",
    "    \n",
    "    weight1 = append_stat(weight1,mid_stat,12);  weight2 = append_stat(weight2,mid_stat,13);  weight3 = append_stat(weight3,mid_stat,14);\n",
    "    weight4 = append_stat(weight4,mid_stat,15);  weight5 = append_stat(weight5,mid_stat,16);  weight6 = append_stat(weight6,mid_stat,17);\n",
    "    \n",
    "    gradw1 = append_stat(gradw1,mid_stat,18); gradw2 = append_stat(gradw2,mid_stat,19); gradw3 = append_stat(gradw3,mid_stat,20);\n",
    "    gradw4 = append_stat(gradw4,mid_stat,21); gradw5 = append_stat(gradw5,mid_stat,22); gradw6 = append_stat(gradw6,mid_stat,23);\n",
    "    \n",
    "    gradp1 = append_stat(gradp1,mid_stat,24); gradp2 = append_stat(gradp2,mid_stat,25); gradp3 = append_stat(gradp3,mid_stat,26);\n",
    "    gradp4 = append_stat(gradp4,mid_stat,27); gradp5 = append_stat(gradp5,mid_stat,28); gradp6 = append_stat(gradp6,mid_stat,29);\n",
    "\n",
    "    gradup1 = append_stat(gradup1,mid_stat,30); gradup2 = append_stat(gradup2,mid_stat,31); gradup3 = append_stat(gradup3,mid_stat,32);\n",
    "    gradup4 = append_stat(gradup4,mid_stat,33); gradup5 = append_stat(gradup5,mid_stat,34); gradup6 = append_stat(gradup6,mid_stat,35);\n",
    "\n",
    "    train_acc.append(avg_acc_train/(len(train_images)/batch_size))\n",
    "    test_acc .append(avg_acc_test / (len(test_images)/batch_size))\n",
    "    # ======================== extract stats ========================\n",
    "    \n",
    "    # ======================== save to image ========================\n",
    "    save_to_image(mid_stat[0:6]   ,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,\"layer\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[6:12]  ,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a,\"layera\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[12:18] ,weight1,weight2,weight3,weight4,weight5,weight6,\"weights\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[18:24] ,gradw1,gradw2,gradw3,gradw4,gradw5,gradw6,\"gradientw\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[24:30] ,gradp1,gradp2,gradp3,gradp4,gradp5,gradp6,\"gradientp\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[30:36] ,gradup1,gradup2,gradup3,gradup4,gradup5,gradup6,\"moment\",train_acc,test_acc,current_exp_name,iter)\n",
    "    # ======================== save to image ========================\n",
    "        \n",
    "    # ======================== print reset ========================\n",
    "    print(\"Current : \"+ str(iter) + \" Train Acc : \" + str(avg_acc_train/(len(train_images)/batch_size)) + \" Test Acc : \" + str(avg_acc_test/(len(test_images)/batch_size)) + '\\n')\n",
    "    avg_acc_train = 0 ; avg_acc_test  = 0\n",
    "    # ======================== print reset ========================\n",
    "\n",
    "np.save(current_exp_name+'/train_acc.npy',train_acc); np.save(current_exp_name+'/test_acc.npy', test_acc)    \n",
    "np.save(current_exp_name+'/llayer1.npy', llayer1);  np.save(current_exp_name+'/llayer2.npy', llayer2);  np.save(current_exp_name+'/llayer3.npy', llayer3); \n",
    "np.save(current_exp_name+'/llayer4.npy', llayer4);  np.save(current_exp_name+'/llayer5.npy', llayer5);  np.save(current_exp_name+'/llayer6.npy', llayer6); \n",
    "\n",
    "np.save(current_exp_name+'/llayer1a.npy', llayer1a);  np.save(current_exp_name+'/llayer2a.npy', llayer2a);  np.save(current_exp_name+'/llayer3a.npy', llayer3a); \n",
    "np.save(current_exp_name+'/llayer4a.npy', llayer4a);  np.save(current_exp_name+'/llayer5a.npy', llayer5a);  np.save(current_exp_name+'/llayer6a.npy', llayer6a); \n",
    "\n",
    "np.save(current_exp_name+'/weight1.npy', weight1);  np.save(current_exp_name+'/weight2.npy', weight2);  np.save(current_exp_name+'/weight3.npy', weight3);  \n",
    "np.save(current_exp_name+'/weight4.npy', weight4);  np.save(current_exp_name+'/weight5.npy', weight5);  np.save(current_exp_name+'/weight6.npy', weight6);  \n",
    "\n",
    "np.save(current_exp_name+'/gradw1.npy', gradw1); np.save(current_exp_name+'/gradw2.npy', gradw2); np.save(current_exp_name+'/gradw3.npy', gradw3);\n",
    "np.save(current_exp_name+'/gradw4.npy', gradw4); np.save(current_exp_name+'/gradw5.npy', gradw5); np.save(current_exp_name+'/gradw6.npy', gradw6);\n",
    "\n",
    "np.save(current_exp_name+'/gradp1.npy', gradp1); np.save(current_exp_name+'/gradp2.npy', gradp2); np.save(current_exp_name+'/gradp3.npy', gradp3);\n",
    "np.save(current_exp_name+'/gradp4.npy', gradp4); np.save(current_exp_name+'/gradp5.npy', gradp5); np.save(current_exp_name+'/gradp6.npy', gradp6);\n",
    "\n",
    "np.save(current_exp_name+'/gradup1.npy', gradup1); np.save(current_exp_name+'/gradup2.npy', gradup2); np.save(current_exp_name+'/gradup3.npy', gradup3);\n",
    "np.save(current_exp_name+'/gradup4.npy', gradup4); np.save(current_exp_name+'/gradup5.npy', gradup5); np.save(current_exp_name+'/gradup6.npy', gradup6);\n",
    "\n",
    "sess.close(); tf.reset_default_graph();\n",
    "\n",
    "%reset_selective -f l1,l2,l3,l4,l5,l6\n",
    "%reset_selective -f layer1,layer2,layer3,layer4,layer5,layer6\n",
    "%reset_selective -f layer1a,layer2a,layer3a,layer4a,layer5a,layer6a\n",
    "%reset_selective -f train_acc,test_acc,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a\n",
    "%reset_selective -f weight1,weight2,weight3,weight4,weight5,weight6\n",
    "%reset_selective -f gradw1,gradw2,gradw3,gradw4,gradw5,gradw6\n",
    "%reset_selective -f gradp1,gradp2,gradp3,gradp4,gradp5,gradp6\n",
    "%reset_selective -f gradup1,gradup2,gradup3,gradup4,gradup5,gradup6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-05T04:46:24.454272Z",
     "start_time": "2019-01-05T03:53:47.474382Z"
    },
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current : 0 Train Acc : 0.10000000189244747 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 1 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 2 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 3 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 4 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 5 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 6 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 7 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 8 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 9 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 10 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 11 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 12 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 13 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 14 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 15 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 16 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 17 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 18 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 19 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 20 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 21 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 22 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 23 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 24 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 25 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 26 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 27 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 28 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 29 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 30 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 31 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 32 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 33 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 34 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 35 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 36 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 37 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 38 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 39 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 40 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 41 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 42 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 43 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 44 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 45 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 46 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 47 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 48 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 49 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 50 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 51 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 52 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 53 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 54 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 55 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 56 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 57 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 58 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 59 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 60 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 61 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 62 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 63 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 64 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 65 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 66 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 67 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 68 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 69 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 70 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 71 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 72 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 73 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 74 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 75 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 76 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 77 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 78 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 79 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 80 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 81 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 82 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 83 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 84 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 85 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 86 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 87 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 88 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 89 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 90 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 91 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 92 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 93 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 94 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 95 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 96 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 97 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 98 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 99 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 100 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 101 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 102 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 103 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 104 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 105 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 106 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 107 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current : 108 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 109 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 110 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 111 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 112 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 113 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 114 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 115 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 116 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 117 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 118 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 119 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 120 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 121 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 122 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 123 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 124 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 125 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 126 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 127 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 128 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 129 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 130 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 131 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 132 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 133 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 134 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 135 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 136 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 137 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 138 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 139 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 140 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 141 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 142 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 143 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 144 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 145 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 146 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 147 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 148 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n",
      "Current : 149 Train Acc : 0.10000000190734863 Test Acc : 0.1000000020954758\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# C\n",
    "current_exp_name = 'C';\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "# create layers\n",
    "l1 = CNN(3,3, 16,which_reg=current_exp_name); \n",
    "l2 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l3 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "\n",
    "l4 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l5 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l6 = CNN(3,16,10,which_reg=current_exp_name); \n",
    "\n",
    "# 2. graph \n",
    "x = tf.placeholder(tf.float32,(batch_size,96,96,3))\n",
    "y = tf.placeholder(tf.float32,(batch_size,10))\n",
    "\n",
    "layer1, layer1a = l1. feedforward(x,stride=2)\n",
    "layer2, layer2a = l2. feedforward(layer1a,stride=2)\n",
    "layer3, layer3a = l3. feedforward(layer2a,stride=2)\n",
    "layer4, layer4a = l4. feedforward(layer3a,stride=2)\n",
    "layer5, layer5a = l5. feedforward(layer4a)\n",
    "layer6, layer6a = l6. feedforward(layer5a)\n",
    "\n",
    "final_layer   = tf.reduce_mean(layer6a,(1,2))\n",
    "final_softmax = tf_softmax(final_layer)\n",
    "cost          = -tf.reduce_mean(y * tf.log(final_softmax + 1e-8))\n",
    "correct_prediction = tf.equal(tf.argmax(final_softmax, 1), tf.argmax(y, 1))\n",
    "accuracy           = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "gradient = tf.tile((final_softmax-y)[:,None,None,:],[1,6,6,1])/batch_size\n",
    "grad6p,grad6w,grad6_up = l6.backprop(gradient)\n",
    "grad5p,grad5w,grad5_up = l5.backprop(grad6p)\n",
    "grad4p,grad4w,grad4_up = l4.backprop(grad5p,stride=2)\n",
    "grad3p,grad3w,grad3_up = l3.backprop(grad4p,stride=2)\n",
    "grad2p,grad2w,grad2_up = l2.backprop(grad3p,stride=2)\n",
    "grad1p,grad1w,grad1_up = l1.backprop(grad2p,stride=2)\n",
    "\n",
    "gradient_update = grad6_up + grad5_up + grad4_up + grad3_up + grad2_up + grad1_up \n",
    "\n",
    "# train\n",
    "sess.run(tf.global_variables_initializer())\n",
    "avg_acc_train = 0; avg_acc_test  = 0; train_acc = [];test_acc = []\n",
    "\n",
    "# mean std skew kurt non-zero\n",
    "llayer1 = [[],[],[],[],[]]; llayer2 = [[],[],[],[],[]]; llayer3 = [[],[],[],[],[]]\n",
    "llayer4 = [[],[],[],[],[]]; llayer5 = [[],[],[],[],[]]; llayer6 = [[],[],[],[],[]]\n",
    "\n",
    "llayer1a = [[],[],[],[],[]]; llayer2a = [[],[],[],[],[]]; llayer3a = [[],[],[],[],[]]\n",
    "llayer4a = [[],[],[],[],[]]; llayer5a = [[],[],[],[],[]]; llayer6a = [[],[],[],[],[]]\n",
    "\n",
    "weight1 = [[],[],[],[],[]]; weight2 = [[],[],[],[],[]]; weight3 = [[],[],[],[],[]];\n",
    "weight4 = [[],[],[],[],[]]; weight5 = [[],[],[],[],[]]; weight6 = [[],[],[],[],[]];\n",
    "\n",
    "gradw1  = [[],[],[],[],[]]; gradw2  = [[],[],[],[],[]]; gradw3  = [[],[],[],[],[]];\n",
    "gradw4  = [[],[],[],[],[]]; gradw5  = [[],[],[],[],[]]; gradw6  = [[],[],[],[],[]];\n",
    "\n",
    "gradp1  = [[],[],[],[],[]]; gradp2  = [[],[],[],[],[]]; gradp3  = [[],[],[],[],[]];\n",
    "gradp4  = [[],[],[],[],[]]; gradp5  = [[],[],[],[],[]]; gradp6  = [[],[],[],[],[]];\n",
    "\n",
    "gradup1  = [[],[],[],[],[]]; gradup2  = [[],[],[],[],[]]; gradup3  = [[],[],[],[],[]];\n",
    "gradup4  = [[],[],[],[],[]]; gradup5  = [[],[],[],[],[]]; gradup6  = [[],[],[],[],[]];\n",
    "\n",
    "list_of_outputs = [\n",
    "    layer1,layer2,layer3,layer4,layer5,layer6,\n",
    "    layer1a,layer2a,layer3a,layer4a,layer5a,layer6a,\n",
    "    l1.getw(),l2.getw(),l3.getw(),l4.getw(),l5.getw(),l6.getw(),\n",
    "    grad1w,grad2w,grad3w,grad4w,grad5w,grad6w,\n",
    "    grad1p,grad2p,grad3p,grad4p,grad5p,grad6p,\n",
    "    grad1_up[0],grad2_up[0],grad3_up[0],grad4_up[0],grad5_up[0],grad6_up[0]\n",
    "]\n",
    "\n",
    "for iter in range(num_epoch):\n",
    "\n",
    "    # Training Accuracy    \n",
    "    for current_batch_index in range(0,len(train_images),batch_size):\n",
    "        current_data  = train_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = train_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy,gradient_update],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(train_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_train = avg_acc_train + sess_results[0]\n",
    "        \n",
    "    # get the results\n",
    "    mid_stat = sess.run(list_of_outputs,feed_dict={x:current_data,y:current_label})\n",
    "    \n",
    "    # Test Accuracy    \n",
    "    for current_batch_index in range(0,len(test_images), batch_size):\n",
    "        current_data  = test_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = test_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(test_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_test = avg_acc_test + sess_results[0]   \n",
    "        \n",
    "    # ======================== extract stats ========================\n",
    "    llayer1 = append_stat(llayer1,mid_stat,0);  llayer2 = append_stat(llayer2,mid_stat,1);  llayer3 = append_stat(llayer3,mid_stat,2);\n",
    "    llayer4 = append_stat(llayer4,mid_stat,3);  llayer5 = append_stat(llayer5,mid_stat,4);  llayer6 = append_stat(llayer6,mid_stat,5);\n",
    "\n",
    "    llayer1a = append_stat(llayer1a,mid_stat,6);  llayer2a = append_stat(llayer2a,mid_stat,7);  llayer3a = append_stat(llayer3a,mid_stat,8);\n",
    "    llayer4a = append_stat(llayer4a,mid_stat,9);  llayer5a = append_stat(llayer5a,mid_stat,10); llayer6a = append_stat(llayer6a,mid_stat,11);\n",
    "    \n",
    "    weight1 = append_stat(weight1,mid_stat,12);  weight2 = append_stat(weight2,mid_stat,13);  weight3 = append_stat(weight3,mid_stat,14);\n",
    "    weight4 = append_stat(weight4,mid_stat,15);  weight5 = append_stat(weight5,mid_stat,16);  weight6 = append_stat(weight6,mid_stat,17);\n",
    "    \n",
    "    gradw1 = append_stat(gradw1,mid_stat,18); gradw2 = append_stat(gradw2,mid_stat,19); gradw3 = append_stat(gradw3,mid_stat,20);\n",
    "    gradw4 = append_stat(gradw4,mid_stat,21); gradw5 = append_stat(gradw5,mid_stat,22); gradw6 = append_stat(gradw6,mid_stat,23);\n",
    "    \n",
    "    gradp1 = append_stat(gradp1,mid_stat,24); gradp2 = append_stat(gradp2,mid_stat,25); gradp3 = append_stat(gradp3,mid_stat,26);\n",
    "    gradp4 = append_stat(gradp4,mid_stat,27); gradp5 = append_stat(gradp5,mid_stat,28); gradp6 = append_stat(gradp6,mid_stat,29);\n",
    "\n",
    "    gradup1 = append_stat(gradup1,mid_stat,30); gradup2 = append_stat(gradup2,mid_stat,31); gradup3 = append_stat(gradup3,mid_stat,32);\n",
    "    gradup4 = append_stat(gradup4,mid_stat,33); gradup5 = append_stat(gradup5,mid_stat,34); gradup6 = append_stat(gradup6,mid_stat,35);\n",
    "\n",
    "    train_acc.append(avg_acc_train/(len(train_images)/batch_size))\n",
    "    test_acc .append(avg_acc_test / (len(test_images)/batch_size))\n",
    "    # ======================== extract stats ========================\n",
    "    \n",
    "    # ======================== save to image ========================\n",
    "    save_to_image(mid_stat[0:6]   ,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,\"layer\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[6:12]  ,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a,\"layera\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[12:18] ,weight1,weight2,weight3,weight4,weight5,weight6,\"weights\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[18:24] ,gradw1,gradw2,gradw3,gradw4,gradw5,gradw6,\"gradientw\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[24:30] ,gradp1,gradp2,gradp3,gradp4,gradp5,gradp6,\"gradientp\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[30:36] ,gradup1,gradup2,gradup3,gradup4,gradup5,gradup6,\"moment\",train_acc,test_acc,current_exp_name,iter)\n",
    "    # ======================== save to image ========================\n",
    "        \n",
    "    # ======================== print reset ========================\n",
    "    print(\"Current : \"+ str(iter) + \" Train Acc : \" + str(avg_acc_train/(len(train_images)/batch_size)) + \" Test Acc : \" + str(avg_acc_test/(len(test_images)/batch_size)) + '\\n')\n",
    "    avg_acc_train = 0 ; avg_acc_test  = 0\n",
    "    # ======================== print reset ========================\n",
    "\n",
    "np.save(current_exp_name+'/train_acc.npy',train_acc); np.save(current_exp_name+'/test_acc.npy', test_acc)    \n",
    "np.save(current_exp_name+'/llayer1.npy', llayer1);  np.save(current_exp_name+'/llayer2.npy', llayer2);  np.save(current_exp_name+'/llayer3.npy', llayer3); \n",
    "np.save(current_exp_name+'/llayer4.npy', llayer4);  np.save(current_exp_name+'/llayer5.npy', llayer5);  np.save(current_exp_name+'/llayer6.npy', llayer6); \n",
    "\n",
    "np.save(current_exp_name+'/llayer1a.npy', llayer1a);  np.save(current_exp_name+'/llayer2a.npy', llayer2a);  np.save(current_exp_name+'/llayer3a.npy', llayer3a); \n",
    "np.save(current_exp_name+'/llayer4a.npy', llayer4a);  np.save(current_exp_name+'/llayer5a.npy', llayer5a);  np.save(current_exp_name+'/llayer6a.npy', llayer6a); \n",
    "\n",
    "np.save(current_exp_name+'/weight1.npy', weight1);  np.save(current_exp_name+'/weight2.npy', weight2);  np.save(current_exp_name+'/weight3.npy', weight3);  \n",
    "np.save(current_exp_name+'/weight4.npy', weight4);  np.save(current_exp_name+'/weight5.npy', weight5);  np.save(current_exp_name+'/weight6.npy', weight6);  \n",
    "\n",
    "np.save(current_exp_name+'/gradw1.npy', gradw1); np.save(current_exp_name+'/gradw2.npy', gradw2); np.save(current_exp_name+'/gradw3.npy', gradw3);\n",
    "np.save(current_exp_name+'/gradw4.npy', gradw4); np.save(current_exp_name+'/gradw5.npy', gradw5); np.save(current_exp_name+'/gradw6.npy', gradw6);\n",
    "\n",
    "np.save(current_exp_name+'/gradp1.npy', gradp1); np.save(current_exp_name+'/gradp2.npy', gradp2); np.save(current_exp_name+'/gradp3.npy', gradp3);\n",
    "np.save(current_exp_name+'/gradp4.npy', gradp4); np.save(current_exp_name+'/gradp5.npy', gradp5); np.save(current_exp_name+'/gradp6.npy', gradp6);\n",
    "\n",
    "np.save(current_exp_name+'/gradup1.npy', gradup1); np.save(current_exp_name+'/gradup2.npy', gradup2); np.save(current_exp_name+'/gradup3.npy', gradup3);\n",
    "np.save(current_exp_name+'/gradup4.npy', gradup4); np.save(current_exp_name+'/gradup5.npy', gradup5); np.save(current_exp_name+'/gradup6.npy', gradup6);\n",
    "\n",
    "sess.close(); tf.reset_default_graph();\n",
    "\n",
    "%reset_selective -f l1,l2,l3,l4,l5,l6\n",
    "%reset_selective -f layer1,layer2,layer3,layer4,layer5,layer6\n",
    "%reset_selective -f layer1a,layer2a,layer3a,layer4a,layer5a,layer6a\n",
    "%reset_selective -f train_acc,test_acc,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a\n",
    "%reset_selective -f weight1,weight2,weight3,weight4,weight5,weight6\n",
    "%reset_selective -f gradw1,gradw2,gradw3,gradw4,gradw5,gradw6\n",
    "%reset_selective -f gradp1,gradp2,gradp3,gradp4,gradp5,gradp6\n",
    "%reset_selective -f gradup1,gradup2,gradup3,gradup4,gradup5,gradup6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-05T05:35:39.460841Z",
     "start_time": "2019-01-05T04:46:24.478435Z"
    },
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current : 0 Train Acc : 0.12040000247955322 Test Acc : 0.16450000292621553\n",
      "\n",
      "Current : 1 Train Acc : 0.16380000235140324 Test Acc : 0.20400000305846333\n",
      "\n",
      "Current : 2 Train Acc : 0.1922000028192997 Test Acc : 0.21750000334344805\n",
      "\n",
      "Current : 3 Train Acc : 0.21020000356435775 Test Acc : 0.23487500294111668\n",
      "\n",
      "Current : 4 Train Acc : 0.230600002810359 Test Acc : 0.24850000254809856\n",
      "\n",
      "Current : 5 Train Acc : 0.2346000032275915 Test Acc : 0.2565000024624169\n",
      "\n",
      "Current : 6 Train Acc : 0.2450000032633543 Test Acc : 0.26362500195391475\n",
      "\n",
      "Current : 7 Train Acc : 0.2528000033199787 Test Acc : 0.26987500249408186\n",
      "\n",
      "Current : 8 Train Acc : 0.26040000274777414 Test Acc : 0.275500002540648\n",
      "\n",
      "Current : 9 Train Acc : 0.2636000022739172 Test Acc : 0.2796250029001385\n",
      "\n",
      "Current : 10 Train Acc : 0.2662000025063753 Test Acc : 0.2858750028256327\n",
      "\n",
      "Current : 11 Train Acc : 0.28240000233054163 Test Acc : 0.2950000028125942\n",
      "\n",
      "Current : 12 Train Acc : 0.2936000028699636 Test Acc : 0.29837500195018946\n",
      "\n",
      "Current : 13 Train Acc : 0.3002000014036894 Test Acc : 0.30550000197254124\n",
      "\n",
      "Current : 14 Train Acc : 0.3052000017762184 Test Acc : 0.30962500187568365\n",
      "\n",
      "Current : 15 Train Acc : 0.3106000017374754 Test Acc : 0.31337500164285303\n",
      "\n",
      "Current : 16 Train Acc : 0.3158000017851591 Test Acc : 0.31600000204518436\n",
      "\n",
      "Current : 17 Train Acc : 0.3198000022917986 Test Acc : 0.3226250022929162\n",
      "\n",
      "Current : 18 Train Acc : 0.3266000021994114 Test Acc : 0.3256250019930303\n",
      "\n",
      "Current : 19 Train Acc : 0.3316000012606382 Test Acc : 0.3306250016111881\n",
      "\n",
      "Current : 20 Train Acc : 0.33500000251829626 Test Acc : 0.33525000161491336\n",
      "\n",
      "Current : 21 Train Acc : 0.34020000183582305 Test Acc : 0.3403750022035092\n",
      "\n",
      "Current : 22 Train Acc : 0.34620000171661375 Test Acc : 0.3433750019222498\n",
      "\n",
      "Current : 23 Train Acc : 0.35500000120699404 Test Acc : 0.34750000180676577\n",
      "\n",
      "Current : 24 Train Acc : 0.3588000008761883 Test Acc : 0.3513750014733523\n",
      "\n",
      "Current : 25 Train Acc : 0.36000000159442425 Test Acc : 0.3566250012814999\n",
      "\n",
      "Current : 26 Train Acc : 0.3628000005632639 Test Acc : 0.3622500019147992\n",
      "\n",
      "Current : 27 Train Acc : 0.3704000002145767 Test Acc : 0.3665000016242266\n",
      "\n",
      "Current : 28 Train Acc : 0.3762000013291836 Test Acc : 0.37212500125169756\n",
      "\n",
      "Current : 29 Train Acc : 0.37600000175833703 Test Acc : 0.37700000176206233\n",
      "\n",
      "Current : 30 Train Acc : 0.378400001347065 Test Acc : 0.38062500124797227\n",
      "\n",
      "Current : 31 Train Acc : 0.3814000015258789 Test Acc : 0.38187500113621353\n",
      "\n",
      "Current : 32 Train Acc : 0.3886000013947487 Test Acc : 0.3827500007115304\n",
      "\n",
      "Current : 33 Train Acc : 0.393400001347065 Test Acc : 0.38250000078231094\n",
      "\n",
      "Current : 34 Train Acc : 0.3966000013649464 Test Acc : 0.3853750004991889\n",
      "\n",
      "Current : 35 Train Acc : 0.40100000098347666 Test Acc : 0.38425000096671286\n",
      "\n",
      "Current : 36 Train Acc : 0.405400002092123 Test Acc : 0.3880000008549541\n",
      "\n",
      "Current : 37 Train Acc : 0.40580000230669977 Test Acc : 0.38787500056438146\n",
      "\n",
      "Current : 38 Train Acc : 0.4102000017762184 Test Acc : 0.3900000011082739\n",
      "\n",
      "Current : 39 Train Acc : 0.41180000227689745 Test Acc : 0.3883750005345792\n",
      "\n",
      "Current : 40 Train Acc : 0.4132000021338463 Test Acc : 0.39100000051781536\n",
      "\n",
      "Current : 41 Train Acc : 0.41520000231266024 Test Acc : 0.39212500082328916\n",
      "\n",
      "Current : 42 Train Acc : 0.41760000163316724 Test Acc : 0.3933750003017485\n",
      "\n",
      "Current : 43 Train Acc : 0.4218000007867813 Test Acc : 0.39787500055506825\n",
      "\n",
      "Current : 44 Train Acc : 0.422200001180172 Test Acc : 0.39787500036880374\n",
      "\n",
      "Current : 45 Train Acc : 0.4238000011444092 Test Acc : 0.3992500001005828\n",
      "\n",
      "Current : 46 Train Acc : 0.42560000163316725 Test Acc : 0.4011250001564622\n",
      "\n",
      "Current : 47 Train Acc : 0.42920000189542773 Test Acc : 0.40225000040605663\n",
      "\n",
      "Current : 48 Train Acc : 0.43420000141859055 Test Acc : 0.4053750005923212\n",
      "\n",
      "Current : 49 Train Acc : 0.43600000178813936 Test Acc : 0.4048750003986061\n",
      "\n",
      "Current : 50 Train Acc : 0.43700000166893005 Test Acc : 0.40762500030919907\n",
      "\n",
      "Current : 51 Train Acc : 0.4378000015020371 Test Acc : 0.4095000009052455\n",
      "\n",
      "Current : 52 Train Acc : 0.44040000063180923 Test Acc : 0.40887500083073974\n",
      "\n",
      "Current : 53 Train Acc : 0.4412000014781952 Test Acc : 0.4088750007003546\n",
      "\n",
      "Current : 54 Train Acc : 0.4442000009417534 Test Acc : 0.4085000005178154\n",
      "\n",
      "Current : 55 Train Acc : 0.44720000052452086 Test Acc : 0.41000000063329933\n",
      "\n",
      "Current : 56 Train Acc : 0.44780000001192094 Test Acc : 0.41125000121071936\n",
      "\n",
      "Current : 57 Train Acc : 0.4488000007271767 Test Acc : 0.40987500056624415\n",
      "\n",
      "Current : 58 Train Acc : 0.45060000026226044 Test Acc : 0.41450000073760745\n",
      "\n",
      "Current : 59 Train Acc : 0.4514000006914139 Test Acc : 0.4143750000372529\n",
      "\n",
      "Current : 60 Train Acc : 0.452600000500679 Test Acc : 0.4170000000298023\n",
      "\n",
      "Current : 61 Train Acc : 0.45600000059604645 Test Acc : 0.41650000028312206\n",
      "\n",
      "Current : 62 Train Acc : 0.45940000033378603 Test Acc : 0.4180000003799796\n",
      "\n",
      "Current : 63 Train Acc : 0.46020000070333483 Test Acc : 0.41962500020861626\n",
      "\n",
      "Current : 64 Train Acc : 0.4632000006437302 Test Acc : 0.4207500005699694\n",
      "\n",
      "Current : 65 Train Acc : 0.4656000007390976 Test Acc : 0.42100000051781533\n",
      "\n",
      "Current : 66 Train Acc : 0.4668000012636185 Test Acc : 0.42262500079348686\n",
      "\n",
      "Current : 67 Train Acc : 0.46680000114440917 Test Acc : 0.42400000074878336\n",
      "\n",
      "Current : 68 Train Acc : 0.4674000015258789 Test Acc : 0.42537500100210307\n",
      "\n",
      "Current : 69 Train Acc : 0.4680000019669533 Test Acc : 0.42575000124052165\n",
      "\n",
      "Current : 70 Train Acc : 0.4718000019192696 Test Acc : 0.4261250011064112\n",
      "\n",
      "Current : 71 Train Acc : 0.47160000133514407 Test Acc : 0.42650000093504786\n",
      "\n",
      "Current : 72 Train Acc : 0.4728000011444092 Test Acc : 0.426000001039356\n",
      "\n",
      "Current : 73 Train Acc : 0.475200001001358 Test Acc : 0.42825000112876294\n",
      "\n",
      "Current : 74 Train Acc : 0.47640000069141386 Test Acc : 0.4290000006183982\n",
      "\n",
      "Current : 75 Train Acc : 0.478600000500679 Test Acc : 0.42875000059604645\n",
      "\n",
      "Current : 76 Train Acc : 0.4806000006198883 Test Acc : 0.42912500094622374\n",
      "\n",
      "Current : 77 Train Acc : 0.4804000009298325 Test Acc : 0.42887500081211327\n",
      "\n",
      "Current : 78 Train Acc : 0.4812000013589859 Test Acc : 0.43050000090152024\n",
      "\n",
      "Current : 79 Train Acc : 0.48360000109672546 Test Acc : 0.4283750005438924\n",
      "\n",
      "Current : 80 Train Acc : 0.4834000008106232 Test Acc : 0.42925000086426734\n",
      "\n",
      "Current : 81 Train Acc : 0.4860000011920929 Test Acc : 0.4306250003352761\n",
      "\n",
      "Current : 82 Train Acc : 0.48740000146627427 Test Acc : 0.43062500037252904\n",
      "\n",
      "Current : 83 Train Acc : 0.48720000129938124 Test Acc : 0.43224999997764824\n",
      "\n",
      "Current : 84 Train Acc : 0.4884000019431114 Test Acc : 0.43275000013411047\n",
      "\n",
      "Current : 85 Train Acc : 0.4906000021696091 Test Acc : 0.43325000047683715\n",
      "\n",
      "Current : 86 Train Acc : 0.49280000191926954 Test Acc : 0.4330000008642674\n",
      "\n",
      "Current : 87 Train Acc : 0.49360000222921374 Test Acc : 0.4336250001564622\n",
      "\n",
      "Current : 88 Train Acc : 0.49520000153779986 Test Acc : 0.4347500006482005\n",
      "\n",
      "Current : 89 Train Acc : 0.49640000122785566 Test Acc : 0.4371250006556511\n",
      "\n",
      "Current : 90 Train Acc : 0.4972000016570091 Test Acc : 0.43787500001490115\n",
      "\n",
      "Current : 91 Train Acc : 0.4982000017762184 Test Acc : 0.43974999990314245\n",
      "\n",
      "Current : 92 Train Acc : 0.4984000020623207 Test Acc : 0.44024999979883433\n",
      "\n",
      "Current : 93 Train Acc : 0.49960000211000444 Test Acc : 0.44075000029057265\n",
      "\n",
      "Current : 94 Train Acc : 0.5012000021338463 Test Acc : 0.4412499999627471\n",
      "\n",
      "Current : 95 Train Acc : 0.5006000016331673 Test Acc : 0.44062499951571227\n",
      "\n",
      "Current : 96 Train Acc : 0.4998000013232231 Test Acc : 0.4421249996125698\n",
      "\n",
      "Current : 97 Train Acc : 0.5052000009417534 Test Acc : 0.4421249997615814\n",
      "\n",
      "Current : 98 Train Acc : 0.5044000006318092 Test Acc : 0.44424999982118607\n",
      "\n",
      "Current : 99 Train Acc : 0.505000001847744 Test Acc : 0.44412499994039534\n",
      "\n",
      "Current : 100 Train Acc : 0.507600001513958 Test Acc : 0.4452499998360872\n",
      "\n",
      "Current : 101 Train Acc : 0.5066000013947487 Test Acc : 0.4462499998137355\n",
      "\n",
      "Current : 102 Train Acc : 0.5080000011324882 Test Acc : 0.44699999984353783\n",
      "\n",
      "Current : 103 Train Acc : 0.5080000011920929 Test Acc : 0.4480000000447035\n",
      "\n",
      "Current : 104 Train Acc : 0.5108000022172928 Test Acc : 0.44862499997019767\n",
      "\n",
      "Current : 105 Train Acc : 0.511200002014637 Test Acc : 0.4497500008717179\n",
      "\n",
      "Current : 106 Train Acc : 0.5146000021100045 Test Acc : 0.451000000461936\n",
      "\n",
      "Current : 107 Train Acc : 0.5154000023007392 Test Acc : 0.45325000055134296\n",
      "\n",
      "Current : 108 Train Acc : 0.5140000016093255 Test Acc : 0.45450000010430813\n",
      "\n",
      "Current : 109 Train Acc : 0.5190000022053719 Test Acc : 0.45487500008195636\n",
      "\n",
      "Current : 110 Train Acc : 0.5184000023007392 Test Acc : 0.4561249999701977\n",
      "\n",
      "Current : 111 Train Acc : 0.5198000015020371 Test Acc : 0.45662500005215406\n",
      "\n",
      "Current : 112 Train Acc : 0.5218000020980835 Test Acc : 0.4578749997541308\n",
      "\n",
      "Current : 113 Train Acc : 0.5240000014305115 Test Acc : 0.45950000070035457\n",
      "\n",
      "Current : 114 Train Acc : 0.5242000017166137 Test Acc : 0.45950000047683714\n",
      "\n",
      "Current : 115 Train Acc : 0.5270000013113022 Test Acc : 0.4600000003352761\n",
      "\n",
      "Current : 116 Train Acc : 0.5278000015020371 Test Acc : 0.46049999993294477\n",
      "\n",
      "Current : 117 Train Acc : 0.5278000016212463 Test Acc : 0.46062499966472387\n",
      "\n",
      "Current : 118 Train Acc : 0.53000000166893 Test Acc : 0.46200000021606685\n",
      "\n",
      "Current : 119 Train Acc : 0.5320000013113022 Test Acc : 0.4626250000670552\n",
      "\n",
      "Current : 120 Train Acc : 0.5334000018239021 Test Acc : 0.4632500005885959\n",
      "\n",
      "Current : 121 Train Acc : 0.5342000011205673 Test Acc : 0.4637500006705523\n",
      "\n",
      "Current : 122 Train Acc : 0.5344000015258789 Test Acc : 0.4655000006407499\n",
      "\n",
      "Current : 123 Train Acc : 0.5358000018000603 Test Acc : 0.4666250002756715\n",
      "\n",
      "Current : 124 Train Acc : 0.5378000010251999 Test Acc : 0.4672500004991889\n",
      "\n",
      "Current : 125 Train Acc : 0.5382000017762184 Test Acc : 0.46737500082701444\n",
      "\n",
      "Current : 126 Train Acc : 0.5392000010609627 Test Acc : 0.4672500007599592\n",
      "\n",
      "Current : 127 Train Acc : 0.5414000015854835 Test Acc : 0.46862500082701447\n",
      "\n",
      "Current : 128 Train Acc : 0.5432000016570091 Test Acc : 0.46862500067800283\n",
      "\n",
      "Current : 129 Train Acc : 0.5456000004410744 Test Acc : 0.4707500006631017\n",
      "\n",
      "Current : 130 Train Acc : 0.546600000679493 Test Acc : 0.46900000143796206\n",
      "\n",
      "Current : 131 Train Acc : 0.548200000822544 Test Acc : 0.4696250011399388\n",
      "\n",
      "Current : 132 Train Acc : 0.5482000004649162 Test Acc : 0.469125000834465\n",
      "\n",
      "Current : 133 Train Acc : 0.5486000009179115 Test Acc : 0.47137500129640103\n",
      "\n",
      "Current : 134 Train Acc : 0.5510000013709069 Test Acc : 0.4715000006183982\n",
      "\n",
      "Current : 135 Train Acc : 0.5510000001788139 Test Acc : 0.4713750011101365\n",
      "\n",
      "Current : 136 Train Acc : 0.5517999992966652 Test Acc : 0.47237500112503766\n",
      "\n",
      "Current : 137 Train Acc : 0.5504000000357628 Test Acc : 0.47212500002235175\n",
      "\n",
      "Current : 138 Train Acc : 0.5528000001311302 Test Acc : 0.47175000082701446\n",
      "\n",
      "Current : 139 Train Acc : 0.5531999997496605 Test Acc : 0.47312500067055224\n",
      "\n",
      "Current : 140 Train Acc : 0.5565999993681907 Test Acc : 0.47300000075250864\n",
      "\n",
      "Current : 141 Train Acc : 0.5559999991059303 Test Acc : 0.4733750008791685\n",
      "\n",
      "Current : 142 Train Acc : 0.5573999989628792 Test Acc : 0.47400000065565107\n",
      "\n",
      "Current : 143 Train Acc : 0.5569999992251397 Test Acc : 0.47512500047683714\n",
      "\n",
      "Current : 144 Train Acc : 0.5571999989151954 Test Acc : 0.4745000005140901\n",
      "\n",
      "Current : 145 Train Acc : 0.557599999845028 Test Acc : 0.47400000046938656\n",
      "\n",
      "Current : 146 Train Acc : 0.558599999845028 Test Acc : 0.4748750001937151\n",
      "\n",
      "Current : 147 Train Acc : 0.5593999997973442 Test Acc : 0.47474999975413085\n",
      "\n",
      "Current : 148 Train Acc : 0.5603999989628792 Test Acc : 0.4752500005438924\n",
      "\n",
      "Current : 149 Train Acc : 0.5599999997019768 Test Acc : 0.4767500003799796\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# D\n",
    "current_exp_name = 'D';\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "# create layers\n",
    "l1 = CNN(3,3, 16,which_reg=current_exp_name); \n",
    "l2 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l3 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "\n",
    "l4 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l5 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l6 = CNN(3,16,10,which_reg=current_exp_name); \n",
    "\n",
    "# 2. graph \n",
    "x = tf.placeholder(tf.float32,(batch_size,96,96,3))\n",
    "y = tf.placeholder(tf.float32,(batch_size,10))\n",
    "\n",
    "layer1, layer1a = l1. feedforward(x,stride=2)\n",
    "layer2, layer2a = l2. feedforward(layer1a,stride=2)\n",
    "layer3, layer3a = l3. feedforward(layer2a,stride=2)\n",
    "layer4, layer4a = l4. feedforward(layer3a,stride=2)\n",
    "layer5, layer5a = l5. feedforward(layer4a)\n",
    "layer6, layer6a = l6. feedforward(layer5a)\n",
    "\n",
    "final_layer   = tf.reduce_mean(layer6a,(1,2))\n",
    "final_softmax = tf_softmax(final_layer)\n",
    "cost          = -tf.reduce_mean(y * tf.log(final_softmax + 1e-8))\n",
    "correct_prediction = tf.equal(tf.argmax(final_softmax, 1), tf.argmax(y, 1))\n",
    "accuracy           = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "gradient = tf.tile((final_softmax-y)[:,None,None,:],[1,6,6,1])/batch_size\n",
    "grad6p,grad6w,grad6_up = l6.backprop(gradient)\n",
    "grad5p,grad5w,grad5_up = l5.backprop(grad6p)\n",
    "grad4p,grad4w,grad4_up = l4.backprop(grad5p,stride=2)\n",
    "grad3p,grad3w,grad3_up = l3.backprop(grad4p,stride=2)\n",
    "grad2p,grad2w,grad2_up = l2.backprop(grad3p,stride=2)\n",
    "grad1p,grad1w,grad1_up = l1.backprop(grad2p,stride=2)\n",
    "\n",
    "gradient_update = grad6_up + grad5_up + grad4_up + grad3_up + grad2_up + grad1_up \n",
    "\n",
    "# train\n",
    "sess.run(tf.global_variables_initializer())\n",
    "avg_acc_train = 0; avg_acc_test  = 0; train_acc = [];test_acc = []\n",
    "\n",
    "# mean std skew kurt non-zero\n",
    "llayer1 = [[],[],[],[],[]]; llayer2 = [[],[],[],[],[]]; llayer3 = [[],[],[],[],[]]\n",
    "llayer4 = [[],[],[],[],[]]; llayer5 = [[],[],[],[],[]]; llayer6 = [[],[],[],[],[]]\n",
    "\n",
    "llayer1a = [[],[],[],[],[]]; llayer2a = [[],[],[],[],[]]; llayer3a = [[],[],[],[],[]]\n",
    "llayer4a = [[],[],[],[],[]]; llayer5a = [[],[],[],[],[]]; llayer6a = [[],[],[],[],[]]\n",
    "\n",
    "weight1 = [[],[],[],[],[]]; weight2 = [[],[],[],[],[]]; weight3 = [[],[],[],[],[]];\n",
    "weight4 = [[],[],[],[],[]]; weight5 = [[],[],[],[],[]]; weight6 = [[],[],[],[],[]];\n",
    "\n",
    "gradw1  = [[],[],[],[],[]]; gradw2  = [[],[],[],[],[]]; gradw3  = [[],[],[],[],[]];\n",
    "gradw4  = [[],[],[],[],[]]; gradw5  = [[],[],[],[],[]]; gradw6  = [[],[],[],[],[]];\n",
    "\n",
    "gradp1  = [[],[],[],[],[]]; gradp2  = [[],[],[],[],[]]; gradp3  = [[],[],[],[],[]];\n",
    "gradp4  = [[],[],[],[],[]]; gradp5  = [[],[],[],[],[]]; gradp6  = [[],[],[],[],[]];\n",
    "\n",
    "gradup1  = [[],[],[],[],[]]; gradup2  = [[],[],[],[],[]]; gradup3  = [[],[],[],[],[]];\n",
    "gradup4  = [[],[],[],[],[]]; gradup5  = [[],[],[],[],[]]; gradup6  = [[],[],[],[],[]];\n",
    "\n",
    "list_of_outputs = [\n",
    "    layer1,layer2,layer3,layer4,layer5,layer6,\n",
    "    layer1a,layer2a,layer3a,layer4a,layer5a,layer6a,\n",
    "    l1.getw(),l2.getw(),l3.getw(),l4.getw(),l5.getw(),l6.getw(),\n",
    "    grad1w,grad2w,grad3w,grad4w,grad5w,grad6w,\n",
    "    grad1p,grad2p,grad3p,grad4p,grad5p,grad6p,\n",
    "    grad1_up[0],grad2_up[0],grad3_up[0],grad4_up[0],grad5_up[0],grad6_up[0]\n",
    "]\n",
    "\n",
    "for iter in range(num_epoch):\n",
    "\n",
    "    # Training Accuracy    \n",
    "    for current_batch_index in range(0,len(train_images),batch_size):\n",
    "        current_data  = train_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = train_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy,gradient_update],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(train_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_train = avg_acc_train + sess_results[0]\n",
    "        \n",
    "    # get the results\n",
    "    mid_stat = sess.run(list_of_outputs,feed_dict={x:current_data,y:current_label})\n",
    "    \n",
    "    # Test Accuracy    \n",
    "    for current_batch_index in range(0,len(test_images), batch_size):\n",
    "        current_data  = test_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = test_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(test_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_test = avg_acc_test + sess_results[0]   \n",
    "        \n",
    "    # ======================== extract stats ========================\n",
    "    llayer1 = append_stat(llayer1,mid_stat,0);  llayer2 = append_stat(llayer2,mid_stat,1);  llayer3 = append_stat(llayer3,mid_stat,2);\n",
    "    llayer4 = append_stat(llayer4,mid_stat,3);  llayer5 = append_stat(llayer5,mid_stat,4);  llayer6 = append_stat(llayer6,mid_stat,5);\n",
    "\n",
    "    llayer1a = append_stat(llayer1a,mid_stat,6);  llayer2a = append_stat(llayer2a,mid_stat,7);  llayer3a = append_stat(llayer3a,mid_stat,8);\n",
    "    llayer4a = append_stat(llayer4a,mid_stat,9);  llayer5a = append_stat(llayer5a,mid_stat,10); llayer6a = append_stat(llayer6a,mid_stat,11);\n",
    "    \n",
    "    weight1 = append_stat(weight1,mid_stat,12);  weight2 = append_stat(weight2,mid_stat,13);  weight3 = append_stat(weight3,mid_stat,14);\n",
    "    weight4 = append_stat(weight4,mid_stat,15);  weight5 = append_stat(weight5,mid_stat,16);  weight6 = append_stat(weight6,mid_stat,17);\n",
    "    \n",
    "    gradw1 = append_stat(gradw1,mid_stat,18); gradw2 = append_stat(gradw2,mid_stat,19); gradw3 = append_stat(gradw3,mid_stat,20);\n",
    "    gradw4 = append_stat(gradw4,mid_stat,21); gradw5 = append_stat(gradw5,mid_stat,22); gradw6 = append_stat(gradw6,mid_stat,23);\n",
    "    \n",
    "    gradp1 = append_stat(gradp1,mid_stat,24); gradp2 = append_stat(gradp2,mid_stat,25); gradp3 = append_stat(gradp3,mid_stat,26);\n",
    "    gradp4 = append_stat(gradp4,mid_stat,27); gradp5 = append_stat(gradp5,mid_stat,28); gradp6 = append_stat(gradp6,mid_stat,29);\n",
    "\n",
    "    gradup1 = append_stat(gradup1,mid_stat,30); gradup2 = append_stat(gradup2,mid_stat,31); gradup3 = append_stat(gradup3,mid_stat,32);\n",
    "    gradup4 = append_stat(gradup4,mid_stat,33); gradup5 = append_stat(gradup5,mid_stat,34); gradup6 = append_stat(gradup6,mid_stat,35);\n",
    "\n",
    "    train_acc.append(avg_acc_train/(len(train_images)/batch_size))\n",
    "    test_acc .append(avg_acc_test / (len(test_images)/batch_size))\n",
    "    # ======================== extract stats ========================\n",
    "    \n",
    "    # ======================== save to image ========================\n",
    "    save_to_image(mid_stat[0:6]   ,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,\"layer\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[6:12]  ,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a,\"layera\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[12:18] ,weight1,weight2,weight3,weight4,weight5,weight6,\"weights\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[18:24] ,gradw1,gradw2,gradw3,gradw4,gradw5,gradw6,\"gradientw\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[24:30] ,gradp1,gradp2,gradp3,gradp4,gradp5,gradp6,\"gradientp\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[30:36] ,gradup1,gradup2,gradup3,gradup4,gradup5,gradup6,\"moment\",train_acc,test_acc,current_exp_name,iter)\n",
    "    # ======================== save to image ========================\n",
    "        \n",
    "    # ======================== print reset ========================\n",
    "    print(\"Current : \"+ str(iter) + \" Train Acc : \" + str(avg_acc_train/(len(train_images)/batch_size)) + \" Test Acc : \" + str(avg_acc_test/(len(test_images)/batch_size)) + '\\n')\n",
    "    avg_acc_train = 0 ; avg_acc_test  = 0\n",
    "    # ======================== print reset ========================\n",
    "\n",
    "np.save(current_exp_name+'/train_acc.npy',train_acc); np.save(current_exp_name+'/test_acc.npy', test_acc)    \n",
    "np.save(current_exp_name+'/llayer1.npy', llayer1);  np.save(current_exp_name+'/llayer2.npy', llayer2);  np.save(current_exp_name+'/llayer3.npy', llayer3); \n",
    "np.save(current_exp_name+'/llayer4.npy', llayer4);  np.save(current_exp_name+'/llayer5.npy', llayer5);  np.save(current_exp_name+'/llayer6.npy', llayer6); \n",
    "\n",
    "np.save(current_exp_name+'/llayer1a.npy', llayer1a);  np.save(current_exp_name+'/llayer2a.npy', llayer2a);  np.save(current_exp_name+'/llayer3a.npy', llayer3a); \n",
    "np.save(current_exp_name+'/llayer4a.npy', llayer4a);  np.save(current_exp_name+'/llayer5a.npy', llayer5a);  np.save(current_exp_name+'/llayer6a.npy', llayer6a); \n",
    "\n",
    "np.save(current_exp_name+'/weight1.npy', weight1);  np.save(current_exp_name+'/weight2.npy', weight2);  np.save(current_exp_name+'/weight3.npy', weight3);  \n",
    "np.save(current_exp_name+'/weight4.npy', weight4);  np.save(current_exp_name+'/weight5.npy', weight5);  np.save(current_exp_name+'/weight6.npy', weight6);  \n",
    "\n",
    "np.save(current_exp_name+'/gradw1.npy', gradw1); np.save(current_exp_name+'/gradw2.npy', gradw2); np.save(current_exp_name+'/gradw3.npy', gradw3);\n",
    "np.save(current_exp_name+'/gradw4.npy', gradw4); np.save(current_exp_name+'/gradw5.npy', gradw5); np.save(current_exp_name+'/gradw6.npy', gradw6);\n",
    "\n",
    "np.save(current_exp_name+'/gradp1.npy', gradp1); np.save(current_exp_name+'/gradp2.npy', gradp2); np.save(current_exp_name+'/gradp3.npy', gradp3);\n",
    "np.save(current_exp_name+'/gradp4.npy', gradp4); np.save(current_exp_name+'/gradp5.npy', gradp5); np.save(current_exp_name+'/gradp6.npy', gradp6);\n",
    "\n",
    "np.save(current_exp_name+'/gradup1.npy', gradup1); np.save(current_exp_name+'/gradup2.npy', gradup2); np.save(current_exp_name+'/gradup3.npy', gradup3);\n",
    "np.save(current_exp_name+'/gradup4.npy', gradup4); np.save(current_exp_name+'/gradup5.npy', gradup5); np.save(current_exp_name+'/gradup6.npy', gradup6);\n",
    "\n",
    "sess.close(); tf.reset_default_graph();\n",
    "\n",
    "%reset_selective -f l1,l2,l3,l4,l5,l6\n",
    "%reset_selective -f layer1,layer2,layer3,layer4,layer5,layer6\n",
    "%reset_selective -f layer1a,layer2a,layer3a,layer4a,layer5a,layer6a\n",
    "%reset_selective -f train_acc,test_acc,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a\n",
    "%reset_selective -f weight1,weight2,weight3,weight4,weight5,weight6\n",
    "%reset_selective -f gradw1,gradw2,gradw3,gradw4,gradw5,gradw6\n",
    "%reset_selective -f gradp1,gradp2,gradp3,gradp4,gradp5,gradp6\n",
    "%reset_selective -f gradup1,gradup2,gradup3,gradup4,gradup5,gradup6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-05T06:24:55.247444Z",
     "start_time": "2019-01-05T05:35:39.490738Z"
    },
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current : 0 Train Acc : 0.13200000239908696 Test Acc : 0.15575000283308327\n",
      "\n",
      "Current : 1 Train Acc : 0.1644000030308962 Test Acc : 0.1888750031683594\n",
      "\n",
      "Current : 2 Train Acc : 0.18620000295341016 Test Acc : 0.2131250027101487\n",
      "\n",
      "Current : 3 Train Acc : 0.20900000317394735 Test Acc : 0.2491250034701079\n",
      "\n",
      "Current : 4 Train Acc : 0.23600000295042992 Test Acc : 0.2661250026524067\n",
      "\n",
      "Current : 5 Train Acc : 0.25700000290572644 Test Acc : 0.284000002052635\n",
      "\n",
      "Current : 6 Train Acc : 0.2758000028282404 Test Acc : 0.29875000251457096\n",
      "\n",
      "Current : 7 Train Acc : 0.2930000028908253 Test Acc : 0.3052500028163195\n",
      "\n",
      "Current : 8 Train Acc : 0.3082000017464161 Test Acc : 0.31075000249780715\n",
      "\n",
      "Current : 9 Train Acc : 0.31800000178813936 Test Acc : 0.31600000252947213\n",
      "\n",
      "Current : 10 Train Acc : 0.322600001975894 Test Acc : 0.3217500022891909\n",
      "\n",
      "Current : 11 Train Acc : 0.3292000019699335 Test Acc : 0.32487500216811893\n",
      "\n",
      "Current : 12 Train Acc : 0.3326000013798475 Test Acc : 0.3263750022277236\n",
      "\n",
      "Current : 13 Train Acc : 0.3368000008761883 Test Acc : 0.33075000228360296\n",
      "\n",
      "Current : 14 Train Acc : 0.34120000106096265 Test Acc : 0.3307500024046749\n",
      "\n",
      "Current : 15 Train Acc : 0.3436000012457371 Test Acc : 0.33225000186823306\n",
      "\n",
      "Current : 16 Train Acc : 0.34420000195503236 Test Acc : 0.33462500184774396\n",
      "\n",
      "Current : 17 Train Acc : 0.34200000232458116 Test Acc : 0.3338750022277236\n",
      "\n",
      "Current : 18 Train Acc : 0.3448000018596649 Test Acc : 0.3375000024214387\n",
      "\n",
      "Current : 19 Train Acc : 0.34720000141859053 Test Acc : 0.3380000026524067\n",
      "\n",
      "Current : 20 Train Acc : 0.3518000014424324 Test Acc : 0.34112500252202155\n",
      "\n",
      "Current : 21 Train Acc : 0.3522000012993813 Test Acc : 0.3471250025555491\n",
      "\n",
      "Current : 22 Train Acc : 0.3574000008106232 Test Acc : 0.3507500019669533\n",
      "\n",
      "Current : 23 Train Acc : 0.36140000051259996 Test Acc : 0.3547500020638108\n",
      "\n",
      "Current : 24 Train Acc : 0.3612000008970499 Test Acc : 0.35675000147894026\n",
      "\n",
      "Current : 25 Train Acc : 0.36480000060796736 Test Acc : 0.3591250016540289\n",
      "\n",
      "Current : 26 Train Acc : 0.36740000122785565 Test Acc : 0.3612500020675361\n",
      "\n",
      "Current : 27 Train Acc : 0.37040000087022784 Test Acc : 0.36650000190362336\n",
      "\n",
      "Current : 28 Train Acc : 0.3728000023663044 Test Acc : 0.36687500163912773\n",
      "\n",
      "Current : 29 Train Acc : 0.3784000023305416 Test Acc : 0.37050000093877317\n",
      "\n",
      "Current : 30 Train Acc : 0.3826000015735626 Test Acc : 0.3767500013485551\n",
      "\n",
      "Current : 31 Train Acc : 0.3866000018119812 Test Acc : 0.37937500078231096\n",
      "\n",
      "Current : 32 Train Acc : 0.3904000023007393 Test Acc : 0.38362500062212346\n",
      "\n",
      "Current : 33 Train Acc : 0.3918000019788742 Test Acc : 0.38662500156089663\n",
      "\n",
      "Current : 34 Train Acc : 0.3916000017523766 Test Acc : 0.3885000006482005\n",
      "\n",
      "Current : 35 Train Acc : 0.3988000012040138 Test Acc : 0.38850000079721214\n",
      "\n",
      "Current : 36 Train Acc : 0.4034000007510185 Test Acc : 0.3906250007264316\n",
      "\n",
      "Current : 37 Train Acc : 0.4052000006437302 Test Acc : 0.39250000050291417\n",
      "\n",
      "Current : 38 Train Acc : 0.40400000029802324 Test Acc : 0.39400000078603625\n",
      "\n",
      "Current : 39 Train Acc : 0.40720000040531157 Test Acc : 0.39462500059977174\n",
      "\n",
      "Current : 40 Train Acc : 0.4096000006198883 Test Acc : 0.3960000009275973\n",
      "\n",
      "Current : 41 Train Acc : 0.4108000003993511 Test Acc : 0.3988750007934868\n",
      "\n",
      "Current : 42 Train Acc : 0.4134000011086464 Test Acc : 0.40137500075623395\n",
      "\n",
      "Current : 43 Train Acc : 0.4152000015079975 Test Acc : 0.4021250004507601\n",
      "\n",
      "Current : 44 Train Acc : 0.42000000083446504 Test Acc : 0.40225000051781534\n",
      "\n",
      "Current : 45 Train Acc : 0.42060000175237655 Test Acc : 0.40350000038743017\n",
      "\n",
      "Current : 46 Train Acc : 0.42200000190734865 Test Acc : 0.4031250009313226\n",
      "\n",
      "Current : 47 Train Acc : 0.4258000019192696 Test Acc : 0.4050000007078052\n",
      "\n",
      "Current : 48 Train Acc : 0.4272000022530556 Test Acc : 0.4055000005103648\n",
      "\n",
      "Current : 49 Train Acc : 0.4302000024318695 Test Acc : 0.40625000031664965\n",
      "\n",
      "Current : 50 Train Acc : 0.43040000200271605 Test Acc : 0.4078750002942979\n",
      "\n",
      "Current : 51 Train Acc : 0.43400000196695326 Test Acc : 0.40912500070407987\n",
      "\n",
      "Current : 52 Train Acc : 0.43480000191926954 Test Acc : 0.4096250003390014\n",
      "\n",
      "Current : 53 Train Acc : 0.43940000182390215 Test Acc : 0.40925000021234154\n",
      "\n",
      "Current : 54 Train Acc : 0.4424000014662743 Test Acc : 0.4105000003054738\n",
      "\n",
      "Current : 55 Train Acc : 0.4452000014781952 Test Acc : 0.41175000041723253\n",
      "\n",
      "Current : 56 Train Acc : 0.4482000015974045 Test Acc : 0.41187500026077034\n",
      "\n",
      "Current : 57 Train Acc : 0.44760000145435336 Test Acc : 0.41275000024586916\n",
      "\n",
      "Current : 58 Train Acc : 0.45020000064373017 Test Acc : 0.41450000036507845\n",
      "\n",
      "Current : 59 Train Acc : 0.4504000011086464 Test Acc : 0.4148750005662441\n",
      "\n",
      "Current : 60 Train Acc : 0.45040000185370443 Test Acc : 0.4156250002607703\n",
      "\n",
      "Current : 61 Train Acc : 0.45280000111460683 Test Acc : 0.41650000065565107\n",
      "\n",
      "Current : 62 Train Acc : 0.4538000011742115 Test Acc : 0.4157500009983778\n",
      "\n",
      "Current : 63 Train Acc : 0.4560000013411045 Test Acc : 0.4177500008791685\n",
      "\n",
      "Current : 64 Train Acc : 0.4576000012457371 Test Acc : 0.41762500096112487\n",
      "\n",
      "Current : 65 Train Acc : 0.4568000005185604 Test Acc : 0.4182500006258488\n",
      "\n",
      "Current : 66 Train Acc : 0.4574000021368265 Test Acc : 0.41962500046938656\n",
      "\n",
      "Current : 67 Train Acc : 0.4604000010639429 Test Acc : 0.4208750008046627\n",
      "\n",
      "Current : 68 Train Acc : 0.46180000199377536 Test Acc : 0.42125000096857546\n",
      "\n",
      "Current : 69 Train Acc : 0.4644000010639429 Test Acc : 0.42175000071525576\n",
      "\n",
      "Current : 70 Train Acc : 0.4664000006467104 Test Acc : 0.42112500086426735\n",
      "\n",
      "Current : 71 Train Acc : 0.46600000132620334 Test Acc : 0.4225000004097819\n",
      "\n",
      "Current : 72 Train Acc : 0.4668000012785196 Test Acc : 0.4241250004991889\n",
      "\n",
      "Current : 73 Train Acc : 0.4704000014215708 Test Acc : 0.424625000320375\n",
      "\n",
      "Current : 74 Train Acc : 0.47180000199377536 Test Acc : 0.4267500006593764\n",
      "\n",
      "Current : 75 Train Acc : 0.47080000199377536 Test Acc : 0.427125000897795\n",
      "\n",
      "Current : 76 Train Acc : 0.4750000016838312 Test Acc : 0.4270000009797513\n",
      "\n",
      "Current : 77 Train Acc : 0.4740000019222498 Test Acc : 0.4298750009573996\n",
      "\n",
      "Current : 78 Train Acc : 0.4758000014573336 Test Acc : 0.42975000128149987\n",
      "\n",
      "Current : 79 Train Acc : 0.47440000173449515 Test Acc : 0.4331250013038516\n",
      "\n",
      "Current : 80 Train Acc : 0.47640000107884406 Test Acc : 0.43475000087171795\n",
      "\n",
      "Current : 81 Train Acc : 0.47740000090003015 Test Acc : 0.4353750007227063\n",
      "\n",
      "Current : 82 Train Acc : 0.47940000000596045 Test Acc : 0.43637500040233135\n",
      "\n",
      "Current : 83 Train Acc : 0.48040000012516976 Test Acc : 0.4350000006519258\n",
      "\n",
      "Current : 84 Train Acc : 0.4823999999761581 Test Acc : 0.4365000007301569\n",
      "\n",
      "Current : 85 Train Acc : 0.48200000017881395 Test Acc : 0.43687500052154066\n",
      "\n",
      "Current : 86 Train Acc : 0.48460000050067903 Test Acc : 0.4398750011995435\n",
      "\n",
      "Current : 87 Train Acc : 0.4850000005960464 Test Acc : 0.44075000084936616\n",
      "\n",
      "Current : 88 Train Acc : 0.48480000084638597 Test Acc : 0.4397500005178154\n",
      "\n",
      "Current : 89 Train Acc : 0.486200001001358 Test Acc : 0.4412500008940697\n",
      "\n",
      "Current : 90 Train Acc : 0.48780000054836276 Test Acc : 0.4415000002831221\n",
      "\n",
      "Current : 91 Train Acc : 0.4880000005960464 Test Acc : 0.442750000692904\n",
      "\n",
      "Current : 92 Train Acc : 0.48940000087022784 Test Acc : 0.4448750003799796\n",
      "\n",
      "Current : 93 Train Acc : 0.4896000006198883 Test Acc : 0.4455000011250377\n",
      "\n",
      "Current : 94 Train Acc : 0.4914000014066696 Test Acc : 0.44525000095367434\n",
      "\n",
      "Current : 95 Train Acc : 0.48920000094175337 Test Acc : 0.44650000020861624\n",
      "\n",
      "Current : 96 Train Acc : 0.49180000060796736 Test Acc : 0.447375000230968\n",
      "\n",
      "Current : 97 Train Acc : 0.49320000112056733 Test Acc : 0.44650000009685753\n",
      "\n",
      "Current : 98 Train Acc : 0.49220000094175337 Test Acc : 0.4477500005811453\n",
      "\n",
      "Current : 99 Train Acc : 0.49400000083446505 Test Acc : 0.44925000112503766\n",
      "\n",
      "Current : 100 Train Acc : 0.4914000006914139 Test Acc : 0.45000000115484\n",
      "\n",
      "Current : 101 Train Acc : 0.4936000015735626 Test Acc : 0.4510000009462237\n",
      "\n",
      "Current : 102 Train Acc : 0.4948000011444092 Test Acc : 0.4522500015422702\n",
      "\n",
      "Current : 103 Train Acc : 0.4962000012397766 Test Acc : 0.45262500137090683\n",
      "\n",
      "Current : 104 Train Acc : 0.4964000011086464 Test Acc : 0.4536250010505319\n",
      "\n",
      "Current : 105 Train Acc : 0.4976000015735626 Test Acc : 0.45537500105798245\n",
      "\n",
      "Current : 106 Train Acc : 0.5026000013947487 Test Acc : 0.4567500006780028\n",
      "\n",
      "Current : 107 Train Acc : 0.5004000017046928 Test Acc : 0.45737500093877315\n",
      "\n",
      "Current : 108 Train Acc : 0.5040000012516975 Test Acc : 0.4581250006519258\n",
      "\n",
      "Current : 109 Train Acc : 0.5032000015377999 Test Acc : 0.4580000011250377\n",
      "\n",
      "Current : 110 Train Acc : 0.5034000014662743 Test Acc : 0.45925000045448544\n",
      "\n",
      "Current : 111 Train Acc : 0.5032000010609626 Test Acc : 0.4593750011920929\n",
      "\n",
      "Current : 112 Train Acc : 0.502600000679493 Test Acc : 0.45837500074878335\n",
      "\n",
      "Current : 113 Train Acc : 0.5044000011086464 Test Acc : 0.46012500032782555\n",
      "\n",
      "Current : 114 Train Acc : 0.5066000011563301 Test Acc : 0.46037500085309147\n",
      "\n",
      "Current : 115 Train Acc : 0.5064000003933906 Test Acc : 0.46100000077858566\n",
      "\n",
      "Current : 116 Train Acc : 0.506400001168251 Test Acc : 0.46250000052154067\n",
      "\n",
      "Current : 117 Train Acc : 0.507600000500679 Test Acc : 0.4618750007078052\n",
      "\n",
      "Current : 118 Train Acc : 0.5070000007748604 Test Acc : 0.4625000006332993\n",
      "\n",
      "Current : 119 Train Acc : 0.5082000010609626 Test Acc : 0.463625000230968\n",
      "\n",
      "Current : 120 Train Acc : 0.5090000004470349 Test Acc : 0.4620000004582107\n",
      "\n",
      "Current : 121 Train Acc : 0.5122000007033348 Test Acc : 0.4623750003241003\n",
      "\n",
      "Current : 122 Train Acc : 0.5132000000476837 Test Acc : 0.46225000023841856\n",
      "\n",
      "Current : 123 Train Acc : 0.5116000000834465 Test Acc : 0.4627500004135072\n",
      "\n",
      "Current : 124 Train Acc : 0.5144000005424023 Test Acc : 0.462250000461936\n",
      "\n",
      "Current : 125 Train Acc : 0.517200000166893 Test Acc : 0.4617500004731119\n",
      "\n",
      "Current : 126 Train Acc : 0.5142000004649162 Test Acc : 0.4640000009909272\n",
      "\n",
      "Current : 127 Train Acc : 0.5152000002264977 Test Acc : 0.4616250005736947\n",
      "\n",
      "Current : 128 Train Acc : 0.5146000009179116 Test Acc : 0.46125000074505806\n",
      "\n",
      "Current : 129 Train Acc : 0.5155999999344348 Test Acc : 0.4626250007376075\n",
      "\n",
      "Current : 130 Train Acc : 0.5182000011503697 Test Acc : 0.46225000053644183\n",
      "\n",
      "Current : 131 Train Acc : 0.5198000010251999 Test Acc : 0.4626250008121133\n",
      "\n",
      "Current : 132 Train Acc : 0.5198000009655952 Test Acc : 0.4636250006407499\n",
      "\n",
      "Current : 133 Train Acc : 0.5204000009894371 Test Acc : 0.4631250010430813\n",
      "\n",
      "Current : 134 Train Acc : 0.5202000006437302 Test Acc : 0.4643750013038516\n",
      "\n",
      "Current : 135 Train Acc : 0.5208000015020371 Test Acc : 0.4667500010505319\n",
      "\n",
      "Current : 136 Train Acc : 0.5212000013589859 Test Acc : 0.4662500011920929\n",
      "\n",
      "Current : 137 Train Acc : 0.5232000014781952 Test Acc : 0.4666250013932586\n",
      "\n",
      "Current : 138 Train Acc : 0.5252000008821487 Test Acc : 0.4676250013336539\n",
      "\n",
      "Current : 139 Train Acc : 0.5254000012874603 Test Acc : 0.4655000012740493\n",
      "\n",
      "Current : 140 Train Acc : 0.5266000012159348 Test Acc : 0.4666250013932586\n",
      "\n",
      "Current : 141 Train Acc : 0.5264000016450882 Test Acc : 0.46812500167638066\n",
      "\n",
      "Current : 142 Train Acc : 0.5278000012636185 Test Acc : 0.4667500014230609\n",
      "\n",
      "Current : 143 Train Acc : 0.5286000008583069 Test Acc : 0.4673750014603138\n",
      "\n",
      "Current : 144 Train Acc : 0.5306000009179115 Test Acc : 0.468000001013279\n",
      "\n",
      "Current : 145 Train Acc : 0.530800000846386 Test Acc : 0.46900000140070913\n",
      "\n",
      "Current : 146 Train Acc : 0.5310000010728836 Test Acc : 0.4693750015646219\n",
      "\n",
      "Current : 147 Train Acc : 0.531200001835823 Test Acc : 0.4701250015944243\n",
      "\n",
      "Current : 148 Train Acc : 0.5344000019431114 Test Acc : 0.4686250014230609\n",
      "\n",
      "Current : 149 Train Acc : 0.5338000017404556 Test Acc : 0.469750001244247\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# E\n",
    "current_exp_name = 'E';\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "# create layers\n",
    "l1 = CNN(3,3, 16,which_reg=current_exp_name); \n",
    "l2 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l3 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "\n",
    "l4 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l5 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l6 = CNN(3,16,10,which_reg=current_exp_name); \n",
    "\n",
    "# 2. graph \n",
    "x = tf.placeholder(tf.float32,(batch_size,96,96,3))\n",
    "y = tf.placeholder(tf.float32,(batch_size,10))\n",
    "\n",
    "layer1, layer1a = l1. feedforward(x,stride=2)\n",
    "layer2, layer2a = l2. feedforward(layer1a,stride=2)\n",
    "layer3, layer3a = l3. feedforward(layer2a,stride=2)\n",
    "layer4, layer4a = l4. feedforward(layer3a,stride=2)\n",
    "layer5, layer5a = l5. feedforward(layer4a)\n",
    "layer6, layer6a = l6. feedforward(layer5a)\n",
    "\n",
    "final_layer   = tf.reduce_mean(layer6a,(1,2))\n",
    "final_softmax = tf_softmax(final_layer)\n",
    "cost          = -tf.reduce_mean(y * tf.log(final_softmax + 1e-8))\n",
    "correct_prediction = tf.equal(tf.argmax(final_softmax, 1), tf.argmax(y, 1))\n",
    "accuracy           = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "gradient = tf.tile((final_softmax-y)[:,None,None,:],[1,6,6,1])/batch_size\n",
    "grad6p,grad6w,grad6_up = l6.backprop(gradient)\n",
    "grad5p,grad5w,grad5_up = l5.backprop(grad6p)\n",
    "grad4p,grad4w,grad4_up = l4.backprop(grad5p,stride=2)\n",
    "grad3p,grad3w,grad3_up = l3.backprop(grad4p,stride=2)\n",
    "grad2p,grad2w,grad2_up = l2.backprop(grad3p,stride=2)\n",
    "grad1p,grad1w,grad1_up = l1.backprop(grad2p,stride=2)\n",
    "\n",
    "gradient_update = grad6_up + grad5_up + grad4_up + grad3_up + grad2_up + grad1_up \n",
    "\n",
    "# train\n",
    "sess.run(tf.global_variables_initializer())\n",
    "avg_acc_train = 0; avg_acc_test  = 0; train_acc = [];test_acc = []\n",
    "\n",
    "# mean std skew kurt non-zero\n",
    "llayer1 = [[],[],[],[],[]]; llayer2 = [[],[],[],[],[]]; llayer3 = [[],[],[],[],[]]\n",
    "llayer4 = [[],[],[],[],[]]; llayer5 = [[],[],[],[],[]]; llayer6 = [[],[],[],[],[]]\n",
    "\n",
    "llayer1a = [[],[],[],[],[]]; llayer2a = [[],[],[],[],[]]; llayer3a = [[],[],[],[],[]]\n",
    "llayer4a = [[],[],[],[],[]]; llayer5a = [[],[],[],[],[]]; llayer6a = [[],[],[],[],[]]\n",
    "\n",
    "weight1 = [[],[],[],[],[]]; weight2 = [[],[],[],[],[]]; weight3 = [[],[],[],[],[]];\n",
    "weight4 = [[],[],[],[],[]]; weight5 = [[],[],[],[],[]]; weight6 = [[],[],[],[],[]];\n",
    "\n",
    "gradw1  = [[],[],[],[],[]]; gradw2  = [[],[],[],[],[]]; gradw3  = [[],[],[],[],[]];\n",
    "gradw4  = [[],[],[],[],[]]; gradw5  = [[],[],[],[],[]]; gradw6  = [[],[],[],[],[]];\n",
    "\n",
    "gradp1  = [[],[],[],[],[]]; gradp2  = [[],[],[],[],[]]; gradp3  = [[],[],[],[],[]];\n",
    "gradp4  = [[],[],[],[],[]]; gradp5  = [[],[],[],[],[]]; gradp6  = [[],[],[],[],[]];\n",
    "\n",
    "gradup1  = [[],[],[],[],[]]; gradup2  = [[],[],[],[],[]]; gradup3  = [[],[],[],[],[]];\n",
    "gradup4  = [[],[],[],[],[]]; gradup5  = [[],[],[],[],[]]; gradup6  = [[],[],[],[],[]];\n",
    "\n",
    "list_of_outputs = [\n",
    "    layer1,layer2,layer3,layer4,layer5,layer6,\n",
    "    layer1a,layer2a,layer3a,layer4a,layer5a,layer6a,\n",
    "    l1.getw(),l2.getw(),l3.getw(),l4.getw(),l5.getw(),l6.getw(),\n",
    "    grad1w,grad2w,grad3w,grad4w,grad5w,grad6w,\n",
    "    grad1p,grad2p,grad3p,grad4p,grad5p,grad6p,\n",
    "    grad1_up[0],grad2_up[0],grad3_up[0],grad4_up[0],grad5_up[0],grad6_up[0]\n",
    "]\n",
    "\n",
    "for iter in range(num_epoch):\n",
    "\n",
    "    # Training Accuracy    \n",
    "    for current_batch_index in range(0,len(train_images),batch_size):\n",
    "        current_data  = train_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = train_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy,gradient_update],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(train_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_train = avg_acc_train + sess_results[0]\n",
    "        \n",
    "    # get the results\n",
    "    mid_stat = sess.run(list_of_outputs,feed_dict={x:current_data,y:current_label})\n",
    "    \n",
    "    # Test Accuracy    \n",
    "    for current_batch_index in range(0,len(test_images), batch_size):\n",
    "        current_data  = test_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = test_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(test_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_test = avg_acc_test + sess_results[0]   \n",
    "        \n",
    "    # ======================== extract stats ========================\n",
    "    llayer1 = append_stat(llayer1,mid_stat,0);  llayer2 = append_stat(llayer2,mid_stat,1);  llayer3 = append_stat(llayer3,mid_stat,2);\n",
    "    llayer4 = append_stat(llayer4,mid_stat,3);  llayer5 = append_stat(llayer5,mid_stat,4);  llayer6 = append_stat(llayer6,mid_stat,5);\n",
    "\n",
    "    llayer1a = append_stat(llayer1a,mid_stat,6);  llayer2a = append_stat(llayer2a,mid_stat,7);  llayer3a = append_stat(llayer3a,mid_stat,8);\n",
    "    llayer4a = append_stat(llayer4a,mid_stat,9);  llayer5a = append_stat(llayer5a,mid_stat,10); llayer6a = append_stat(llayer6a,mid_stat,11);\n",
    "    \n",
    "    weight1 = append_stat(weight1,mid_stat,12);  weight2 = append_stat(weight2,mid_stat,13);  weight3 = append_stat(weight3,mid_stat,14);\n",
    "    weight4 = append_stat(weight4,mid_stat,15);  weight5 = append_stat(weight5,mid_stat,16);  weight6 = append_stat(weight6,mid_stat,17);\n",
    "    \n",
    "    gradw1 = append_stat(gradw1,mid_stat,18); gradw2 = append_stat(gradw2,mid_stat,19); gradw3 = append_stat(gradw3,mid_stat,20);\n",
    "    gradw4 = append_stat(gradw4,mid_stat,21); gradw5 = append_stat(gradw5,mid_stat,22); gradw6 = append_stat(gradw6,mid_stat,23);\n",
    "    \n",
    "    gradp1 = append_stat(gradp1,mid_stat,24); gradp2 = append_stat(gradp2,mid_stat,25); gradp3 = append_stat(gradp3,mid_stat,26);\n",
    "    gradp4 = append_stat(gradp4,mid_stat,27); gradp5 = append_stat(gradp5,mid_stat,28); gradp6 = append_stat(gradp6,mid_stat,29);\n",
    "\n",
    "    gradup1 = append_stat(gradup1,mid_stat,30); gradup2 = append_stat(gradup2,mid_stat,31); gradup3 = append_stat(gradup3,mid_stat,32);\n",
    "    gradup4 = append_stat(gradup4,mid_stat,33); gradup5 = append_stat(gradup5,mid_stat,34); gradup6 = append_stat(gradup6,mid_stat,35);\n",
    "\n",
    "    train_acc.append(avg_acc_train/(len(train_images)/batch_size))\n",
    "    test_acc .append(avg_acc_test / (len(test_images)/batch_size))\n",
    "    # ======================== extract stats ========================\n",
    "    \n",
    "    # ======================== save to image ========================\n",
    "    save_to_image(mid_stat[0:6]   ,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,\"layer\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[6:12]  ,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a,\"layera\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[12:18] ,weight1,weight2,weight3,weight4,weight5,weight6,\"weights\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[18:24] ,gradw1,gradw2,gradw3,gradw4,gradw5,gradw6,\"gradientw\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[24:30] ,gradp1,gradp2,gradp3,gradp4,gradp5,gradp6,\"gradientp\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[30:36] ,gradup1,gradup2,gradup3,gradup4,gradup5,gradup6,\"moment\",train_acc,test_acc,current_exp_name,iter)\n",
    "    # ======================== save to image ========================\n",
    "        \n",
    "    # ======================== print reset ========================\n",
    "    print(\"Current : \"+ str(iter) + \" Train Acc : \" + str(avg_acc_train/(len(train_images)/batch_size)) + \" Test Acc : \" + str(avg_acc_test/(len(test_images)/batch_size)) + '\\n')\n",
    "    avg_acc_train = 0 ; avg_acc_test  = 0\n",
    "    # ======================== print reset ========================\n",
    "\n",
    "np.save(current_exp_name+'/train_acc.npy',train_acc); np.save(current_exp_name+'/test_acc.npy', test_acc)    \n",
    "np.save(current_exp_name+'/llayer1.npy', llayer1);  np.save(current_exp_name+'/llayer2.npy', llayer2);  np.save(current_exp_name+'/llayer3.npy', llayer3); \n",
    "np.save(current_exp_name+'/llayer4.npy', llayer4);  np.save(current_exp_name+'/llayer5.npy', llayer5);  np.save(current_exp_name+'/llayer6.npy', llayer6); \n",
    "\n",
    "np.save(current_exp_name+'/llayer1a.npy', llayer1a);  np.save(current_exp_name+'/llayer2a.npy', llayer2a);  np.save(current_exp_name+'/llayer3a.npy', llayer3a); \n",
    "np.save(current_exp_name+'/llayer4a.npy', llayer4a);  np.save(current_exp_name+'/llayer5a.npy', llayer5a);  np.save(current_exp_name+'/llayer6a.npy', llayer6a); \n",
    "\n",
    "np.save(current_exp_name+'/weight1.npy', weight1);  np.save(current_exp_name+'/weight2.npy', weight2);  np.save(current_exp_name+'/weight3.npy', weight3);  \n",
    "np.save(current_exp_name+'/weight4.npy', weight4);  np.save(current_exp_name+'/weight5.npy', weight5);  np.save(current_exp_name+'/weight6.npy', weight6);  \n",
    "\n",
    "np.save(current_exp_name+'/gradw1.npy', gradw1); np.save(current_exp_name+'/gradw2.npy', gradw2); np.save(current_exp_name+'/gradw3.npy', gradw3);\n",
    "np.save(current_exp_name+'/gradw4.npy', gradw4); np.save(current_exp_name+'/gradw5.npy', gradw5); np.save(current_exp_name+'/gradw6.npy', gradw6);\n",
    "\n",
    "np.save(current_exp_name+'/gradp1.npy', gradp1); np.save(current_exp_name+'/gradp2.npy', gradp2); np.save(current_exp_name+'/gradp3.npy', gradp3);\n",
    "np.save(current_exp_name+'/gradp4.npy', gradp4); np.save(current_exp_name+'/gradp5.npy', gradp5); np.save(current_exp_name+'/gradp6.npy', gradp6);\n",
    "\n",
    "np.save(current_exp_name+'/gradup1.npy', gradup1); np.save(current_exp_name+'/gradup2.npy', gradup2); np.save(current_exp_name+'/gradup3.npy', gradup3);\n",
    "np.save(current_exp_name+'/gradup4.npy', gradup4); np.save(current_exp_name+'/gradup5.npy', gradup5); np.save(current_exp_name+'/gradup6.npy', gradup6);\n",
    "\n",
    "sess.close(); tf.reset_default_graph();\n",
    "\n",
    "%reset_selective -f l1,l2,l3,l4,l5,l6\n",
    "%reset_selective -f layer1,layer2,layer3,layer4,layer5,layer6\n",
    "%reset_selective -f layer1a,layer2a,layer3a,layer4a,layer5a,layer6a\n",
    "%reset_selective -f train_acc,test_acc,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a\n",
    "%reset_selective -f weight1,weight2,weight3,weight4,weight5,weight6\n",
    "%reset_selective -f gradw1,gradw2,gradw3,gradw4,gradw5,gradw6\n",
    "%reset_selective -f gradp1,gradp2,gradp3,gradp4,gradp5,gradp6\n",
    "%reset_selective -f gradup1,gradup2,gradup3,gradup4,gradup5,gradup6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-05T07:13:34.136557Z",
     "start_time": "2019-01-05T06:24:55.278360Z"
    },
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current : 0 Train Acc : 0.1306000026911497 Test Acc : 0.1513750028051436\n",
      "\n",
      "Current : 1 Train Acc : 0.16440000282227993 Test Acc : 0.18000000331550836\n",
      "\n",
      "Current : 2 Train Acc : 0.16740000273287298 Test Acc : 0.18512500330805778\n",
      "\n",
      "Current : 3 Train Acc : 0.1822000031322241 Test Acc : 0.20650000332854687\n",
      "\n",
      "Current : 4 Train Acc : 0.2102000034302473 Test Acc : 0.2323750031646341\n",
      "\n",
      "Current : 5 Train Acc : 0.23520000298321248 Test Acc : 0.25650000273250045\n",
      "\n",
      "Current : 6 Train Acc : 0.24180000299215318 Test Acc : 0.2665000023879111\n",
      "\n",
      "Current : 7 Train Acc : 0.255400003015995 Test Acc : 0.2702500027231872\n",
      "\n",
      "Current : 8 Train Acc : 0.2594000026732683 Test Acc : 0.2755000024382025\n",
      "\n",
      "Current : 9 Train Acc : 0.2652000028640032 Test Acc : 0.2793750024866313\n",
      "\n",
      "Current : 10 Train Acc : 0.27000000336766244 Test Acc : 0.28562500251457096\n",
      "\n",
      "Current : 11 Train Acc : 0.2710000034272671 Test Acc : 0.2848750032763928\n",
      "\n",
      "Current : 12 Train Acc : 0.2724000031650066 Test Acc : 0.29062500288709997\n",
      "\n",
      "Current : 13 Train Acc : 0.27720000271499157 Test Acc : 0.2942500021960586\n",
      "\n",
      "Current : 14 Train Acc : 0.2760000022500753 Test Acc : 0.3058750022109598\n",
      "\n",
      "Current : 15 Train Acc : 0.2888000023961067 Test Acc : 0.310250002136454\n",
      "\n",
      "Current : 16 Train Acc : 0.3018000024706125 Test Acc : 0.31012500210665167\n",
      "\n",
      "Current : 17 Train Acc : 0.31140000294148923 Test Acc : 0.3131250020116568\n",
      "\n",
      "Current : 18 Train Acc : 0.31860000255703924 Test Acc : 0.3162500018719584\n",
      "\n",
      "Current : 19 Train Acc : 0.3222000015974045 Test Acc : 0.3286250023264438\n",
      "\n",
      "Current : 20 Train Acc : 0.3354000014364719 Test Acc : 0.32950000199489293\n",
      "\n",
      "Current : 21 Train Acc : 0.3328000014722347 Test Acc : 0.33300000220537185\n",
      "\n",
      "Current : 22 Train Acc : 0.3346000015735626 Test Acc : 0.338375001642853\n",
      "\n",
      "Current : 23 Train Acc : 0.3366000009775162 Test Acc : 0.3388750019390136\n",
      "\n",
      "Current : 24 Train Acc : 0.33800000077486037 Test Acc : 0.3407500021997839\n",
      "\n",
      "Current : 25 Train Acc : 0.3412000014185905 Test Acc : 0.341625001905486\n",
      "\n",
      "Current : 26 Train Acc : 0.34380000162124635 Test Acc : 0.3453750016447157\n",
      "\n",
      "Current : 27 Train Acc : 0.3472000018954277 Test Acc : 0.34837500198744237\n",
      "\n",
      "Current : 28 Train Acc : 0.3524000014066696 Test Acc : 0.3513750019203872\n",
      "\n",
      "Current : 29 Train Acc : 0.3558000011444092 Test Acc : 0.35512500188313423\n",
      "\n",
      "Current : 30 Train Acc : 0.3586000016927719 Test Acc : 0.35875000145286323\n",
      "\n",
      "Current : 31 Train Acc : 0.36180000159144404 Test Acc : 0.3627500011026859\n",
      "\n",
      "Current : 32 Train Acc : 0.3632000018656254 Test Acc : 0.3652500009536743\n",
      "\n",
      "Current : 33 Train Acc : 0.3694000016748905 Test Acc : 0.3685000014305115\n",
      "\n",
      "Current : 34 Train Acc : 0.37160000124573705 Test Acc : 0.37037500102072957\n",
      "\n",
      "Current : 35 Train Acc : 0.37280000177025796 Test Acc : 0.3705000014416873\n",
      "\n",
      "Current : 36 Train Acc : 0.37300000149011614 Test Acc : 0.37175000134855507\n",
      "\n",
      "Current : 37 Train Acc : 0.3758000013232231 Test Acc : 0.3746250014938414\n",
      "\n",
      "Current : 38 Train Acc : 0.3760000014305115 Test Acc : 0.3763750010356307\n",
      "\n",
      "Current : 39 Train Acc : 0.37800000175833703 Test Acc : 0.3767500013485551\n",
      "\n",
      "Current : 40 Train Acc : 0.3826000018715858 Test Acc : 0.3795000011101365\n",
      "\n",
      "Current : 41 Train Acc : 0.38680000221729277 Test Acc : 0.3798750012926757\n",
      "\n",
      "Current : 42 Train Acc : 0.38900000232458115 Test Acc : 0.3818750014156103\n",
      "\n",
      "Current : 43 Train Acc : 0.39000000232458115 Test Acc : 0.3847500014677644\n",
      "\n",
      "Current : 44 Train Acc : 0.39200000220537184 Test Acc : 0.3887500016391277\n",
      "\n",
      "Current : 45 Train Acc : 0.3952000017166138 Test Acc : 0.3915000015869737\n",
      "\n",
      "Current : 46 Train Acc : 0.39800000190734863 Test Acc : 0.39300000086426734\n",
      "\n",
      "Current : 47 Train Acc : 0.400800001680851 Test Acc : 0.3961250006780028\n",
      "\n",
      "Current : 48 Train Acc : 0.4040000013709068 Test Acc : 0.3966250005736947\n",
      "\n",
      "Current : 49 Train Acc : 0.40580000162124635 Test Acc : 0.39762500058859584\n",
      "\n",
      "Current : 50 Train Acc : 0.40880000120401383 Test Acc : 0.3986250006780028\n",
      "\n",
      "Current : 51 Train Acc : 0.41160000121593476 Test Acc : 0.4016250004991889\n",
      "\n",
      "Current : 52 Train Acc : 0.41140000185370446 Test Acc : 0.4036250007525086\n",
      "\n",
      "Current : 53 Train Acc : 0.4114000013768673 Test Acc : 0.4041250006482005\n",
      "\n",
      "Current : 54 Train Acc : 0.41400000151991845 Test Acc : 0.4053750004060566\n",
      "\n",
      "Current : 55 Train Acc : 0.41680000159144404 Test Acc : 0.4083750005997717\n",
      "\n",
      "Current : 56 Train Acc : 0.4208000011742115 Test Acc : 0.40937500005587935\n",
      "\n",
      "Current : 57 Train Acc : 0.421200001180172 Test Acc : 0.41087500052526593\n",
      "\n",
      "Current : 58 Train Acc : 0.4228000013232231 Test Acc : 0.41125000042840837\n",
      "\n",
      "Current : 59 Train Acc : 0.42740000104904174 Test Acc : 0.41200000086799266\n",
      "\n",
      "Current : 60 Train Acc : 0.4288000010251999 Test Acc : 0.4115000013820827\n",
      "\n",
      "Current : 61 Train Acc : 0.43180000185966494 Test Acc : 0.41287500141188505\n",
      "\n",
      "Current : 62 Train Acc : 0.4336000010073185 Test Acc : 0.4140000014938414\n",
      "\n",
      "Current : 63 Train Acc : 0.4334000016748905 Test Acc : 0.41412500167265537\n",
      "\n",
      "Current : 64 Train Acc : 0.4340000014603138 Test Acc : 0.4128750014305115\n",
      "\n",
      "Current : 65 Train Acc : 0.43300000098347663 Test Acc : 0.4143750013411045\n",
      "\n",
      "Current : 66 Train Acc : 0.4336000007092953 Test Acc : 0.41500000081956384\n",
      "\n",
      "Current : 67 Train Acc : 0.43720000120997426 Test Acc : 0.41675000090152026\n",
      "\n",
      "Current : 68 Train Acc : 0.43940000066161156 Test Acc : 0.41687500081956386\n",
      "\n",
      "Current : 69 Train Acc : 0.4402000009715557 Test Acc : 0.41725000116974115\n",
      "\n",
      "Current : 70 Train Acc : 0.4420000014603138 Test Acc : 0.4172500006575137\n",
      "\n",
      "Current : 71 Train Acc : 0.4458000009357929 Test Acc : 0.4176250008214265\n",
      "\n",
      "Current : 72 Train Acc : 0.44740000185370443 Test Acc : 0.4206250007078052\n",
      "\n",
      "Current : 73 Train Acc : 0.44800000163912773 Test Acc : 0.4218750004842877\n",
      "\n",
      "Current : 74 Train Acc : 0.4488000017106533 Test Acc : 0.4237500005308539\n",
      "\n",
      "Current : 75 Train Acc : 0.45140000149607656 Test Acc : 0.4246250007022172\n",
      "\n",
      "Current : 76 Train Acc : 0.451800001591444 Test Acc : 0.4252500006277114\n",
      "\n",
      "Current : 77 Train Acc : 0.45440000090003013 Test Acc : 0.4266250008158386\n",
      "\n",
      "Current : 78 Train Acc : 0.45680000153183936 Test Acc : 0.42650000085122886\n",
      "\n",
      "Current : 79 Train Acc : 0.4578000014126301 Test Acc : 0.42812500086613\n",
      "\n",
      "Current : 80 Train Acc : 0.4582000017464161 Test Acc : 0.42737500091083347\n",
      "\n",
      "Current : 81 Train Acc : 0.45840000227093697 Test Acc : 0.4271250007022172\n",
      "\n",
      "Current : 82 Train Acc : 0.46040000185370444 Test Acc : 0.42950000063516197\n",
      "\n",
      "Current : 83 Train Acc : 0.4618000020682812 Test Acc : 0.43137500105425713\n",
      "\n",
      "Current : 84 Train Acc : 0.4636000010073185 Test Acc : 0.431875000949949\n",
      "\n",
      "Current : 85 Train Acc : 0.4642000012099743 Test Acc : 0.4331250008288771\n",
      "\n",
      "Current : 86 Train Acc : 0.4660000011026859 Test Acc : 0.4328750012908131\n",
      "\n",
      "Current : 87 Train Acc : 0.46740000137686727 Test Acc : 0.43700000130571426\n",
      "\n",
      "Current : 88 Train Acc : 0.4688000008761883 Test Acc : 0.437500001238659\n",
      "\n",
      "Current : 89 Train Acc : 0.4710000011026859 Test Acc : 0.43637500071898105\n",
      "\n",
      "Current : 90 Train Acc : 0.4706000012457371 Test Acc : 0.43662500085309147\n",
      "\n",
      "Current : 91 Train Acc : 0.473000000923872 Test Acc : 0.43737500127404927\n",
      "\n",
      "Current : 92 Train Acc : 0.4748000014722347 Test Acc : 0.4366250007972121\n",
      "\n",
      "Current : 93 Train Acc : 0.4764000016748905 Test Acc : 0.4365000009536743\n",
      "\n",
      "Current : 94 Train Acc : 0.477000001758337 Test Acc : 0.43562500085681677\n",
      "\n",
      "Current : 95 Train Acc : 0.4772000019848347 Test Acc : 0.4366250004991889\n",
      "\n",
      "Current : 96 Train Acc : 0.47860000148415566 Test Acc : 0.4367500006407499\n",
      "\n",
      "Current : 97 Train Acc : 0.48080000206828116 Test Acc : 0.4371250008046627\n",
      "\n",
      "Current : 98 Train Acc : 0.4826000025570393 Test Acc : 0.43550000078976153\n",
      "\n",
      "Current : 99 Train Acc : 0.4848000026643276 Test Acc : 0.43637500148266556\n",
      "\n",
      "Current : 100 Train Acc : 0.4858000031411648 Test Acc : 0.43625000115484\n",
      "\n",
      "Current : 101 Train Acc : 0.48760000213980675 Test Acc : 0.43662500064820053\n",
      "\n",
      "Current : 102 Train Acc : 0.49000000187754633 Test Acc : 0.43675000082701443\n",
      "\n",
      "Current : 103 Train Acc : 0.4918000020682812 Test Acc : 0.4362500000372529\n",
      "\n",
      "Current : 104 Train Acc : 0.49560000148415567 Test Acc : 0.4358749998733401\n",
      "\n",
      "Current : 105 Train Acc : 0.4960000014603138 Test Acc : 0.43674999989569185\n",
      "\n",
      "Current : 106 Train Acc : 0.4970000008642673 Test Acc : 0.4376249999180436\n",
      "\n",
      "Current : 107 Train Acc : 0.4982000007927418 Test Acc : 0.437625000923872\n",
      "\n",
      "Current : 108 Train Acc : 0.4978000010550022 Test Acc : 0.4386250010877848\n",
      "\n",
      "Current : 109 Train Acc : 0.4986000011265278 Test Acc : 0.4393750009313226\n",
      "\n",
      "Current : 110 Train Acc : 0.4994000017940998 Test Acc : 0.43825000043958423\n",
      "\n",
      "Current : 111 Train Acc : 0.5016000012457371 Test Acc : 0.4368750011920929\n",
      "\n",
      "Current : 112 Train Acc : 0.5004000017642974 Test Acc : 0.43812500085681677\n",
      "\n",
      "Current : 113 Train Acc : 0.5010000017881393 Test Acc : 0.43925000112503765\n",
      "\n",
      "Current : 114 Train Acc : 0.5022000014781952 Test Acc : 0.4401250010356307\n",
      "\n",
      "Current : 115 Train Acc : 0.5026000015735627 Test Acc : 0.43962500054389236\n",
      "\n",
      "Current : 116 Train Acc : 0.5064000015258789 Test Acc : 0.4398750004172325\n",
      "\n",
      "Current : 117 Train Acc : 0.5066000014543534 Test Acc : 0.440875000692904\n",
      "\n",
      "Current : 118 Train Acc : 0.506400000989437 Test Acc : 0.4413750009983778\n",
      "\n",
      "Current : 119 Train Acc : 0.5078000018000602 Test Acc : 0.44200000070035456\n",
      "\n",
      "Current : 120 Train Acc : 0.5096000017523765 Test Acc : 0.4428750006482005\n",
      "\n",
      "Current : 121 Train Acc : 0.5122000017762184 Test Acc : 0.44462500102818014\n",
      "\n",
      "Current : 122 Train Acc : 0.5142000015377999 Test Acc : 0.4463750009611249\n",
      "\n",
      "Current : 123 Train Acc : 0.5130000010132789 Test Acc : 0.44675000105053186\n",
      "\n",
      "Current : 124 Train Acc : 0.5150000013709068 Test Acc : 0.4466250009834766\n",
      "\n",
      "Current : 125 Train Acc : 0.5152000009417533 Test Acc : 0.4481250014528632\n",
      "\n",
      "Current : 126 Train Acc : 0.5172000009417533 Test Acc : 0.44650000128895045\n",
      "\n",
      "Current : 127 Train Acc : 0.5192000009417533 Test Acc : 0.44837500143796205\n",
      "\n",
      "Current : 128 Train Acc : 0.5186000000834465 Test Acc : 0.44787500109523537\n",
      "\n",
      "Current : 129 Train Acc : 0.5195999997258186 Test Acc : 0.44787500109523537\n",
      "\n",
      "Current : 130 Train Acc : 0.5208000003099441 Test Acc : 0.4501250010728836\n",
      "\n",
      "Current : 131 Train Acc : 0.5216000001430512 Test Acc : 0.4512500008940697\n",
      "\n",
      "Current : 132 Train Acc : 0.5220000004768371 Test Acc : 0.4525000007450581\n",
      "\n",
      "Current : 133 Train Acc : 0.5228000003099441 Test Acc : 0.452125001065433\n",
      "\n",
      "Current : 134 Train Acc : 0.522600000500679 Test Acc : 0.4522500007227063\n",
      "\n",
      "Current : 135 Train Acc : 0.523600000500679 Test Acc : 0.4522500003874302\n",
      "\n",
      "Current : 136 Train Acc : 0.5248000011444092 Test Acc : 0.4518750005215406\n",
      "\n",
      "Current : 137 Train Acc : 0.5266000014543534 Test Acc : 0.4527500008791685\n",
      "\n",
      "Current : 138 Train Acc : 0.5282000017166137 Test Acc : 0.45162500023841856\n",
      "\n",
      "Current : 139 Train Acc : 0.5270000017881393 Test Acc : 0.45187500040978196\n",
      "\n",
      "Current : 140 Train Acc : 0.5294000017642975 Test Acc : 0.4501250004768372\n",
      "\n",
      "Current : 141 Train Acc : 0.5282000014781952 Test Acc : 0.4505000005662441\n",
      "\n",
      "Current : 142 Train Acc : 0.529200002193451 Test Acc : 0.4517500004917383\n",
      "\n",
      "Current : 143 Train Acc : 0.5300000022649765 Test Acc : 0.45250000063329937\n",
      "\n",
      "Current : 144 Train Acc : 0.5298000024557113 Test Acc : 0.4535000006482005\n",
      "\n",
      "Current : 145 Train Acc : 0.5318000019788742 Test Acc : 0.45375000063329934\n",
      "\n",
      "Current : 146 Train Acc : 0.53200000166893 Test Acc : 0.45512500062584876\n",
      "\n",
      "Current : 147 Train Acc : 0.5322000020742417 Test Acc : 0.45625000074505806\n",
      "\n",
      "Current : 148 Train Acc : 0.53180000269413 Test Acc : 0.4556250011175871\n",
      "\n",
      "Current : 149 Train Acc : 0.5340000025033951 Test Acc : 0.45637500058859587\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# F\n",
    "current_exp_name = 'F';\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "# create layers\n",
    "l1 = CNN(3,3, 16,which_reg=current_exp_name); \n",
    "l2 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l3 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "\n",
    "l4 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l5 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l6 = CNN(3,16,10,which_reg=current_exp_name); \n",
    "\n",
    "# 2. graph \n",
    "x = tf.placeholder(tf.float32,(batch_size,96,96,3))\n",
    "y = tf.placeholder(tf.float32,(batch_size,10))\n",
    "\n",
    "layer1, layer1a = l1. feedforward(x,stride=2)\n",
    "layer2, layer2a = l2. feedforward(layer1a,stride=2)\n",
    "layer3, layer3a = l3. feedforward(layer2a,stride=2)\n",
    "layer4, layer4a = l4. feedforward(layer3a,stride=2)\n",
    "layer5, layer5a = l5. feedforward(layer4a)\n",
    "layer6, layer6a = l6. feedforward(layer5a)\n",
    "\n",
    "final_layer   = tf.reduce_mean(layer6a,(1,2))\n",
    "final_softmax = tf_softmax(final_layer)\n",
    "cost          = -tf.reduce_mean(y * tf.log(final_softmax + 1e-8))\n",
    "correct_prediction = tf.equal(tf.argmax(final_softmax, 1), tf.argmax(y, 1))\n",
    "accuracy           = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "gradient = tf.tile((final_softmax-y)[:,None,None,:],[1,6,6,1])/batch_size\n",
    "grad6p,grad6w,grad6_up = l6.backprop(gradient)\n",
    "grad5p,grad5w,grad5_up = l5.backprop(grad6p)\n",
    "grad4p,grad4w,grad4_up = l4.backprop(grad5p,stride=2)\n",
    "grad3p,grad3w,grad3_up = l3.backprop(grad4p,stride=2)\n",
    "grad2p,grad2w,grad2_up = l2.backprop(grad3p,stride=2)\n",
    "grad1p,grad1w,grad1_up = l1.backprop(grad2p,stride=2)\n",
    "\n",
    "gradient_update = grad6_up + grad5_up + grad4_up + grad3_up + grad2_up + grad1_up \n",
    "\n",
    "# train\n",
    "sess.run(tf.global_variables_initializer())\n",
    "avg_acc_train = 0; avg_acc_test  = 0; train_acc = [];test_acc = []\n",
    "\n",
    "# mean std skew kurt non-zero\n",
    "llayer1 = [[],[],[],[],[]]; llayer2 = [[],[],[],[],[]]; llayer3 = [[],[],[],[],[]]\n",
    "llayer4 = [[],[],[],[],[]]; llayer5 = [[],[],[],[],[]]; llayer6 = [[],[],[],[],[]]\n",
    "\n",
    "llayer1a = [[],[],[],[],[]]; llayer2a = [[],[],[],[],[]]; llayer3a = [[],[],[],[],[]]\n",
    "llayer4a = [[],[],[],[],[]]; llayer5a = [[],[],[],[],[]]; llayer6a = [[],[],[],[],[]]\n",
    "\n",
    "weight1 = [[],[],[],[],[]]; weight2 = [[],[],[],[],[]]; weight3 = [[],[],[],[],[]];\n",
    "weight4 = [[],[],[],[],[]]; weight5 = [[],[],[],[],[]]; weight6 = [[],[],[],[],[]];\n",
    "\n",
    "gradw1  = [[],[],[],[],[]]; gradw2  = [[],[],[],[],[]]; gradw3  = [[],[],[],[],[]];\n",
    "gradw4  = [[],[],[],[],[]]; gradw5  = [[],[],[],[],[]]; gradw6  = [[],[],[],[],[]];\n",
    "\n",
    "gradp1  = [[],[],[],[],[]]; gradp2  = [[],[],[],[],[]]; gradp3  = [[],[],[],[],[]];\n",
    "gradp4  = [[],[],[],[],[]]; gradp5  = [[],[],[],[],[]]; gradp6  = [[],[],[],[],[]];\n",
    "\n",
    "gradup1  = [[],[],[],[],[]]; gradup2  = [[],[],[],[],[]]; gradup3  = [[],[],[],[],[]];\n",
    "gradup4  = [[],[],[],[],[]]; gradup5  = [[],[],[],[],[]]; gradup6  = [[],[],[],[],[]];\n",
    "\n",
    "list_of_outputs = [\n",
    "    layer1,layer2,layer3,layer4,layer5,layer6,\n",
    "    layer1a,layer2a,layer3a,layer4a,layer5a,layer6a,\n",
    "    l1.getw(),l2.getw(),l3.getw(),l4.getw(),l5.getw(),l6.getw(),\n",
    "    grad1w,grad2w,grad3w,grad4w,grad5w,grad6w,\n",
    "    grad1p,grad2p,grad3p,grad4p,grad5p,grad6p,\n",
    "    grad1_up[0],grad2_up[0],grad3_up[0],grad4_up[0],grad5_up[0],grad6_up[0]\n",
    "]\n",
    "\n",
    "for iter in range(num_epoch):\n",
    "\n",
    "    # Training Accuracy    \n",
    "    for current_batch_index in range(0,len(train_images),batch_size):\n",
    "        current_data  = train_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = train_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy,gradient_update],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(train_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_train = avg_acc_train + sess_results[0]\n",
    "        \n",
    "    # get the results\n",
    "    mid_stat = sess.run(list_of_outputs,feed_dict={x:current_data,y:current_label})\n",
    "    \n",
    "    # Test Accuracy    \n",
    "    for current_batch_index in range(0,len(test_images), batch_size):\n",
    "        current_data  = test_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = test_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(test_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_test = avg_acc_test + sess_results[0]   \n",
    "        \n",
    "    # ======================== extract stats ========================\n",
    "    llayer1 = append_stat(llayer1,mid_stat,0);  llayer2 = append_stat(llayer2,mid_stat,1);  llayer3 = append_stat(llayer3,mid_stat,2);\n",
    "    llayer4 = append_stat(llayer4,mid_stat,3);  llayer5 = append_stat(llayer5,mid_stat,4);  llayer6 = append_stat(llayer6,mid_stat,5);\n",
    "\n",
    "    llayer1a = append_stat(llayer1a,mid_stat,6);  llayer2a = append_stat(llayer2a,mid_stat,7);  llayer3a = append_stat(llayer3a,mid_stat,8);\n",
    "    llayer4a = append_stat(llayer4a,mid_stat,9);  llayer5a = append_stat(llayer5a,mid_stat,10); llayer6a = append_stat(llayer6a,mid_stat,11);\n",
    "    \n",
    "    weight1 = append_stat(weight1,mid_stat,12);  weight2 = append_stat(weight2,mid_stat,13);  weight3 = append_stat(weight3,mid_stat,14);\n",
    "    weight4 = append_stat(weight4,mid_stat,15);  weight5 = append_stat(weight5,mid_stat,16);  weight6 = append_stat(weight6,mid_stat,17);\n",
    "    \n",
    "    gradw1 = append_stat(gradw1,mid_stat,18); gradw2 = append_stat(gradw2,mid_stat,19); gradw3 = append_stat(gradw3,mid_stat,20);\n",
    "    gradw4 = append_stat(gradw4,mid_stat,21); gradw5 = append_stat(gradw5,mid_stat,22); gradw6 = append_stat(gradw6,mid_stat,23);\n",
    "    \n",
    "    gradp1 = append_stat(gradp1,mid_stat,24); gradp2 = append_stat(gradp2,mid_stat,25); gradp3 = append_stat(gradp3,mid_stat,26);\n",
    "    gradp4 = append_stat(gradp4,mid_stat,27); gradp5 = append_stat(gradp5,mid_stat,28); gradp6 = append_stat(gradp6,mid_stat,29);\n",
    "\n",
    "    gradup1 = append_stat(gradup1,mid_stat,30); gradup2 = append_stat(gradup2,mid_stat,31); gradup3 = append_stat(gradup3,mid_stat,32);\n",
    "    gradup4 = append_stat(gradup4,mid_stat,33); gradup5 = append_stat(gradup5,mid_stat,34); gradup6 = append_stat(gradup6,mid_stat,35);\n",
    "\n",
    "    train_acc.append(avg_acc_train/(len(train_images)/batch_size))\n",
    "    test_acc .append(avg_acc_test / (len(test_images)/batch_size))\n",
    "    # ======================== extract stats ========================\n",
    "    \n",
    "    # ======================== save to image ========================\n",
    "    save_to_image(mid_stat[0:6]   ,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,\"layer\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[6:12]  ,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a,\"layera\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[12:18] ,weight1,weight2,weight3,weight4,weight5,weight6,\"weights\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[18:24] ,gradw1,gradw2,gradw3,gradw4,gradw5,gradw6,\"gradientw\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[24:30] ,gradp1,gradp2,gradp3,gradp4,gradp5,gradp6,\"gradientp\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[30:36] ,gradup1,gradup2,gradup3,gradup4,gradup5,gradup6,\"moment\",train_acc,test_acc,current_exp_name,iter)\n",
    "    # ======================== save to image ========================\n",
    "        \n",
    "    # ======================== print reset ========================\n",
    "    print(\"Current : \"+ str(iter) + \" Train Acc : \" + str(avg_acc_train/(len(train_images)/batch_size)) + \" Test Acc : \" + str(avg_acc_test/(len(test_images)/batch_size)) + '\\n')\n",
    "    avg_acc_train = 0 ; avg_acc_test  = 0\n",
    "    # ======================== print reset ========================\n",
    "\n",
    "np.save(current_exp_name+'/train_acc.npy',train_acc); np.save(current_exp_name+'/test_acc.npy', test_acc)    \n",
    "np.save(current_exp_name+'/llayer1.npy', llayer1);  np.save(current_exp_name+'/llayer2.npy', llayer2);  np.save(current_exp_name+'/llayer3.npy', llayer3); \n",
    "np.save(current_exp_name+'/llayer4.npy', llayer4);  np.save(current_exp_name+'/llayer5.npy', llayer5);  np.save(current_exp_name+'/llayer6.npy', llayer6); \n",
    "\n",
    "np.save(current_exp_name+'/llayer1a.npy', llayer1a);  np.save(current_exp_name+'/llayer2a.npy', llayer2a);  np.save(current_exp_name+'/llayer3a.npy', llayer3a); \n",
    "np.save(current_exp_name+'/llayer4a.npy', llayer4a);  np.save(current_exp_name+'/llayer5a.npy', llayer5a);  np.save(current_exp_name+'/llayer6a.npy', llayer6a); \n",
    "\n",
    "np.save(current_exp_name+'/weight1.npy', weight1);  np.save(current_exp_name+'/weight2.npy', weight2);  np.save(current_exp_name+'/weight3.npy', weight3);  \n",
    "np.save(current_exp_name+'/weight4.npy', weight4);  np.save(current_exp_name+'/weight5.npy', weight5);  np.save(current_exp_name+'/weight6.npy', weight6);  \n",
    "\n",
    "np.save(current_exp_name+'/gradw1.npy', gradw1); np.save(current_exp_name+'/gradw2.npy', gradw2); np.save(current_exp_name+'/gradw3.npy', gradw3);\n",
    "np.save(current_exp_name+'/gradw4.npy', gradw4); np.save(current_exp_name+'/gradw5.npy', gradw5); np.save(current_exp_name+'/gradw6.npy', gradw6);\n",
    "\n",
    "np.save(current_exp_name+'/gradp1.npy', gradp1); np.save(current_exp_name+'/gradp2.npy', gradp2); np.save(current_exp_name+'/gradp3.npy', gradp3);\n",
    "np.save(current_exp_name+'/gradp4.npy', gradp4); np.save(current_exp_name+'/gradp5.npy', gradp5); np.save(current_exp_name+'/gradp6.npy', gradp6);\n",
    "\n",
    "np.save(current_exp_name+'/gradup1.npy', gradup1); np.save(current_exp_name+'/gradup2.npy', gradup2); np.save(current_exp_name+'/gradup3.npy', gradup3);\n",
    "np.save(current_exp_name+'/gradup4.npy', gradup4); np.save(current_exp_name+'/gradup5.npy', gradup5); np.save(current_exp_name+'/gradup6.npy', gradup6);\n",
    "\n",
    "sess.close(); tf.reset_default_graph();\n",
    "\n",
    "%reset_selective -f l1,l2,l3,l4,l5,l6\n",
    "%reset_selective -f layer1,layer2,layer3,layer4,layer5,layer6\n",
    "%reset_selective -f layer1a,layer2a,layer3a,layer4a,layer5a,layer6a\n",
    "%reset_selective -f train_acc,test_acc,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a\n",
    "%reset_selective -f weight1,weight2,weight3,weight4,weight5,weight6\n",
    "%reset_selective -f gradw1,gradw2,gradw3,gradw4,gradw5,gradw6\n",
    "%reset_selective -f gradp1,gradp2,gradp3,gradp4,gradp5,gradp6\n",
    "%reset_selective -f gradup1,gradup2,gradup3,gradup4,gradup5,gradup6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-05T08:03:07.384001Z",
     "start_time": "2019-01-05T07:13:34.160509Z"
    },
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current : 0 Train Acc : 0.12100000256299973 Test Acc : 0.12725000255741178\n",
      "\n",
      "Current : 1 Train Acc : 0.17580000293254852 Test Acc : 0.17625000304542482\n",
      "\n",
      "Current : 2 Train Acc : 0.20820000354945659 Test Acc : 0.23425000340677798\n",
      "\n",
      "Current : 3 Train Acc : 0.26400000271201135 Test Acc : 0.27900000267662106\n",
      "\n",
      "Current : 4 Train Acc : 0.29400000210106375 Test Acc : 0.2908750024344772\n",
      "\n",
      "Current : 5 Train Acc : 0.3216000020503998 Test Acc : 0.334125001905486\n",
      "\n",
      "Current : 6 Train Acc : 0.338800002142787 Test Acc : 0.35450000101700424\n",
      "\n",
      "Current : 7 Train Acc : 0.3538000019490719 Test Acc : 0.359625001186505\n",
      "\n",
      "Current : 8 Train Acc : 0.367600001513958 Test Acc : 0.3646250015683472\n",
      "\n",
      "Current : 9 Train Acc : 0.37560000106692315 Test Acc : 0.3672500008903444\n",
      "\n",
      "Current : 10 Train Acc : 0.38960000106692316 Test Acc : 0.37550000080838797\n",
      "\n",
      "Current : 11 Train Acc : 0.3996000011563301 Test Acc : 0.37537500135600566\n",
      "\n",
      "Current : 12 Train Acc : 0.40080000001192095 Test Acc : 0.38412500109523534\n",
      "\n",
      "Current : 13 Train Acc : 0.4062000007033348 Test Acc : 0.3875000009313226\n",
      "\n",
      "Current : 14 Train Acc : 0.4114000008702278 Test Acc : 0.3930000007338822\n",
      "\n",
      "Current : 15 Train Acc : 0.41600000074505805 Test Acc : 0.3948750008456409\n",
      "\n",
      "Current : 16 Train Acc : 0.4162000009119511 Test Acc : 0.40212500086054204\n",
      "\n",
      "Current : 17 Train Acc : 0.4240000011920929 Test Acc : 0.40025000175461173\n",
      "\n",
      "Current : 18 Train Acc : 0.4254000018835068 Test Acc : 0.38887500077486037\n",
      "\n",
      "Current : 19 Train Acc : 0.4308000018000603 Test Acc : 0.39287500059232117\n",
      "\n",
      "Current : 20 Train Acc : 0.43640000200271606 Test Acc : 0.39775000093504786\n",
      "\n",
      "Current : 21 Train Acc : 0.43940000212192537 Test Acc : 0.4007500009611249\n",
      "\n",
      "Current : 22 Train Acc : 0.44400000125169753 Test Acc : 0.4062500012665987\n",
      "\n",
      "Current : 23 Train Acc : 0.4450000002980232 Test Acc : 0.4106250012293458\n",
      "\n",
      "Current : 24 Train Acc : 0.44620000064373017 Test Acc : 0.4135000010952353\n",
      "\n",
      "Current : 25 Train Acc : 0.4496000007390976 Test Acc : 0.41787500098347663\n",
      "\n",
      "Current : 26 Train Acc : 0.4520000009536743 Test Acc : 0.42312500175088646\n",
      "\n",
      "Current : 27 Train Acc : 0.45240000092983246 Test Acc : 0.42625000182539224\n",
      "\n",
      "Current : 28 Train Acc : 0.4568000013232231 Test Acc : 0.4293750014528632\n",
      "\n",
      "Current : 29 Train Acc : 0.4598000013828278 Test Acc : 0.4318750016018748\n",
      "\n",
      "Current : 30 Train Acc : 0.46100000125169754 Test Acc : 0.43500000175088643\n",
      "\n",
      "Current : 31 Train Acc : 0.4656000015735626 Test Acc : 0.43825000170618295\n",
      "\n",
      "Current : 32 Train Acc : 0.46760000109672545 Test Acc : 0.43925000190734864\n",
      "\n",
      "Current : 33 Train Acc : 0.47000000125169755 Test Acc : 0.44150000140070916\n",
      "\n",
      "Current : 34 Train Acc : 0.47120000183582306 Test Acc : 0.4423750012740493\n",
      "\n",
      "Current : 35 Train Acc : 0.4752000017762184 Test Acc : 0.4435000015050173\n",
      "\n",
      "Current : 36 Train Acc : 0.4764000017046928 Test Acc : 0.4452500017732382\n",
      "\n",
      "Current : 37 Train Acc : 0.4780000013709068 Test Acc : 0.44612500227987767\n",
      "\n",
      "Current : 38 Train Acc : 0.480600001513958 Test Acc : 0.4482500018551946\n",
      "\n",
      "Current : 39 Train Acc : 0.48220000088214876 Test Acc : 0.4493750012293458\n",
      "\n",
      "Current : 40 Train Acc : 0.48320000094175336 Test Acc : 0.45012500137090683\n",
      "\n",
      "Current : 41 Train Acc : 0.48520000153779985 Test Acc : 0.4531250014528632\n",
      "\n",
      "Current : 42 Train Acc : 0.486000001847744 Test Acc : 0.4538750013709068\n",
      "\n",
      "Current : 43 Train Acc : 0.4880000016093254 Test Acc : 0.45775000110268593\n",
      "\n",
      "Current : 44 Train Acc : 0.4898000016212463 Test Acc : 0.4573750011622906\n",
      "\n",
      "Current : 45 Train Acc : 0.4924000014066696 Test Acc : 0.45612500090152025\n",
      "\n",
      "Current : 46 Train Acc : 0.4956000010967255 Test Acc : 0.4571250012516975\n",
      "\n",
      "Current : 47 Train Acc : 0.5002000010013581 Test Acc : 0.4578750004991889\n",
      "\n",
      "Current : 48 Train Acc : 0.5034000010490417 Test Acc : 0.4583750007674098\n",
      "\n",
      "Current : 49 Train Acc : 0.5070000013709068 Test Acc : 0.4605000004544854\n",
      "\n",
      "Current : 50 Train Acc : 0.509600000679493 Test Acc : 0.462250000461936\n",
      "\n",
      "Current : 51 Train Acc : 0.5122000009417534 Test Acc : 0.46137500043958424\n",
      "\n",
      "Current : 52 Train Acc : 0.513800000846386 Test Acc : 0.46300000060349705\n",
      "\n",
      "Current : 53 Train Acc : 0.5168000006079674 Test Acc : 0.4632500010356307\n",
      "\n",
      "Current : 54 Train Acc : 0.5204000006318092 Test Acc : 0.46562500096857545\n",
      "\n",
      "Current : 55 Train Acc : 0.5204000001549721 Test Acc : 0.46437500074505805\n",
      "\n",
      "Current : 56 Train Acc : 0.522400000333786 Test Acc : 0.46562500022351744\n",
      "\n",
      "Current : 57 Train Acc : 0.5231999999880791 Test Acc : 0.4655000005289912\n",
      "\n",
      "Current : 58 Train Acc : 0.5246000002622604 Test Acc : 0.4663749999180436\n",
      "\n",
      "Current : 59 Train Acc : 0.5248000012040138 Test Acc : 0.466500000320375\n",
      "\n",
      "Current : 60 Train Acc : 0.5256000007390976 Test Acc : 0.4665000002458692\n",
      "\n",
      "Current : 61 Train Acc : 0.5272000011205673 Test Acc : 0.465875000320375\n",
      "\n",
      "Current : 62 Train Acc : 0.5294000006914139 Test Acc : 0.4660000007972121\n",
      "\n",
      "Current : 63 Train Acc : 0.5304000016450882 Test Acc : 0.4672500009462237\n",
      "\n",
      "Current : 64 Train Acc : 0.5328000017404556 Test Acc : 0.46600000113248824\n",
      "\n",
      "Current : 65 Train Acc : 0.5354000021219254 Test Acc : 0.4646250007674098\n",
      "\n",
      "Current : 66 Train Acc : 0.534800002336502 Test Acc : 0.46650000113993884\n",
      "\n",
      "Current : 67 Train Acc : 0.5366000026464463 Test Acc : 0.46725000116974114\n",
      "\n",
      "Current : 68 Train Acc : 0.5374000027179718 Test Acc : 0.46775000136345624\n",
      "\n",
      "Current : 69 Train Acc : 0.5388000029325485 Test Acc : 0.4698750013485551\n",
      "\n",
      "Current : 70 Train Acc : 0.5414000036716461 Test Acc : 0.4711250014230609\n",
      "\n",
      "Current : 71 Train Acc : 0.5442000029087066 Test Acc : 0.4720000014826655\n",
      "\n",
      "Current : 72 Train Acc : 0.5466000028848648 Test Acc : 0.47250000182539226\n",
      "\n",
      "Current : 73 Train Acc : 0.5496000028848648 Test Acc : 0.47287500187754633\n",
      "\n",
      "Current : 74 Train Acc : 0.5508000024557114 Test Acc : 0.47300000209361315\n",
      "\n",
      "Current : 75 Train Acc : 0.5526000020503998 Test Acc : 0.4726250021532178\n",
      "\n",
      "Current : 76 Train Acc : 0.5528000017404556 Test Acc : 0.47112500175833705\n",
      "\n",
      "Current : 77 Train Acc : 0.5566000016927719 Test Acc : 0.47312500201165675\n",
      "\n",
      "Current : 78 Train Acc : 0.5578000016212463 Test Acc : 0.47300000179558993\n",
      "\n",
      "Current : 79 Train Acc : 0.5588000023365021 Test Acc : 0.4722500021010637\n",
      "\n",
      "Current : 80 Train Acc : 0.5604000014066696 Test Acc : 0.4718750024959445\n",
      "\n",
      "Current : 81 Train Acc : 0.5622000018358231 Test Acc : 0.4720000025257468\n",
      "\n",
      "Current : 82 Train Acc : 0.5650000010728836 Test Acc : 0.47175000235438347\n",
      "\n",
      "Current : 83 Train Acc : 0.566600001335144 Test Acc : 0.47237500190734866\n",
      "\n",
      "Current : 84 Train Acc : 0.5688000017404556 Test Acc : 0.47200000174343587\n",
      "\n",
      "Current : 85 Train Acc : 0.5698000020980835 Test Acc : 0.47112500190734863\n",
      "\n",
      "Current : 86 Train Acc : 0.5718000022172928 Test Acc : 0.47137500159442425\n",
      "\n",
      "Current : 87 Train Acc : 0.5732000021934509 Test Acc : 0.47262500151991843\n",
      "\n",
      "Current : 88 Train Acc : 0.5756000015735626 Test Acc : 0.4721250016987324\n",
      "\n",
      "Current : 89 Train Acc : 0.5772000024318695 Test Acc : 0.47287500198930504\n",
      "\n",
      "Current : 90 Train Acc : 0.5800000026226043 Test Acc : 0.47312500156462195\n",
      "\n",
      "Current : 91 Train Acc : 0.581400002360344 Test Acc : 0.47262500137090685\n",
      "\n",
      "Current : 92 Train Acc : 0.5832000020742416 Test Acc : 0.47300000179558993\n",
      "\n",
      "Current : 93 Train Acc : 0.5840000015497208 Test Acc : 0.4751250018924475\n",
      "\n",
      "Current : 94 Train Acc : 0.5866000016927719 Test Acc : 0.4752500024810433\n",
      "\n",
      "Current : 95 Train Acc : 0.588800001859665 Test Acc : 0.4750000026449561\n",
      "\n",
      "Current : 96 Train Acc : 0.5914000010490418 Test Acc : 0.4752500020712614\n",
      "\n",
      "Current : 97 Train Acc : 0.591800001859665 Test Acc : 0.4751250015944242\n",
      "\n",
      "Current : 98 Train Acc : 0.5942000018358231 Test Acc : 0.4747500019147992\n",
      "\n",
      "Current : 99 Train Acc : 0.5942000018358231 Test Acc : 0.47425000194460154\n",
      "\n",
      "Current : 100 Train Acc : 0.5948000010251999 Test Acc : 0.4733750021085143\n",
      "\n",
      "Current : 101 Train Acc : 0.5984000009298325 Test Acc : 0.4746250022575259\n",
      "\n",
      "Current : 102 Train Acc : 0.598800000667572 Test Acc : 0.4716250021755695\n",
      "\n",
      "Current : 103 Train Acc : 0.6020000007152557 Test Acc : 0.47175000198185446\n",
      "\n",
      "Current : 104 Train Acc : 0.601800001502037 Test Acc : 0.4713750015571713\n",
      "\n",
      "Current : 105 Train Acc : 0.6044000010490418 Test Acc : 0.4723750017210841\n",
      "\n",
      "Current : 106 Train Acc : 0.6068000000715256 Test Acc : 0.4727500019967556\n",
      "\n",
      "Current : 107 Train Acc : 0.6091999998092651 Test Acc : 0.4723750012740493\n",
      "\n",
      "Current : 108 Train Acc : 0.6105999995470047 Test Acc : 0.47262500148266556\n",
      "\n",
      "Current : 109 Train Acc : 0.6118000004291534 Test Acc : 0.47362500075250863\n",
      "\n",
      "Current : 110 Train Acc : 0.6149999995231629 Test Acc : 0.4737500013038516\n",
      "\n",
      "Current : 111 Train Acc : 0.6165999999046325 Test Acc : 0.4737500014528632\n",
      "\n",
      "Current : 112 Train Acc : 0.6153999993801117 Test Acc : 0.4738750013336539\n",
      "\n",
      "Current : 113 Train Acc : 0.616199999332428 Test Acc : 0.4737500016018748\n",
      "\n",
      "Current : 114 Train Acc : 0.6199999986886978 Test Acc : 0.4741250010207295\n",
      "\n",
      "Current : 115 Train Acc : 0.6185999981164932 Test Acc : 0.47462500128895047\n",
      "\n",
      "Current : 116 Train Acc : 0.6205999987125397 Test Acc : 0.4742500013113022\n",
      "\n",
      "Current : 117 Train Acc : 0.6243999996185303 Test Acc : 0.4735000010952353\n",
      "\n",
      "Current : 118 Train Acc : 0.625 Test Acc : 0.4735000010952353\n",
      "\n",
      "Current : 119 Train Acc : 0.6267999993562698 Test Acc : 0.4730000011622906\n",
      "\n",
      "Current : 120 Train Acc : 0.6279999989271164 Test Acc : 0.4732500009983778\n",
      "\n",
      "Current : 121 Train Acc : 0.6287999991178512 Test Acc : 0.473375001065433\n",
      "\n",
      "Current : 122 Train Acc : 0.6310000002384186 Test Acc : 0.47287500090897083\n",
      "\n",
      "Current : 123 Train Acc : 0.6308000009059906 Test Acc : 0.4716250010207295\n",
      "\n",
      "Current : 124 Train Acc : 0.6316000012159347 Test Acc : 0.47125000137835743\n",
      "\n",
      "Current : 125 Train Acc : 0.6336000008583069 Test Acc : 0.47125000149011614\n",
      "\n",
      "Current : 126 Train Acc : 0.6346000010967254 Test Acc : 0.46950000133365394\n",
      "\n",
      "Current : 127 Train Acc : 0.6358000007867813 Test Acc : 0.4701250020414591\n",
      "\n",
      "Current : 128 Train Acc : 0.6362000004053115 Test Acc : 0.4696250019222498\n",
      "\n",
      "Current : 129 Train Acc : 0.6364000014066696 Test Acc : 0.46925000127404926\n",
      "\n",
      "Current : 130 Train Acc : 0.6388000010251998 Test Acc : 0.4702500012516975\n",
      "\n",
      "Current : 131 Train Acc : 0.6390000008344651 Test Acc : 0.46912500116974115\n",
      "\n",
      "Current : 132 Train Acc : 0.6422000007629395 Test Acc : 0.4686250014975667\n",
      "\n",
      "Current : 133 Train Acc : 0.6454000000953675 Test Acc : 0.46975000195205213\n",
      "\n",
      "Current : 134 Train Acc : 0.6470000007152558 Test Acc : 0.4711250015348196\n",
      "\n",
      "Current : 135 Train Acc : 0.6478000004291534 Test Acc : 0.469875001385808\n",
      "\n",
      "Current : 136 Train Acc : 0.6502000005245209 Test Acc : 0.46850000105798245\n",
      "\n",
      "Current : 137 Train Acc : 0.6508000004291534 Test Acc : 0.46912500105798244\n",
      "\n",
      "Current : 138 Train Acc : 0.6532000000476837 Test Acc : 0.4686250014975667\n",
      "\n",
      "Current : 139 Train Acc : 0.6561999998092651 Test Acc : 0.46937500149011613\n",
      "\n",
      "Current : 140 Train Acc : 0.6550000004768372 Test Acc : 0.46812500085681674\n",
      "\n",
      "Current : 141 Train Acc : 0.6594000006914139 Test Acc : 0.46837500125169756\n",
      "\n",
      "Current : 142 Train Acc : 0.6599999998807907 Test Acc : 0.46775000154972074\n",
      "\n",
      "Current : 143 Train Acc : 0.6610000002384185 Test Acc : 0.4680000015720725\n",
      "\n",
      "Current : 144 Train Acc : 0.6628000001907348 Test Acc : 0.4670000008493662\n",
      "\n",
      "Current : 145 Train Acc : 0.6635999997854233 Test Acc : 0.4681250012293458\n",
      "\n",
      "Current : 146 Train Acc : 0.6669999995231628 Test Acc : 0.46937500085681677\n",
      "\n",
      "Current : 147 Train Acc : 0.6677999992370606 Test Acc : 0.46837500084191563\n",
      "\n",
      "Current : 148 Train Acc : 0.6693999994993209 Test Acc : 0.46725000079721213\n",
      "\n",
      "Current : 149 Train Acc : 0.6705999995470047 Test Acc : 0.46775000128895045\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# G\n",
    "current_exp_name = 'G';\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "# create layers\n",
    "l1 = CNN(3,3, 16,which_reg=current_exp_name); \n",
    "l2 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l3 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "\n",
    "l4 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l5 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l6 = CNN(3,16,10,which_reg=current_exp_name); \n",
    "\n",
    "# 2. graph \n",
    "x = tf.placeholder(tf.float32,(batch_size,96,96,3))\n",
    "y = tf.placeholder(tf.float32,(batch_size,10))\n",
    "\n",
    "layer1, layer1a = l1. feedforward(x,stride=2)\n",
    "layer2, layer2a = l2. feedforward(layer1a,stride=2)\n",
    "layer3, layer3a = l3. feedforward(layer2a,stride=2)\n",
    "layer4, layer4a = l4. feedforward(layer3a,stride=2)\n",
    "layer5, layer5a = l5. feedforward(layer4a)\n",
    "layer6, layer6a = l6. feedforward(layer5a)\n",
    "\n",
    "final_layer   = tf.reduce_mean(layer6a,(1,2))\n",
    "final_softmax = tf_softmax(final_layer)\n",
    "cost          = -tf.reduce_mean(y * tf.log(final_softmax + 1e-8))\n",
    "correct_prediction = tf.equal(tf.argmax(final_softmax, 1), tf.argmax(y, 1))\n",
    "accuracy           = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "gradient = tf.tile((final_softmax-y)[:,None,None,:],[1,6,6,1])/batch_size\n",
    "grad6p,grad6w,grad6_up = l6.backprop(gradient)\n",
    "grad5p,grad5w,grad5_up = l5.backprop(grad6p)\n",
    "grad4p,grad4w,grad4_up = l4.backprop(grad5p,stride=2)\n",
    "grad3p,grad3w,grad3_up = l3.backprop(grad4p,stride=2)\n",
    "grad2p,grad2w,grad2_up = l2.backprop(grad3p,stride=2)\n",
    "grad1p,grad1w,grad1_up = l1.backprop(grad2p,stride=2)\n",
    "\n",
    "gradient_update = grad6_up + grad5_up + grad4_up + grad3_up + grad2_up + grad1_up \n",
    "\n",
    "# train\n",
    "sess.run(tf.global_variables_initializer())\n",
    "avg_acc_train = 0; avg_acc_test  = 0; train_acc = [];test_acc = []\n",
    "\n",
    "# mean std skew kurt non-zero\n",
    "llayer1 = [[],[],[],[],[]]; llayer2 = [[],[],[],[],[]]; llayer3 = [[],[],[],[],[]]\n",
    "llayer4 = [[],[],[],[],[]]; llayer5 = [[],[],[],[],[]]; llayer6 = [[],[],[],[],[]]\n",
    "\n",
    "llayer1a = [[],[],[],[],[]]; llayer2a = [[],[],[],[],[]]; llayer3a = [[],[],[],[],[]]\n",
    "llayer4a = [[],[],[],[],[]]; llayer5a = [[],[],[],[],[]]; llayer6a = [[],[],[],[],[]]\n",
    "\n",
    "weight1 = [[],[],[],[],[]]; weight2 = [[],[],[],[],[]]; weight3 = [[],[],[],[],[]];\n",
    "weight4 = [[],[],[],[],[]]; weight5 = [[],[],[],[],[]]; weight6 = [[],[],[],[],[]];\n",
    "\n",
    "gradw1  = [[],[],[],[],[]]; gradw2  = [[],[],[],[],[]]; gradw3  = [[],[],[],[],[]];\n",
    "gradw4  = [[],[],[],[],[]]; gradw5  = [[],[],[],[],[]]; gradw6  = [[],[],[],[],[]];\n",
    "\n",
    "gradp1  = [[],[],[],[],[]]; gradp2  = [[],[],[],[],[]]; gradp3  = [[],[],[],[],[]];\n",
    "gradp4  = [[],[],[],[],[]]; gradp5  = [[],[],[],[],[]]; gradp6  = [[],[],[],[],[]];\n",
    "\n",
    "gradup1  = [[],[],[],[],[]]; gradup2  = [[],[],[],[],[]]; gradup3  = [[],[],[],[],[]];\n",
    "gradup4  = [[],[],[],[],[]]; gradup5  = [[],[],[],[],[]]; gradup6  = [[],[],[],[],[]];\n",
    "\n",
    "list_of_outputs = [\n",
    "    layer1,layer2,layer3,layer4,layer5,layer6,\n",
    "    layer1a,layer2a,layer3a,layer4a,layer5a,layer6a,\n",
    "    l1.getw(),l2.getw(),l3.getw(),l4.getw(),l5.getw(),l6.getw(),\n",
    "    grad1w,grad2w,grad3w,grad4w,grad5w,grad6w,\n",
    "    grad1p,grad2p,grad3p,grad4p,grad5p,grad6p,\n",
    "    grad1_up[0],grad2_up[0],grad3_up[0],grad4_up[0],grad5_up[0],grad6_up[0]\n",
    "]\n",
    "\n",
    "for iter in range(num_epoch):\n",
    "\n",
    "    # Training Accuracy    \n",
    "    for current_batch_index in range(0,len(train_images),batch_size):\n",
    "        current_data  = train_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = train_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy,gradient_update],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(train_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_train = avg_acc_train + sess_results[0]\n",
    "        \n",
    "    # get the results\n",
    "    mid_stat = sess.run(list_of_outputs,feed_dict={x:current_data,y:current_label})\n",
    "    \n",
    "    # Test Accuracy    \n",
    "    for current_batch_index in range(0,len(test_images), batch_size):\n",
    "        current_data  = test_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = test_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(test_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_test = avg_acc_test + sess_results[0]   \n",
    "        \n",
    "    # ======================== extract stats ========================\n",
    "    llayer1 = append_stat(llayer1,mid_stat,0);  llayer2 = append_stat(llayer2,mid_stat,1);  llayer3 = append_stat(llayer3,mid_stat,2);\n",
    "    llayer4 = append_stat(llayer4,mid_stat,3);  llayer5 = append_stat(llayer5,mid_stat,4);  llayer6 = append_stat(llayer6,mid_stat,5);\n",
    "\n",
    "    llayer1a = append_stat(llayer1a,mid_stat,6);  llayer2a = append_stat(llayer2a,mid_stat,7);  llayer3a = append_stat(llayer3a,mid_stat,8);\n",
    "    llayer4a = append_stat(llayer4a,mid_stat,9);  llayer5a = append_stat(llayer5a,mid_stat,10); llayer6a = append_stat(llayer6a,mid_stat,11);\n",
    "    \n",
    "    weight1 = append_stat(weight1,mid_stat,12);  weight2 = append_stat(weight2,mid_stat,13);  weight3 = append_stat(weight3,mid_stat,14);\n",
    "    weight4 = append_stat(weight4,mid_stat,15);  weight5 = append_stat(weight5,mid_stat,16);  weight6 = append_stat(weight6,mid_stat,17);\n",
    "    \n",
    "    gradw1 = append_stat(gradw1,mid_stat,18); gradw2 = append_stat(gradw2,mid_stat,19); gradw3 = append_stat(gradw3,mid_stat,20);\n",
    "    gradw4 = append_stat(gradw4,mid_stat,21); gradw5 = append_stat(gradw5,mid_stat,22); gradw6 = append_stat(gradw6,mid_stat,23);\n",
    "    \n",
    "    gradp1 = append_stat(gradp1,mid_stat,24); gradp2 = append_stat(gradp2,mid_stat,25); gradp3 = append_stat(gradp3,mid_stat,26);\n",
    "    gradp4 = append_stat(gradp4,mid_stat,27); gradp5 = append_stat(gradp5,mid_stat,28); gradp6 = append_stat(gradp6,mid_stat,29);\n",
    "\n",
    "    gradup1 = append_stat(gradup1,mid_stat,30); gradup2 = append_stat(gradup2,mid_stat,31); gradup3 = append_stat(gradup3,mid_stat,32);\n",
    "    gradup4 = append_stat(gradup4,mid_stat,33); gradup5 = append_stat(gradup5,mid_stat,34); gradup6 = append_stat(gradup6,mid_stat,35);\n",
    "\n",
    "    train_acc.append(avg_acc_train/(len(train_images)/batch_size))\n",
    "    test_acc .append(avg_acc_test / (len(test_images)/batch_size))\n",
    "    # ======================== extract stats ========================\n",
    "    \n",
    "    # ======================== save to image ========================\n",
    "    save_to_image(mid_stat[0:6]   ,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,\"layer\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[6:12]  ,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a,\"layera\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[12:18] ,weight1,weight2,weight3,weight4,weight5,weight6,\"weights\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[18:24] ,gradw1,gradw2,gradw3,gradw4,gradw5,gradw6,\"gradientw\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[24:30] ,gradp1,gradp2,gradp3,gradp4,gradp5,gradp6,\"gradientp\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[30:36] ,gradup1,gradup2,gradup3,gradup4,gradup5,gradup6,\"moment\",train_acc,test_acc,current_exp_name,iter)\n",
    "    # ======================== save to image ========================\n",
    "        \n",
    "    # ======================== print reset ========================\n",
    "    print(\"Current : \"+ str(iter) + \" Train Acc : \" + str(avg_acc_train/(len(train_images)/batch_size)) + \" Test Acc : \" + str(avg_acc_test/(len(test_images)/batch_size)) + '\\n')\n",
    "    avg_acc_train = 0 ; avg_acc_test  = 0\n",
    "    # ======================== print reset ========================\n",
    "\n",
    "np.save(current_exp_name+'/train_acc.npy',train_acc); np.save(current_exp_name+'/test_acc.npy', test_acc)    \n",
    "np.save(current_exp_name+'/llayer1.npy', llayer1);  np.save(current_exp_name+'/llayer2.npy', llayer2);  np.save(current_exp_name+'/llayer3.npy', llayer3); \n",
    "np.save(current_exp_name+'/llayer4.npy', llayer4);  np.save(current_exp_name+'/llayer5.npy', llayer5);  np.save(current_exp_name+'/llayer6.npy', llayer6); \n",
    "\n",
    "np.save(current_exp_name+'/llayer1a.npy', llayer1a);  np.save(current_exp_name+'/llayer2a.npy', llayer2a);  np.save(current_exp_name+'/llayer3a.npy', llayer3a); \n",
    "np.save(current_exp_name+'/llayer4a.npy', llayer4a);  np.save(current_exp_name+'/llayer5a.npy', llayer5a);  np.save(current_exp_name+'/llayer6a.npy', llayer6a); \n",
    "\n",
    "np.save(current_exp_name+'/weight1.npy', weight1);  np.save(current_exp_name+'/weight2.npy', weight2);  np.save(current_exp_name+'/weight3.npy', weight3);  \n",
    "np.save(current_exp_name+'/weight4.npy', weight4);  np.save(current_exp_name+'/weight5.npy', weight5);  np.save(current_exp_name+'/weight6.npy', weight6);  \n",
    "\n",
    "np.save(current_exp_name+'/gradw1.npy', gradw1); np.save(current_exp_name+'/gradw2.npy', gradw2); np.save(current_exp_name+'/gradw3.npy', gradw3);\n",
    "np.save(current_exp_name+'/gradw4.npy', gradw4); np.save(current_exp_name+'/gradw5.npy', gradw5); np.save(current_exp_name+'/gradw6.npy', gradw6);\n",
    "\n",
    "np.save(current_exp_name+'/gradp1.npy', gradp1); np.save(current_exp_name+'/gradp2.npy', gradp2); np.save(current_exp_name+'/gradp3.npy', gradp3);\n",
    "np.save(current_exp_name+'/gradp4.npy', gradp4); np.save(current_exp_name+'/gradp5.npy', gradp5); np.save(current_exp_name+'/gradp6.npy', gradp6);\n",
    "\n",
    "np.save(current_exp_name+'/gradup1.npy', gradup1); np.save(current_exp_name+'/gradup2.npy', gradup2); np.save(current_exp_name+'/gradup3.npy', gradup3);\n",
    "np.save(current_exp_name+'/gradup4.npy', gradup4); np.save(current_exp_name+'/gradup5.npy', gradup5); np.save(current_exp_name+'/gradup6.npy', gradup6);\n",
    "\n",
    "sess.close(); tf.reset_default_graph();\n",
    "\n",
    "%reset_selective -f train_acc,test_acc,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a\n",
    "%reset_selective -f weight1,weight2,weight3,weight4,weight5,weight6\n",
    "%reset_selective -f gradw1,gradw2,gradw3,gradw4,gradw5,gradw6\n",
    "%reset_selective -f gradp1,gradp2,gradp3,gradp4,gradp5,gradp6\n",
    "%reset_selective -f gradup1,gradup2,gradup3,gradup4,gradup5,gradup6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-05T08:53:17.654001Z",
     "start_time": "2019-01-05T08:03:07.417716Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current : 0 Train Acc : 0.11660000205039978 Test Acc : 0.15762500289827586\n",
      "\n",
      "Current : 1 Train Acc : 0.162000003144145 Test Acc : 0.19175000336952508\n",
      "\n",
      "Current : 2 Train Acc : 0.2112000034302473 Test Acc : 0.21912500316277148\n",
      "\n",
      "Current : 3 Train Acc : 0.2452000031620264 Test Acc : 0.26387500318698587\n",
      "\n",
      "Current : 4 Train Acc : 0.2856000025719404 Test Acc : 0.29600000239908697\n",
      "\n",
      "Current : 5 Train Acc : 0.3072000023275614 Test Acc : 0.3161250020284206\n",
      "\n",
      "Current : 6 Train Acc : 0.31620000217854977 Test Acc : 0.3295000014267862\n",
      "\n",
      "Current : 7 Train Acc : 0.3226000022441149 Test Acc : 0.3398750021122396\n",
      "\n",
      "Current : 8 Train Acc : 0.33020000158250334 Test Acc : 0.34825000225566327\n",
      "\n",
      "Current : 9 Train Acc : 0.3402000017911196 Test Acc : 0.35137500138953326\n",
      "\n",
      "Current : 10 Train Acc : 0.34520000125467776 Test Acc : 0.3661250007338822\n",
      "\n",
      "Current : 11 Train Acc : 0.35660000146925447 Test Acc : 0.37237500129267576\n",
      "\n",
      "Current : 12 Train Acc : 0.36620000141859055 Test Acc : 0.3760000010579824\n",
      "\n",
      "Current : 13 Train Acc : 0.3744000009000301 Test Acc : 0.38062500055879356\n",
      "\n",
      "Current : 14 Train Acc : 0.38280000138282777 Test Acc : 0.3803750010021031\n",
      "\n",
      "Current : 15 Train Acc : 0.3878000011742115 Test Acc : 0.3842500011995435\n",
      "\n",
      "Current : 16 Train Acc : 0.39140000110864637 Test Acc : 0.38600000116974115\n",
      "\n",
      "Current : 17 Train Acc : 0.3948000023365021 Test Acc : 0.3906250014528632\n",
      "\n",
      "Current : 18 Train Acc : 0.39780000180006025 Test Acc : 0.3927500019595027\n",
      "\n",
      "Current : 19 Train Acc : 0.4002000015974045 Test Acc : 0.39525000121444465\n",
      "\n",
      "Current : 20 Train Acc : 0.4040000017285347 Test Acc : 0.39750000186264517\n",
      "\n",
      "Current : 21 Train Acc : 0.4096000009775162 Test Acc : 0.40150000154972076\n",
      "\n",
      "Current : 22 Train Acc : 0.4108000014424324 Test Acc : 0.40087500125169756\n",
      "\n",
      "Current : 23 Train Acc : 0.41320000144839286 Test Acc : 0.4051250009983778\n",
      "\n",
      "Current : 24 Train Acc : 0.4166000013947487 Test Acc : 0.4081250014901161\n",
      "\n",
      "Current : 25 Train Acc : 0.42040000146627426 Test Acc : 0.412375001385808\n",
      "\n",
      "Current : 26 Train Acc : 0.42180000185966493 Test Acc : 0.4135000017285347\n",
      "\n",
      "Current : 27 Train Acc : 0.42360000163316724 Test Acc : 0.4161250017955899\n",
      "\n",
      "Current : 28 Train Acc : 0.42620000159740445 Test Acc : 0.4157500010356307\n",
      "\n",
      "Current : 29 Train Acc : 0.42820000159740446 Test Acc : 0.41850000064820053\n",
      "\n",
      "Current : 30 Train Acc : 0.42840000170469283 Test Acc : 0.41912500094622374\n",
      "\n",
      "Current : 31 Train Acc : 0.43300000154972074 Test Acc : 0.4222500008530915\n",
      "\n",
      "Current : 32 Train Acc : 0.43540000200271606 Test Acc : 0.4230000007711351\n",
      "\n",
      "Current : 33 Train Acc : 0.4372000018954277 Test Acc : 0.4245000006072223\n",
      "\n",
      "Current : 34 Train Acc : 0.43820000141859056 Test Acc : 0.4290000004135072\n",
      "\n",
      "Current : 35 Train Acc : 0.43920000225305555 Test Acc : 0.4313750004954636\n",
      "\n",
      "Current : 36 Train Acc : 0.4420000025629997 Test Acc : 0.43262500027194617\n",
      "\n",
      "Current : 37 Train Acc : 0.44420000237226487 Test Acc : 0.43487500058487055\n",
      "\n",
      "Current : 38 Train Acc : 0.44840000253915785 Test Acc : 0.43837500067427754\n",
      "\n",
      "Current : 39 Train Acc : 0.4504000021219254 Test Acc : 0.44112500013783573\n",
      "\n",
      "Current : 40 Train Acc : 0.4522000026702881 Test Acc : 0.442249999884516\n",
      "\n",
      "Current : 41 Train Acc : 0.45480000245571134 Test Acc : 0.44262499975040553\n",
      "\n",
      "Current : 42 Train Acc : 0.45660000228881836 Test Acc : 0.44424999989569186\n",
      "\n",
      "Current : 43 Train Acc : 0.4602000021934509 Test Acc : 0.44337500013411046\n",
      "\n",
      "Current : 44 Train Acc : 0.46120000183582305 Test Acc : 0.4431250002235174\n",
      "\n",
      "Current : 45 Train Acc : 0.46340000194311143 Test Acc : 0.4432500003278255\n",
      "\n",
      "Current : 46 Train Acc : 0.46540000146627425 Test Acc : 0.4446250008791685\n",
      "\n",
      "Current : 47 Train Acc : 0.46620000153779984 Test Acc : 0.4462500007636845\n",
      "\n",
      "Current : 48 Train Acc : 0.46680000096559526 Test Acc : 0.44600000070407986\n",
      "\n",
      "Current : 49 Train Acc : 0.46920000118017197 Test Acc : 0.44862500043585896\n",
      "\n",
      "Current : 50 Train Acc : 0.4710000014901161 Test Acc : 0.45075000112876296\n",
      "\n",
      "Current : 51 Train Acc : 0.47200000125169755 Test Acc : 0.4531250008381903\n",
      "\n",
      "Current : 52 Train Acc : 0.4736000005602837 Test Acc : 0.45312500135973094\n",
      "\n",
      "Current : 53 Train Acc : 0.47340000069141386 Test Acc : 0.45362500129267574\n",
      "\n",
      "Current : 54 Train Acc : 0.4778000007867813 Test Acc : 0.45462500108405945\n",
      "\n",
      "Current : 55 Train Acc : 0.47920000088214876 Test Acc : 0.4550000013969839\n",
      "\n",
      "Current : 56 Train Acc : 0.47900000095367434 Test Acc : 0.4552500015310943\n",
      "\n",
      "Current : 57 Train Acc : 0.48120000088214876 Test Acc : 0.45500000124797224\n",
      "\n",
      "Current : 58 Train Acc : 0.4820000007152557 Test Acc : 0.4551250007562339\n",
      "\n",
      "Current : 59 Train Acc : 0.48340000051259996 Test Acc : 0.45687500117346647\n",
      "\n",
      "Current : 60 Train Acc : 0.4858000004887581 Test Acc : 0.4583750014938414\n",
      "\n",
      "Current : 61 Train Acc : 0.4846000004410744 Test Acc : 0.4590000009350479\n",
      "\n",
      "Current : 62 Train Acc : 0.4866000004410744 Test Acc : 0.45862500118091704\n",
      "\n",
      "Current : 63 Train Acc : 0.4874000009894371 Test Acc : 0.45962500086054203\n",
      "\n",
      "Current : 64 Train Acc : 0.48980000108480454 Test Acc : 0.46062500121071936\n",
      "\n",
      "Current : 65 Train Acc : 0.4894000011086464 Test Acc : 0.4632500007562339\n",
      "\n",
      "Current : 66 Train Acc : 0.4912000008225441 Test Acc : 0.4641250011138618\n",
      "\n",
      "Current : 67 Train Acc : 0.4916000008583069 Test Acc : 0.4635000012628734\n",
      "\n",
      "Current : 68 Train Acc : 0.4920000012516975 Test Acc : 0.4635000007040799\n",
      "\n",
      "Current : 69 Train Acc : 0.4932000016570091 Test Acc : 0.46500000115483997\n",
      "\n",
      "Current : 70 Train Acc : 0.49540000092983244 Test Acc : 0.4662500013783574\n",
      "\n",
      "Current : 71 Train Acc : 0.49620000171661377 Test Acc : 0.46650000136345626\n",
      "\n",
      "Current : 72 Train Acc : 0.49820000195503233 Test Acc : 0.467875001616776\n",
      "\n",
      "Current : 73 Train Acc : 0.5012000011205673 Test Acc : 0.46837500188499687\n",
      "\n",
      "Current : 74 Train Acc : 0.5002000012397766 Test Acc : 0.46950000151991844\n",
      "\n",
      "Current : 75 Train Acc : 0.5018000016212464 Test Acc : 0.47000000182539226\n",
      "\n",
      "Current : 76 Train Acc : 0.5020000016689301 Test Acc : 0.4705000011995435\n",
      "\n",
      "Current : 77 Train Acc : 0.5044000015258789 Test Acc : 0.4692500014230609\n",
      "\n",
      "Current : 78 Train Acc : 0.5052000011205673 Test Acc : 0.47037500090897083\n",
      "\n",
      "Current : 79 Train Acc : 0.5068000017404556 Test Acc : 0.4720000011101365\n",
      "\n",
      "Current : 80 Train Acc : 0.5072000019550323 Test Acc : 0.4752500011399388\n",
      "\n",
      "Current : 81 Train Acc : 0.5072000020742417 Test Acc : 0.4745000010728836\n",
      "\n",
      "Current : 82 Train Acc : 0.5086000024080276 Test Acc : 0.47512500077486036\n",
      "\n",
      "Current : 83 Train Acc : 0.5100000021457672 Test Acc : 0.4745000011101365\n",
      "\n",
      "Current : 84 Train Acc : 0.5090000023841857 Test Acc : 0.4767500010505319\n",
      "\n",
      "Current : 85 Train Acc : 0.5104000021219254 Test Acc : 0.4772500005736947\n",
      "\n",
      "Current : 86 Train Acc : 0.5108000016212464 Test Acc : 0.47912500105798245\n",
      "\n",
      "Current : 87 Train Acc : 0.5114000014662743 Test Acc : 0.48112500112503764\n",
      "\n",
      "Current : 88 Train Acc : 0.5150000019073486 Test Acc : 0.47925000108778476\n",
      "\n",
      "Current : 89 Train Acc : 0.5148000013828278 Test Acc : 0.4806250013783574\n",
      "\n",
      "Current : 90 Train Acc : 0.5160000011920929 Test Acc : 0.48175000105053184\n",
      "\n",
      "Current : 91 Train Acc : 0.5168000016808509 Test Acc : 0.48250000111758706\n",
      "\n",
      "Current : 92 Train Acc : 0.5216000024080276 Test Acc : 0.48075000155717135\n",
      "\n",
      "Current : 93 Train Acc : 0.5200000010728836 Test Acc : 0.48475000128149986\n",
      "\n",
      "Current : 94 Train Acc : 0.5204000011086464 Test Acc : 0.4855000012740493\n",
      "\n",
      "Current : 95 Train Acc : 0.5210000013709069 Test Acc : 0.48637500129640104\n",
      "\n",
      "Current : 96 Train Acc : 0.5196000013947487 Test Acc : 0.486250001527369\n",
      "\n",
      "Current : 97 Train Acc : 0.5218000018000603 Test Acc : 0.48750000160187484\n",
      "\n",
      "Current : 98 Train Acc : 0.5234000015854835 Test Acc : 0.48775000136345625\n",
      "\n",
      "Current : 99 Train Acc : 0.5264000018239021 Test Acc : 0.48737500097602604\n",
      "\n",
      "Current : 100 Train Acc : 0.528000001847744 Test Acc : 0.4887500012665987\n",
      "\n",
      "Current : 101 Train Acc : 0.5274000023007392 Test Acc : 0.49000000149011613\n",
      "\n",
      "Current : 102 Train Acc : 0.5266000019907952 Test Acc : 0.4910000006109476\n",
      "\n",
      "Current : 103 Train Acc : 0.5292000021338463 Test Acc : 0.4923750005662441\n",
      "\n",
      "Current : 104 Train Acc : 0.5314000019431114 Test Acc : 0.49312500074505805\n",
      "\n",
      "Current : 105 Train Acc : 0.5340000010132789 Test Acc : 0.4946250007301569\n",
      "\n",
      "Current : 106 Train Acc : 0.5350000010132789 Test Acc : 0.4970000007748604\n",
      "\n",
      "Current : 107 Train Acc : 0.5356000004410744 Test Acc : 0.49512500055134295\n",
      "\n",
      "Current : 108 Train Acc : 0.5376000007987023 Test Acc : 0.4956250012665987\n",
      "\n",
      "Current : 109 Train Acc : 0.5394000008106231 Test Acc : 0.4980000010505319\n",
      "\n",
      "Current : 110 Train Acc : 0.5400000009536743 Test Acc : 0.49850000072270634\n",
      "\n",
      "Current : 111 Train Acc : 0.5432000004649162 Test Acc : 0.49912500036880375\n",
      "\n",
      "Current : 112 Train Acc : 0.5438000002503395 Test Acc : 0.49937500098720194\n",
      "\n",
      "Current : 113 Train Acc : 0.5454000002741813 Test Acc : 0.5015000008791685\n",
      "\n",
      "Current : 114 Train Acc : 0.5476000003218651 Test Acc : 0.5007500011846423\n",
      "\n",
      "Current : 115 Train Acc : 0.5473999998569489 Test Acc : 0.5006250010058284\n",
      "\n",
      "Current : 116 Train Acc : 0.547800000011921 Test Acc : 0.5012500009685755\n",
      "\n",
      "Current : 117 Train Acc : 0.5493999999761582 Test Acc : 0.5011250009760261\n",
      "\n",
      "Current : 118 Train Acc : 0.5508000009059906 Test Acc : 0.5017500009015202\n",
      "\n",
      "Current : 119 Train Acc : 0.5514000002145767 Test Acc : 0.5015000007860363\n",
      "\n",
      "Current : 120 Train Acc : 0.553200001358986 Test Acc : 0.5028750011511147\n",
      "\n",
      "Current : 121 Train Acc : 0.5548000006675721 Test Acc : 0.5040000018104911\n",
      "\n",
      "Current : 122 Train Acc : 0.556400000333786 Test Acc : 0.5036250016279519\n",
      "\n",
      "Current : 123 Train Acc : 0.5578000001907348 Test Acc : 0.5077500014007091\n",
      "\n",
      "Current : 124 Train Acc : 0.559200000166893 Test Acc : 0.507500001359731\n",
      "\n",
      "Current : 125 Train Acc : 0.5599999998807907 Test Acc : 0.5081250014156103\n",
      "\n",
      "Current : 126 Train Acc : 0.5624000005722046 Test Acc : 0.5086250013113022\n",
      "\n",
      "Current : 127 Train Acc : 0.5648000007867813 Test Acc : 0.5081250016391278\n",
      "\n",
      "Current : 128 Train Acc : 0.5646000015735626 Test Acc : 0.5086250010877847\n",
      "\n",
      "Current : 129 Train Acc : 0.5682000014781952 Test Acc : 0.5095000015199185\n",
      "\n",
      "Current : 130 Train Acc : 0.568600001335144 Test Acc : 0.5100000014901162\n",
      "\n",
      "Current : 131 Train Acc : 0.5692000017166138 Test Acc : 0.5107500014454126\n",
      "\n",
      "Current : 132 Train Acc : 0.5706000014543533 Test Acc : 0.5110000012814999\n",
      "\n",
      "Current : 133 Train Acc : 0.5700000016689301 Test Acc : 0.5123750014603138\n",
      "\n",
      "Current : 134 Train Acc : 0.5714000014066696 Test Acc : 0.5108750015497208\n",
      "\n",
      "Current : 135 Train Acc : 0.5734000011682511 Test Acc : 0.5121250016987324\n",
      "\n",
      "Current : 136 Train Acc : 0.5768000009059906 Test Acc : 0.5126250015944243\n",
      "\n",
      "Current : 137 Train Acc : 0.5782000004053116 Test Acc : 0.5130000013113022\n",
      "\n",
      "Current : 138 Train Acc : 0.5782000006437301 Test Acc : 0.5130000013113022\n",
      "\n",
      "Current : 139 Train Acc : 0.5796000001430511 Test Acc : 0.5127500014007091\n",
      "\n",
      "Current : 140 Train Acc : 0.5802000007629394 Test Acc : 0.5153750013560057\n",
      "\n",
      "Current : 141 Train Acc : 0.5812000006437301 Test Acc : 0.5146250013262034\n",
      "\n",
      "Current : 142 Train Acc : 0.5820000011920929 Test Acc : 0.5150000008195639\n",
      "\n",
      "Current : 143 Train Acc : 0.583800000667572 Test Acc : 0.514125000834465\n",
      "\n",
      "Current : 144 Train Acc : 0.5842000008821487 Test Acc : 0.5151250004768372\n",
      "\n",
      "Current : 145 Train Acc : 0.5850000009536743 Test Acc : 0.5152500012516975\n",
      "\n",
      "Current : 146 Train Acc : 0.5856000009775162 Test Acc : 0.5161250009387731\n",
      "\n",
      "Current : 147 Train Acc : 0.5868000013828277 Test Acc : 0.5168750010617077\n",
      "\n",
      "Current : 148 Train Acc : 0.587000001192093 Test Acc : 0.5183750009723007\n",
      "\n",
      "Current : 149 Train Acc : 0.5884000008106232 Test Acc : 0.5178750013746322\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# H\n",
    "current_exp_name = 'H';\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "# create layers\n",
    "l1 = CNN(3,3, 16,which_reg=current_exp_name); \n",
    "l2 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l3 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "\n",
    "l4 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l5 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l6 = CNN(3,16,10,which_reg=current_exp_name); \n",
    "\n",
    "# 2. graph \n",
    "x = tf.placeholder(tf.float32,(batch_size,96,96,3))\n",
    "y = tf.placeholder(tf.float32,(batch_size,10))\n",
    "\n",
    "layer1, layer1a = l1. feedforward(x,stride=2)\n",
    "layer2, layer2a = l2. feedforward(layer1a,stride=2)\n",
    "layer3, layer3a = l3. feedforward(layer2a,stride=2)\n",
    "layer4, layer4a = l4. feedforward(layer3a,stride=2)\n",
    "layer5, layer5a = l5. feedforward(layer4a)\n",
    "layer6, layer6a = l6. feedforward(layer5a)\n",
    "\n",
    "final_layer   = tf.reduce_mean(layer6a,(1,2))\n",
    "final_softmax = tf_softmax(final_layer)\n",
    "cost          = -tf.reduce_mean(y * tf.log(final_softmax + 1e-8))\n",
    "correct_prediction = tf.equal(tf.argmax(final_softmax, 1), tf.argmax(y, 1))\n",
    "accuracy           = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "gradient = tf.tile((final_softmax-y)[:,None,None,:],[1,6,6,1])/batch_size\n",
    "grad6p,grad6w,grad6_up = l6.backprop(gradient)\n",
    "grad5p,grad5w,grad5_up = l5.backprop(grad6p)\n",
    "grad4p,grad4w,grad4_up = l4.backprop(grad5p,stride=2)\n",
    "grad3p,grad3w,grad3_up = l3.backprop(grad4p,stride=2)\n",
    "grad2p,grad2w,grad2_up = l2.backprop(grad3p,stride=2)\n",
    "grad1p,grad1w,grad1_up = l1.backprop(grad2p,stride=2)\n",
    "\n",
    "gradient_update = grad6_up + grad5_up + grad4_up + grad3_up + grad2_up + grad1_up \n",
    "\n",
    "# train\n",
    "sess.run(tf.global_variables_initializer())\n",
    "avg_acc_train = 0; avg_acc_test  = 0; train_acc = [];test_acc = []\n",
    "\n",
    "# mean std skew kurt non-zero\n",
    "llayer1 = [[],[],[],[],[]]; llayer2 = [[],[],[],[],[]]; llayer3 = [[],[],[],[],[]]\n",
    "llayer4 = [[],[],[],[],[]]; llayer5 = [[],[],[],[],[]]; llayer6 = [[],[],[],[],[]]\n",
    "\n",
    "llayer1a = [[],[],[],[],[]]; llayer2a = [[],[],[],[],[]]; llayer3a = [[],[],[],[],[]]\n",
    "llayer4a = [[],[],[],[],[]]; llayer5a = [[],[],[],[],[]]; llayer6a = [[],[],[],[],[]]\n",
    "\n",
    "weight1 = [[],[],[],[],[]]; weight2 = [[],[],[],[],[]]; weight3 = [[],[],[],[],[]];\n",
    "weight4 = [[],[],[],[],[]]; weight5 = [[],[],[],[],[]]; weight6 = [[],[],[],[],[]];\n",
    "\n",
    "gradw1  = [[],[],[],[],[]]; gradw2  = [[],[],[],[],[]]; gradw3  = [[],[],[],[],[]];\n",
    "gradw4  = [[],[],[],[],[]]; gradw5  = [[],[],[],[],[]]; gradw6  = [[],[],[],[],[]];\n",
    "\n",
    "gradp1  = [[],[],[],[],[]]; gradp2  = [[],[],[],[],[]]; gradp3  = [[],[],[],[],[]];\n",
    "gradp4  = [[],[],[],[],[]]; gradp5  = [[],[],[],[],[]]; gradp6  = [[],[],[],[],[]];\n",
    "\n",
    "gradup1  = [[],[],[],[],[]]; gradup2  = [[],[],[],[],[]]; gradup3  = [[],[],[],[],[]];\n",
    "gradup4  = [[],[],[],[],[]]; gradup5  = [[],[],[],[],[]]; gradup6  = [[],[],[],[],[]];\n",
    "\n",
    "list_of_outputs = [\n",
    "    layer1,layer2,layer3,layer4,layer5,layer6,\n",
    "    layer1a,layer2a,layer3a,layer4a,layer5a,layer6a,\n",
    "    l1.getw(),l2.getw(),l3.getw(),l4.getw(),l5.getw(),l6.getw(),\n",
    "    grad1w,grad2w,grad3w,grad4w,grad5w,grad6w,\n",
    "    grad1p,grad2p,grad3p,grad4p,grad5p,grad6p,\n",
    "    grad1_up[0],grad2_up[0],grad3_up[0],grad4_up[0],grad5_up[0],grad6_up[0]\n",
    "]\n",
    "\n",
    "for iter in range(num_epoch):\n",
    "\n",
    "    # Training Accuracy    \n",
    "    for current_batch_index in range(0,len(train_images),batch_size):\n",
    "        current_data  = train_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = train_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy,gradient_update],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(train_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_train = avg_acc_train + sess_results[0]\n",
    "        \n",
    "    # get the results\n",
    "    mid_stat = sess.run(list_of_outputs,feed_dict={x:current_data,y:current_label})\n",
    "    \n",
    "    # Test Accuracy    \n",
    "    for current_batch_index in range(0,len(test_images), batch_size):\n",
    "        current_data  = test_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = test_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(test_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_test = avg_acc_test + sess_results[0]   \n",
    "        \n",
    "    # ======================== extract stats ========================\n",
    "    llayer1 = append_stat(llayer1,mid_stat,0);  llayer2 = append_stat(llayer2,mid_stat,1);  llayer3 = append_stat(llayer3,mid_stat,2);\n",
    "    llayer4 = append_stat(llayer4,mid_stat,3);  llayer5 = append_stat(llayer5,mid_stat,4);  llayer6 = append_stat(llayer6,mid_stat,5);\n",
    "\n",
    "    llayer1a = append_stat(llayer1a,mid_stat,6);  llayer2a = append_stat(llayer2a,mid_stat,7);  llayer3a = append_stat(llayer3a,mid_stat,8);\n",
    "    llayer4a = append_stat(llayer4a,mid_stat,9);  llayer5a = append_stat(llayer5a,mid_stat,10); llayer6a = append_stat(llayer6a,mid_stat,11);\n",
    "    \n",
    "    weight1 = append_stat(weight1,mid_stat,12);  weight2 = append_stat(weight2,mid_stat,13);  weight3 = append_stat(weight3,mid_stat,14);\n",
    "    weight4 = append_stat(weight4,mid_stat,15);  weight5 = append_stat(weight5,mid_stat,16);  weight6 = append_stat(weight6,mid_stat,17);\n",
    "    \n",
    "    gradw1 = append_stat(gradw1,mid_stat,18); gradw2 = append_stat(gradw2,mid_stat,19); gradw3 = append_stat(gradw3,mid_stat,20);\n",
    "    gradw4 = append_stat(gradw4,mid_stat,21); gradw5 = append_stat(gradw5,mid_stat,22); gradw6 = append_stat(gradw6,mid_stat,23);\n",
    "    \n",
    "    gradp1 = append_stat(gradp1,mid_stat,24); gradp2 = append_stat(gradp2,mid_stat,25); gradp3 = append_stat(gradp3,mid_stat,26);\n",
    "    gradp4 = append_stat(gradp4,mid_stat,27); gradp5 = append_stat(gradp5,mid_stat,28); gradp6 = append_stat(gradp6,mid_stat,29);\n",
    "\n",
    "    gradup1 = append_stat(gradup1,mid_stat,30); gradup2 = append_stat(gradup2,mid_stat,31); gradup3 = append_stat(gradup3,mid_stat,32);\n",
    "    gradup4 = append_stat(gradup4,mid_stat,33); gradup5 = append_stat(gradup5,mid_stat,34); gradup6 = append_stat(gradup6,mid_stat,35);\n",
    "\n",
    "    train_acc.append(avg_acc_train/(len(train_images)/batch_size))\n",
    "    test_acc .append(avg_acc_test / (len(test_images)/batch_size))\n",
    "    # ======================== extract stats ========================\n",
    "    \n",
    "    # ======================== save to image ========================\n",
    "    save_to_image(mid_stat[0:6]   ,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,\"layer\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[6:12]  ,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a,\"layera\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[12:18] ,weight1,weight2,weight3,weight4,weight5,weight6,\"weights\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[18:24] ,gradw1,gradw2,gradw3,gradw4,gradw5,gradw6,\"gradientw\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[24:30] ,gradp1,gradp2,gradp3,gradp4,gradp5,gradp6,\"gradientp\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[30:36] ,gradup1,gradup2,gradup3,gradup4,gradup5,gradup6,\"moment\",train_acc,test_acc,current_exp_name,iter)\n",
    "    # ======================== save to image ========================\n",
    "        \n",
    "    # ======================== print reset ========================\n",
    "    print(\"Current : \"+ str(iter) + \" Train Acc : \" + str(avg_acc_train/(len(train_images)/batch_size)) + \" Test Acc : \" + str(avg_acc_test/(len(test_images)/batch_size)) + '\\n')\n",
    "    avg_acc_train = 0 ; avg_acc_test  = 0\n",
    "    # ======================== print reset ========================\n",
    "\n",
    "np.save(current_exp_name+'/train_acc.npy',train_acc); np.save(current_exp_name+'/test_acc.npy', test_acc)    \n",
    "np.save(current_exp_name+'/llayer1.npy', llayer1);  np.save(current_exp_name+'/llayer2.npy', llayer2);  np.save(current_exp_name+'/llayer3.npy', llayer3); \n",
    "np.save(current_exp_name+'/llayer4.npy', llayer4);  np.save(current_exp_name+'/llayer5.npy', llayer5);  np.save(current_exp_name+'/llayer6.npy', llayer6); \n",
    "\n",
    "np.save(current_exp_name+'/llayer1a.npy', llayer1a);  np.save(current_exp_name+'/llayer2a.npy', llayer2a);  np.save(current_exp_name+'/llayer3a.npy', llayer3a); \n",
    "np.save(current_exp_name+'/llayer4a.npy', llayer4a);  np.save(current_exp_name+'/llayer5a.npy', llayer5a);  np.save(current_exp_name+'/llayer6a.npy', llayer6a); \n",
    "\n",
    "np.save(current_exp_name+'/weight1.npy', weight1);  np.save(current_exp_name+'/weight2.npy', weight2);  np.save(current_exp_name+'/weight3.npy', weight3);  \n",
    "np.save(current_exp_name+'/weight4.npy', weight4);  np.save(current_exp_name+'/weight5.npy', weight5);  np.save(current_exp_name+'/weight6.npy', weight6);  \n",
    "\n",
    "np.save(current_exp_name+'/gradw1.npy', gradw1); np.save(current_exp_name+'/gradw2.npy', gradw2); np.save(current_exp_name+'/gradw3.npy', gradw3);\n",
    "np.save(current_exp_name+'/gradw4.npy', gradw4); np.save(current_exp_name+'/gradw5.npy', gradw5); np.save(current_exp_name+'/gradw6.npy', gradw6);\n",
    "\n",
    "np.save(current_exp_name+'/gradp1.npy', gradp1); np.save(current_exp_name+'/gradp2.npy', gradp2); np.save(current_exp_name+'/gradp3.npy', gradp3);\n",
    "np.save(current_exp_name+'/gradp4.npy', gradp4); np.save(current_exp_name+'/gradp5.npy', gradp5); np.save(current_exp_name+'/gradp6.npy', gradp6);\n",
    "\n",
    "np.save(current_exp_name+'/gradup1.npy', gradup1); np.save(current_exp_name+'/gradup2.npy', gradup2); np.save(current_exp_name+'/gradup3.npy', gradup3);\n",
    "np.save(current_exp_name+'/gradup4.npy', gradup4); np.save(current_exp_name+'/gradup5.npy', gradup5); np.save(current_exp_name+'/gradup6.npy', gradup6);\n",
    "\n",
    "sess.close(); tf.reset_default_graph();\n",
    "\n",
    "%reset_selective -f l1,l2,l3,l4,l5,l6\n",
    "%reset_selective -f layer1,layer2,layer3,layer4,layer5,layer6\n",
    "%reset_selective -f layer1a,layer2a,layer3a,layer4a,layer5a,layer6a\n",
    "%reset_selective -f train_acc,test_acc,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a\n",
    "%reset_selective -f weight1,weight2,weight3,weight4,weight5,weight6\n",
    "%reset_selective -f gradw1,gradw2,gradw3,gradw4,gradw5,gradw6\n",
    "%reset_selective -f gradp1,gradp2,gradp3,gradp4,gradp5,gradp6\n",
    "%reset_selective -f gradup1,gradup2,gradup3,gradup4,gradup5,gradup6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-05T09:43:30.495501Z",
     "start_time": "2019-01-05T08:53:17.688614Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current : 0 Train Acc : 0.1282000027000904 Test Acc : 0.16050000282935797\n",
      "\n",
      "Current : 1 Train Acc : 0.20300000336766244 Test Acc : 0.25075000267475844\n",
      "\n",
      "Current : 2 Train Acc : 0.28500000208616255 Test Acc : 0.3080000021215528\n",
      "\n",
      "Current : 3 Train Acc : 0.2996000024974346 Test Acc : 0.3130000018142164\n",
      "\n",
      "Current : 4 Train Acc : 0.3142000027894974 Test Acc : 0.3287500016577542\n",
      "\n",
      "Current : 5 Train Acc : 0.32520000368356705 Test Acc : 0.3353750020172447\n",
      "\n",
      "Current : 6 Train Acc : 0.334200002014637 Test Acc : 0.34000000222586096\n",
      "\n",
      "Current : 7 Train Acc : 0.3384000027179718 Test Acc : 0.345125001585111\n",
      "\n",
      "Current : 8 Train Acc : 0.34240000146627425 Test Acc : 0.34775000139139595\n",
      "\n",
      "Current : 9 Train Acc : 0.34820000201463697 Test Acc : 0.3542500012461096\n",
      "\n",
      "Current : 10 Train Acc : 0.3538000017106533 Test Acc : 0.3602500017080456\n",
      "\n",
      "Current : 11 Train Acc : 0.3592000021338463 Test Acc : 0.36525000097230076\n",
      "\n",
      "Current : 12 Train Acc : 0.3654000018835068 Test Acc : 0.37137500088661907\n",
      "\n",
      "Current : 13 Train Acc : 0.37200000193715094 Test Acc : 0.3752500010281801\n",
      "\n",
      "Current : 14 Train Acc : 0.37860000267624855 Test Acc : 0.37825000047683716\n",
      "\n",
      "Current : 15 Train Acc : 0.38680000278353693 Test Acc : 0.38100000098347664\n",
      "\n",
      "Current : 16 Train Acc : 0.3896000021994114 Test Acc : 0.384500000551343\n",
      "\n",
      "Current : 17 Train Acc : 0.3930000023543835 Test Acc : 0.38825000045821073\n",
      "\n",
      "Current : 18 Train Acc : 0.39660000228881837 Test Acc : 0.3918750003539026\n",
      "\n",
      "Current : 19 Train Acc : 0.4038000023365021 Test Acc : 0.3962500004284084\n",
      "\n",
      "Current : 20 Train Acc : 0.40820000278949736 Test Acc : 0.3997499999962747\n",
      "\n",
      "Current : 21 Train Acc : 0.4108000022172928 Test Acc : 0.40037500045262275\n",
      "\n",
      "Current : 22 Train Acc : 0.41560000240802764 Test Acc : 0.401375000430271\n",
      "\n",
      "Current : 23 Train Acc : 0.41920000249147416 Test Acc : 0.4040000009443611\n",
      "\n",
      "Current : 24 Train Acc : 0.4198000022172928 Test Acc : 0.4063750008121133\n",
      "\n",
      "Current : 25 Train Acc : 0.423600002348423 Test Acc : 0.4081250006146729\n",
      "\n",
      "Current : 26 Train Acc : 0.42340000182390214 Test Acc : 0.4116250006109476\n",
      "\n",
      "Current : 27 Train Acc : 0.4258000025749207 Test Acc : 0.41312500094994903\n",
      "\n",
      "Current : 28 Train Acc : 0.42820000195503233 Test Acc : 0.41312500048428774\n",
      "\n",
      "Current : 29 Train Acc : 0.429200002014637 Test Acc : 0.41612500077113507\n",
      "\n",
      "Current : 30 Train Acc : 0.43460000115633013 Test Acc : 0.41962500104680656\n",
      "\n",
      "Current : 31 Train Acc : 0.43700000113248827 Test Acc : 0.4201250016503036\n",
      "\n",
      "Current : 32 Train Acc : 0.4400000006556511 Test Acc : 0.420750002451241\n",
      "\n",
      "Current : 33 Train Acc : 0.44100000005960466 Test Acc : 0.42312500208616255\n",
      "\n",
      "Current : 34 Train Acc : 0.4453999997675419 Test Acc : 0.4273750020936131\n",
      "\n",
      "Current : 35 Train Acc : 0.4495999999344349 Test Acc : 0.42537500167265535\n",
      "\n",
      "Current : 36 Train Acc : 0.4536000013649464 Test Acc : 0.4276250017248094\n",
      "\n",
      "Current : 37 Train Acc : 0.4596000020205975 Test Acc : 0.42812500139698384\n",
      "\n",
      "Current : 38 Train Acc : 0.4608000014722347 Test Acc : 0.42962500095367434\n",
      "\n",
      "Current : 39 Train Acc : 0.4640000012814999 Test Acc : 0.42937500173226\n",
      "\n",
      "Current : 40 Train Acc : 0.4654000011086464 Test Acc : 0.4308750009536743\n",
      "\n",
      "Current : 41 Train Acc : 0.46680000084638595 Test Acc : 0.43400000100955366\n",
      "\n",
      "Current : 42 Train Acc : 0.46800000077486037 Test Acc : 0.4366250016912818\n",
      "\n",
      "Current : 43 Train Acc : 0.46980000084638596 Test Acc : 0.4386250014975667\n",
      "\n",
      "Current : 44 Train Acc : 0.4694000008702278 Test Acc : 0.43950000155717134\n",
      "\n",
      "Current : 45 Train Acc : 0.4712000007033348 Test Acc : 0.4395000014454126\n",
      "\n",
      "Current : 46 Train Acc : 0.47160000014305115 Test Acc : 0.44150000080466273\n",
      "\n",
      "Current : 47 Train Acc : 0.47380000019073487 Test Acc : 0.4437500013783574\n",
      "\n",
      "Current : 48 Train Acc : 0.47520000088214875 Test Acc : 0.44462500136345623\n",
      "\n",
      "Current : 49 Train Acc : 0.47960000014305115 Test Acc : 0.4450000016018748\n",
      "\n",
      "Current : 50 Train Acc : 0.4808000010251999 Test Acc : 0.4443750019744039\n",
      "\n",
      "Current : 51 Train Acc : 0.48400000083446504 Test Acc : 0.4458750016614795\n",
      "\n",
      "Current : 52 Train Acc : 0.48820000094175336 Test Acc : 0.4455000013485551\n",
      "\n",
      "Current : 53 Train Acc : 0.4880000009536743 Test Acc : 0.44662500116974113\n",
      "\n",
      "Current : 54 Train Acc : 0.4920000007748604 Test Acc : 0.4467500016465783\n",
      "\n",
      "Current : 55 Train Acc : 0.494200001180172 Test Acc : 0.4458750006183982\n",
      "\n",
      "Current : 56 Train Acc : 0.4948000013232231 Test Acc : 0.44437500040978195\n",
      "\n",
      "Current : 57 Train Acc : 0.49600000113248827 Test Acc : 0.44562500044703485\n",
      "\n",
      "Current : 58 Train Acc : 0.4978000014424324 Test Acc : 0.4458750006183982\n",
      "\n",
      "Current : 59 Train Acc : 0.49840000075101853 Test Acc : 0.44675000142306087\n",
      "\n",
      "Current : 60 Train Acc : 0.49800000113248827 Test Acc : 0.44825000159442424\n",
      "\n",
      "Current : 61 Train Acc : 0.5010000012516975 Test Acc : 0.448875001296401\n",
      "\n",
      "Current : 62 Train Acc : 0.5010000017881393 Test Acc : 0.4503750016540289\n",
      "\n",
      "Current : 63 Train Acc : 0.5034000012874603 Test Acc : 0.45200000125914813\n",
      "\n",
      "Current : 64 Train Acc : 0.506400001168251 Test Acc : 0.4527500009164214\n",
      "\n",
      "Current : 65 Train Acc : 0.5058000016212464 Test Acc : 0.4556250012665987\n",
      "\n",
      "Current : 66 Train Acc : 0.5096000018119812 Test Acc : 0.4561250014603138\n",
      "\n",
      "Current : 67 Train Acc : 0.5120000015497208 Test Acc : 0.4568750010058284\n",
      "\n",
      "Current : 68 Train Acc : 0.510200001835823 Test Acc : 0.4571250010281801\n",
      "\n",
      "Current : 69 Train Acc : 0.5126000018119812 Test Acc : 0.4560000010207295\n",
      "\n",
      "Current : 70 Train Acc : 0.5126000012159347 Test Acc : 0.4570000009611249\n",
      "\n",
      "Current : 71 Train Acc : 0.5142000017166137 Test Acc : 0.4537500012665987\n",
      "\n",
      "Current : 72 Train Acc : 0.5158000013828278 Test Acc : 0.45237500097602606\n",
      "\n",
      "Current : 73 Train Acc : 0.5138000011444092 Test Acc : 0.4518750009313226\n",
      "\n",
      "Current : 74 Train Acc : 0.5166000009775161 Test Acc : 0.45312500059604643\n",
      "\n",
      "Current : 75 Train Acc : 0.5208000015020371 Test Acc : 0.4545000002533197\n",
      "\n",
      "Current : 76 Train Acc : 0.5220000019073486 Test Acc : 0.46075000062584875\n",
      "\n",
      "Current : 77 Train Acc : 0.5220000014305115 Test Acc : 0.4572500008717179\n",
      "\n",
      "Current : 78 Train Acc : 0.5262000010609627 Test Acc : 0.45687500115484\n",
      "\n",
      "Current : 79 Train Acc : 0.527200001180172 Test Acc : 0.459500001296401\n",
      "\n",
      "Current : 80 Train Acc : 0.5300000016093254 Test Acc : 0.4583750008419156\n",
      "\n",
      "Current : 81 Train Acc : 0.5310000011324882 Test Acc : 0.4582500009983778\n",
      "\n",
      "Current : 82 Train Acc : 0.5308000009655952 Test Acc : 0.4583750008791685\n",
      "\n",
      "Current : 83 Train Acc : 0.5340000002980232 Test Acc : 0.4575000010430813\n",
      "\n",
      "Current : 84 Train Acc : 0.5328000006079674 Test Acc : 0.4545000006631017\n",
      "\n",
      "Current : 85 Train Acc : 0.5346000004410744 Test Acc : 0.45400000032037496\n",
      "\n",
      "Current : 86 Train Acc : 0.5364000020623207 Test Acc : 0.4532500005885959\n",
      "\n",
      "Current : 87 Train Acc : 0.5382000018954277 Test Acc : 0.45300000071525576\n",
      "\n",
      "Current : 88 Train Acc : 0.5402000016570091 Test Acc : 0.4532500011101365\n",
      "\n",
      "Current : 89 Train Acc : 0.5416000017523765 Test Acc : 0.45250000115484\n",
      "\n",
      "Current : 90 Train Acc : 0.5440000010132789 Test Acc : 0.452125001065433\n",
      "\n",
      "Current : 91 Train Acc : 0.5452000003457069 Test Acc : 0.4537500012293458\n",
      "\n",
      "Current : 92 Train Acc : 0.5476000010371208 Test Acc : 0.4531250006705523\n",
      "\n",
      "Current : 93 Train Acc : 0.5492000005841255 Test Acc : 0.4517500002682209\n",
      "\n",
      "Current : 94 Train Acc : 0.549200000822544 Test Acc : 0.45275000046938657\n",
      "\n",
      "Current : 95 Train Acc : 0.5496000004410744 Test Acc : 0.4525000010430813\n",
      "\n",
      "Current : 96 Train Acc : 0.5514000000357628 Test Acc : 0.452125000692904\n",
      "\n",
      "Current : 97 Train Acc : 0.5504000012278557 Test Acc : 0.45112500082701446\n",
      "\n",
      "Current : 98 Train Acc : 0.5528000006079674 Test Acc : 0.45125000074505806\n",
      "\n",
      "Current : 99 Train Acc : 0.5535999994874 Test Acc : 0.45200000070035457\n",
      "\n",
      "Current : 100 Train Acc : 0.5554000003933907 Test Acc : 0.4520000009983778\n",
      "\n",
      "Current : 101 Train Acc : 0.5563999995589256 Test Acc : 0.4500000007450581\n",
      "\n",
      "Current : 102 Train Acc : 0.557599999845028 Test Acc : 0.4490000004321337\n",
      "\n",
      "Current : 103 Train Acc : 0.5584000008702278 Test Acc : 0.4491250003874302\n",
      "\n",
      "Current : 104 Train Acc : 0.5588000007271766 Test Acc : 0.4520000000670552\n",
      "\n",
      "Current : 105 Train Acc : 0.5600000006556511 Test Acc : 0.45262500062584876\n",
      "\n",
      "Current : 106 Train Acc : 0.5602000004649162 Test Acc : 0.45450000047683714\n",
      "\n",
      "Current : 107 Train Acc : 0.5621999993920326 Test Acc : 0.4562500002980232\n",
      "\n",
      "Current : 108 Train Acc : 0.5629999998211861 Test Acc : 0.45650000005960467\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current : 109 Train Acc : 0.5633999999165535 Test Acc : 0.4596250006556511\n",
      "\n",
      "Current : 110 Train Acc : 0.5656000003218651 Test Acc : 0.45937500029802325\n",
      "\n",
      "Current : 111 Train Acc : 0.5654000009894371 Test Acc : 0.4588750005140901\n",
      "\n",
      "Current : 112 Train Acc : 0.5666000007987022 Test Acc : 0.46012500070035456\n",
      "\n",
      "Current : 113 Train Acc : 0.5666000002026558 Test Acc : 0.4597500006854534\n",
      "\n",
      "Current : 114 Train Acc : 0.5688000006079674 Test Acc : 0.4597500008717179\n",
      "\n",
      "Current : 115 Train Acc : 0.5696000002026558 Test Acc : 0.4577500012516975\n",
      "\n",
      "Current : 116 Train Acc : 0.5691999997496605 Test Acc : 0.45862500064074996\n",
      "\n",
      "Current : 117 Train Acc : 0.5686000004410744 Test Acc : 0.4572500007599592\n",
      "\n",
      "Current : 118 Train Acc : 0.5689999997019768 Test Acc : 0.45500000048428774\n",
      "\n",
      "Current : 119 Train Acc : 0.5694000007510185 Test Acc : 0.4537500003352761\n",
      "\n",
      "Current : 120 Train Acc : 0.5712000005841256 Test Acc : 0.4532500002160668\n",
      "\n",
      "Current : 121 Train Acc : 0.5743999996781349 Test Acc : 0.4572500005364418\n",
      "\n",
      "Current : 122 Train Acc : 0.5747999998927117 Test Acc : 0.4555000000447035\n",
      "\n",
      "Current : 123 Train Acc : 0.5754000000357627 Test Acc : 0.4567500001192093\n",
      "\n",
      "Current : 124 Train Acc : 0.5766000003218651 Test Acc : 0.4595000006258488\n",
      "\n",
      "Current : 125 Train Acc : 0.5786000005602837 Test Acc : 0.4628750007972121\n",
      "\n",
      "Current : 126 Train Acc : 0.5792000007033348 Test Acc : 0.4628750005364418\n",
      "\n",
      "Current : 127 Train Acc : 0.5778000003695488 Test Acc : 0.46187500044703483\n",
      "\n",
      "Current : 128 Train Acc : 0.5816000007987022 Test Acc : 0.464750000461936\n",
      "\n",
      "Current : 129 Train Acc : 0.5824000001549721 Test Acc : 0.46475000083446505\n",
      "\n",
      "Current : 130 Train Acc : 0.583400000333786 Test Acc : 0.46575000032782554\n",
      "\n",
      "Current : 131 Train Acc : 0.5822000002861023 Test Acc : 0.4641250003874302\n",
      "\n",
      "Current : 132 Train Acc : 0.5853999997377396 Test Acc : 0.465125000551343\n",
      "\n",
      "Current : 133 Train Acc : 0.5868000000715256 Test Acc : 0.46662500090897086\n",
      "\n",
      "Current : 134 Train Acc : 0.5895999997854233 Test Acc : 0.46712500110268595\n",
      "\n",
      "Current : 135 Train Acc : 0.591400000333786 Test Acc : 0.4666250007599592\n",
      "\n",
      "Current : 136 Train Acc : 0.5930000005960464 Test Acc : 0.46662500105798244\n",
      "\n",
      "Current : 137 Train Acc : 0.5960000009536743 Test Acc : 0.4683750006556511\n",
      "\n",
      "Current : 138 Train Acc : 0.5963999999761581 Test Acc : 0.46687500081956385\n",
      "\n",
      "Current : 139 Train Acc : 0.597 Test Acc : 0.46925000108778475\n",
      "\n",
      "Current : 140 Train Acc : 0.5981999995708466 Test Acc : 0.46800000078976156\n",
      "\n",
      "Current : 141 Train Acc : 0.6001999992132186 Test Acc : 0.4687500013038516\n",
      "\n",
      "Current : 142 Train Acc : 0.6033999993801117 Test Acc : 0.46862500116229056\n",
      "\n",
      "Current : 143 Train Acc : 0.6035999993085861 Test Acc : 0.46800000093877314\n",
      "\n",
      "Current : 144 Train Acc : 0.6031999998092651 Test Acc : 0.46737500086426736\n",
      "\n",
      "Current : 145 Train Acc : 0.6045999999046325 Test Acc : 0.4686250015348196\n",
      "\n",
      "Current : 146 Train Acc : 0.6045999993085861 Test Acc : 0.4662500020116568\n",
      "\n",
      "Current : 147 Train Acc : 0.6047999994754791 Test Acc : 0.4645000016689301\n",
      "\n",
      "Current : 148 Train Acc : 0.6059999994039535 Test Acc : 0.46587500169873236\n",
      "\n",
      "Current : 149 Train Acc : 0.6063999992609024 Test Acc : 0.4660000016540289\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# I\n",
    "current_exp_name = 'I';\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "# create layers\n",
    "l1 = CNN(3,3, 16,which_reg=current_exp_name); \n",
    "l2 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l3 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "\n",
    "l4 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l5 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l6 = CNN(3,16,10,which_reg=current_exp_name); \n",
    "\n",
    "# 2. graph \n",
    "x = tf.placeholder(tf.float32,(batch_size,96,96,3))\n",
    "y = tf.placeholder(tf.float32,(batch_size,10))\n",
    "\n",
    "layer1, layer1a = l1. feedforward(x,stride=2)\n",
    "layer2, layer2a = l2. feedforward(layer1a,stride=2)\n",
    "layer3, layer3a = l3. feedforward(layer2a,stride=2)\n",
    "layer4, layer4a = l4. feedforward(layer3a,stride=2)\n",
    "layer5, layer5a = l5. feedforward(layer4a)\n",
    "layer6, layer6a = l6. feedforward(layer5a)\n",
    "\n",
    "final_layer   = tf.reduce_mean(layer6a,(1,2))\n",
    "final_softmax = tf_softmax(final_layer)\n",
    "cost          = -tf.reduce_mean(y * tf.log(final_softmax + 1e-8))\n",
    "correct_prediction = tf.equal(tf.argmax(final_softmax, 1), tf.argmax(y, 1))\n",
    "accuracy           = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "gradient = tf.tile((final_softmax-y)[:,None,None,:],[1,6,6,1])/batch_size\n",
    "grad6p,grad6w,grad6_up = l6.backprop(gradient)\n",
    "grad5p,grad5w,grad5_up = l5.backprop(grad6p)\n",
    "grad4p,grad4w,grad4_up = l4.backprop(grad5p,stride=2)\n",
    "grad3p,grad3w,grad3_up = l3.backprop(grad4p,stride=2)\n",
    "grad2p,grad2w,grad2_up = l2.backprop(grad3p,stride=2)\n",
    "grad1p,grad1w,grad1_up = l1.backprop(grad2p,stride=2)\n",
    "\n",
    "gradient_update = grad6_up + grad5_up + grad4_up + grad3_up + grad2_up + grad1_up \n",
    "\n",
    "# train\n",
    "sess.run(tf.global_variables_initializer())\n",
    "avg_acc_train = 0; avg_acc_test  = 0; train_acc = [];test_acc = []\n",
    "\n",
    "# mean std skew kurt non-zero\n",
    "llayer1 = [[],[],[],[],[]]; llayer2 = [[],[],[],[],[]]; llayer3 = [[],[],[],[],[]]\n",
    "llayer4 = [[],[],[],[],[]]; llayer5 = [[],[],[],[],[]]; llayer6 = [[],[],[],[],[]]\n",
    "\n",
    "llayer1a = [[],[],[],[],[]]; llayer2a = [[],[],[],[],[]]; llayer3a = [[],[],[],[],[]]\n",
    "llayer4a = [[],[],[],[],[]]; llayer5a = [[],[],[],[],[]]; llayer6a = [[],[],[],[],[]]\n",
    "\n",
    "weight1 = [[],[],[],[],[]]; weight2 = [[],[],[],[],[]]; weight3 = [[],[],[],[],[]];\n",
    "weight4 = [[],[],[],[],[]]; weight5 = [[],[],[],[],[]]; weight6 = [[],[],[],[],[]];\n",
    "\n",
    "gradw1  = [[],[],[],[],[]]; gradw2  = [[],[],[],[],[]]; gradw3  = [[],[],[],[],[]];\n",
    "gradw4  = [[],[],[],[],[]]; gradw5  = [[],[],[],[],[]]; gradw6  = [[],[],[],[],[]];\n",
    "\n",
    "gradp1  = [[],[],[],[],[]]; gradp2  = [[],[],[],[],[]]; gradp3  = [[],[],[],[],[]];\n",
    "gradp4  = [[],[],[],[],[]]; gradp5  = [[],[],[],[],[]]; gradp6  = [[],[],[],[],[]];\n",
    "\n",
    "gradup1  = [[],[],[],[],[]]; gradup2  = [[],[],[],[],[]]; gradup3  = [[],[],[],[],[]];\n",
    "gradup4  = [[],[],[],[],[]]; gradup5  = [[],[],[],[],[]]; gradup6  = [[],[],[],[],[]];\n",
    "\n",
    "list_of_outputs = [\n",
    "    layer1,layer2,layer3,layer4,layer5,layer6,\n",
    "    layer1a,layer2a,layer3a,layer4a,layer5a,layer6a,\n",
    "    l1.getw(),l2.getw(),l3.getw(),l4.getw(),l5.getw(),l6.getw(),\n",
    "    grad1w,grad2w,grad3w,grad4w,grad5w,grad6w,\n",
    "    grad1p,grad2p,grad3p,grad4p,grad5p,grad6p,\n",
    "    grad1_up[0],grad2_up[0],grad3_up[0],grad4_up[0],grad5_up[0],grad6_up[0]\n",
    "]\n",
    "\n",
    "for iter in range(num_epoch):\n",
    "\n",
    "    # Training Accuracy    \n",
    "    for current_batch_index in range(0,len(train_images),batch_size):\n",
    "        current_data  = train_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = train_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy,gradient_update],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(train_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_train = avg_acc_train + sess_results[0]\n",
    "        \n",
    "    # get the results\n",
    "    mid_stat = sess.run(list_of_outputs,feed_dict={x:current_data,y:current_label})\n",
    "    \n",
    "    # Test Accuracy    \n",
    "    for current_batch_index in range(0,len(test_images), batch_size):\n",
    "        current_data  = test_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = test_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(test_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_test = avg_acc_test + sess_results[0]   \n",
    "        \n",
    "    # ======================== extract stats ========================\n",
    "    llayer1 = append_stat(llayer1,mid_stat,0);  llayer2 = append_stat(llayer2,mid_stat,1);  llayer3 = append_stat(llayer3,mid_stat,2);\n",
    "    llayer4 = append_stat(llayer4,mid_stat,3);  llayer5 = append_stat(llayer5,mid_stat,4);  llayer6 = append_stat(llayer6,mid_stat,5);\n",
    "\n",
    "    llayer1a = append_stat(llayer1a,mid_stat,6);  llayer2a = append_stat(llayer2a,mid_stat,7);  llayer3a = append_stat(llayer3a,mid_stat,8);\n",
    "    llayer4a = append_stat(llayer4a,mid_stat,9);  llayer5a = append_stat(llayer5a,mid_stat,10); llayer6a = append_stat(llayer6a,mid_stat,11);\n",
    "    \n",
    "    weight1 = append_stat(weight1,mid_stat,12);  weight2 = append_stat(weight2,mid_stat,13);  weight3 = append_stat(weight3,mid_stat,14);\n",
    "    weight4 = append_stat(weight4,mid_stat,15);  weight5 = append_stat(weight5,mid_stat,16);  weight6 = append_stat(weight6,mid_stat,17);\n",
    "    \n",
    "    gradw1 = append_stat(gradw1,mid_stat,18); gradw2 = append_stat(gradw2,mid_stat,19); gradw3 = append_stat(gradw3,mid_stat,20);\n",
    "    gradw4 = append_stat(gradw4,mid_stat,21); gradw5 = append_stat(gradw5,mid_stat,22); gradw6 = append_stat(gradw6,mid_stat,23);\n",
    "    \n",
    "    gradp1 = append_stat(gradp1,mid_stat,24); gradp2 = append_stat(gradp2,mid_stat,25); gradp3 = append_stat(gradp3,mid_stat,26);\n",
    "    gradp4 = append_stat(gradp4,mid_stat,27); gradp5 = append_stat(gradp5,mid_stat,28); gradp6 = append_stat(gradp6,mid_stat,29);\n",
    "\n",
    "    gradup1 = append_stat(gradup1,mid_stat,30); gradup2 = append_stat(gradup2,mid_stat,31); gradup3 = append_stat(gradup3,mid_stat,32);\n",
    "    gradup4 = append_stat(gradup4,mid_stat,33); gradup5 = append_stat(gradup5,mid_stat,34); gradup6 = append_stat(gradup6,mid_stat,35);\n",
    "\n",
    "    train_acc.append(avg_acc_train/(len(train_images)/batch_size))\n",
    "    test_acc .append(avg_acc_test / (len(test_images)/batch_size))\n",
    "    # ======================== extract stats ========================\n",
    "    \n",
    "    # ======================== save to image ========================\n",
    "    save_to_image(mid_stat[0:6]   ,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,\"layer\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[6:12]  ,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a,\"layera\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[12:18] ,weight1,weight2,weight3,weight4,weight5,weight6,\"weights\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[18:24] ,gradw1,gradw2,gradw3,gradw4,gradw5,gradw6,\"gradientw\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[24:30] ,gradp1,gradp2,gradp3,gradp4,gradp5,gradp6,\"gradientp\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[30:36] ,gradup1,gradup2,gradup3,gradup4,gradup5,gradup6,\"moment\",train_acc,test_acc,current_exp_name,iter)\n",
    "    # ======================== save to image ========================\n",
    "        \n",
    "    # ======================== print reset ========================\n",
    "    print(\"Current : \"+ str(iter) + \" Train Acc : \" + str(avg_acc_train/(len(train_images)/batch_size)) + \" Test Acc : \" + str(avg_acc_test/(len(test_images)/batch_size)) + '\\n')\n",
    "    avg_acc_train = 0 ; avg_acc_test  = 0\n",
    "    # ======================== print reset ========================\n",
    "\n",
    "np.save(current_exp_name+'/train_acc.npy',train_acc); np.save(current_exp_name+'/test_acc.npy', test_acc)    \n",
    "np.save(current_exp_name+'/llayer1.npy', llayer1);  np.save(current_exp_name+'/llayer2.npy', llayer2);  np.save(current_exp_name+'/llayer3.npy', llayer3); \n",
    "np.save(current_exp_name+'/llayer4.npy', llayer4);  np.save(current_exp_name+'/llayer5.npy', llayer5);  np.save(current_exp_name+'/llayer6.npy', llayer6); \n",
    "\n",
    "np.save(current_exp_name+'/llayer1a.npy', llayer1a);  np.save(current_exp_name+'/llayer2a.npy', llayer2a);  np.save(current_exp_name+'/llayer3a.npy', llayer3a); \n",
    "np.save(current_exp_name+'/llayer4a.npy', llayer4a);  np.save(current_exp_name+'/llayer5a.npy', llayer5a);  np.save(current_exp_name+'/llayer6a.npy', llayer6a); \n",
    "\n",
    "np.save(current_exp_name+'/weight1.npy', weight1);  np.save(current_exp_name+'/weight2.npy', weight2);  np.save(current_exp_name+'/weight3.npy', weight3);  \n",
    "np.save(current_exp_name+'/weight4.npy', weight4);  np.save(current_exp_name+'/weight5.npy', weight5);  np.save(current_exp_name+'/weight6.npy', weight6);  \n",
    "\n",
    "np.save(current_exp_name+'/gradw1.npy', gradw1); np.save(current_exp_name+'/gradw2.npy', gradw2); np.save(current_exp_name+'/gradw3.npy', gradw3);\n",
    "np.save(current_exp_name+'/gradw4.npy', gradw4); np.save(current_exp_name+'/gradw5.npy', gradw5); np.save(current_exp_name+'/gradw6.npy', gradw6);\n",
    "\n",
    "np.save(current_exp_name+'/gradp1.npy', gradp1); np.save(current_exp_name+'/gradp2.npy', gradp2); np.save(current_exp_name+'/gradp3.npy', gradp3);\n",
    "np.save(current_exp_name+'/gradp4.npy', gradp4); np.save(current_exp_name+'/gradp5.npy', gradp5); np.save(current_exp_name+'/gradp6.npy', gradp6);\n",
    "\n",
    "np.save(current_exp_name+'/gradup1.npy', gradup1); np.save(current_exp_name+'/gradup2.npy', gradup2); np.save(current_exp_name+'/gradup3.npy', gradup3);\n",
    "np.save(current_exp_name+'/gradup4.npy', gradup4); np.save(current_exp_name+'/gradup5.npy', gradup5); np.save(current_exp_name+'/gradup6.npy', gradup6);\n",
    "\n",
    "sess.close(); tf.reset_default_graph();\n",
    "\n",
    "%reset_selective -f l1,l2,l3,l4,l5,l6\n",
    "%reset_selective -f layer1,layer2,layer3,layer4,layer5,layer6\n",
    "%reset_selective -f layer1a,layer2a,layer3a,layer4a,layer5a,layer6a\n",
    "%reset_selective -f train_acc,test_acc,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a\n",
    "%reset_selective -f weight1,weight2,weight3,weight4,weight5,weight6\n",
    "%reset_selective -f gradw1,gradw2,gradw3,gradw4,gradw5,gradw6\n",
    "%reset_selective -f gradp1,gradp2,gradp3,gradp4,gradp5,gradp6\n",
    "%reset_selective -f gradup1,gradup2,gradup3,gradup4,gradup5,gradup6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-05T10:32:12.592282Z",
     "start_time": "2019-01-05T09:43:30.532375Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current : 0 Train Acc : 0.13660000264644623 Test Acc : 0.18250000298954547\n",
      "\n",
      "Current : 1 Train Acc : 0.1886000033020973 Test Acc : 0.24550000268034636\n",
      "\n",
      "Current : 2 Train Acc : 0.26500000309944155 Test Acc : 0.28837500270456073\n",
      "\n",
      "Current : 3 Train Acc : 0.3016000020503998 Test Acc : 0.30175000315532086\n",
      "\n",
      "Current : 4 Train Acc : 0.31720000216364863 Test Acc : 0.3260000018123537\n",
      "\n",
      "Current : 5 Train Acc : 0.33960000149905684 Test Acc : 0.33725000120699405\n",
      "\n",
      "Current : 6 Train Acc : 0.35220000141859054 Test Acc : 0.3482500013336539\n",
      "\n",
      "Current : 7 Train Acc : 0.35900000178813934 Test Acc : 0.350250001354143\n",
      "\n",
      "Current : 8 Train Acc : 0.36860000175237656 Test Acc : 0.3653750009648502\n",
      "\n",
      "Current : 9 Train Acc : 0.3840000012218952 Test Acc : 0.3838750014733523\n",
      "\n",
      "Current : 10 Train Acc : 0.3950000014901161 Test Acc : 0.39362500051036475\n",
      "\n",
      "Current : 11 Train Acc : 0.40060000133514406 Test Acc : 0.40037500105798246\n",
      "\n",
      "Current : 12 Train Acc : 0.405400001347065 Test Acc : 0.40200000124052165\n",
      "\n",
      "Current : 13 Train Acc : 0.40860000199079516 Test Acc : 0.40675000151619317\n",
      "\n",
      "Current : 14 Train Acc : 0.4096000016927719 Test Acc : 0.4077500013448298\n",
      "\n",
      "Current : 15 Train Acc : 0.41220000201463697 Test Acc : 0.41175000118091704\n",
      "\n",
      "Current : 16 Train Acc : 0.41620000123977663 Test Acc : 0.41300000140443444\n",
      "\n",
      "Current : 17 Train Acc : 0.41980000185966493 Test Acc : 0.4147500008530915\n",
      "\n",
      "Current : 18 Train Acc : 0.4226000012159348 Test Acc : 0.41612500106915834\n",
      "\n",
      "Current : 19 Train Acc : 0.42500000101327895 Test Acc : 0.4166250010207295\n",
      "\n",
      "Current : 20 Train Acc : 0.43160000002384186 Test Acc : 0.4195000008493662\n",
      "\n",
      "Current : 21 Train Acc : 0.43480000057816504 Test Acc : 0.4201250014081597\n",
      "\n",
      "Current : 22 Train Acc : 0.4309999998509884 Test Acc : 0.4203750006482005\n",
      "\n",
      "Current : 23 Train Acc : 0.4392000005841255 Test Acc : 0.42387500137090683\n",
      "\n",
      "Current : 24 Train Acc : 0.4402000004053116 Test Acc : 0.42300000101327895\n",
      "\n",
      "Current : 25 Train Acc : 0.442000000834465 Test Acc : 0.4245000015571713\n",
      "\n",
      "Current : 26 Train Acc : 0.4428000010251999 Test Acc : 0.4271250019595027\n",
      "\n",
      "Current : 27 Train Acc : 0.4462000013589859 Test Acc : 0.42675000220537185\n",
      "\n",
      "Current : 28 Train Acc : 0.45040000200271607 Test Acc : 0.4296250017359853\n",
      "\n",
      "Current : 29 Train Acc : 0.4514000011086464 Test Acc : 0.4320000013336539\n",
      "\n",
      "Current : 30 Train Acc : 0.4548000004291534 Test Acc : 0.43375000104308126\n",
      "\n",
      "Current : 31 Train Acc : 0.45780000030994417 Test Acc : 0.43412500075995925\n",
      "\n",
      "Current : 32 Train Acc : 0.4592000004649162 Test Acc : 0.43537500109523536\n",
      "\n",
      "Current : 33 Train Acc : 0.46039999967813494 Test Acc : 0.435625000782311\n",
      "\n",
      "Current : 34 Train Acc : 0.4636000003814697 Test Acc : 0.43562500055879355\n",
      "\n",
      "Current : 35 Train Acc : 0.46459999918937683 Test Acc : 0.43862500093877316\n",
      "\n",
      "Current : 36 Train Acc : 0.4646000012755394 Test Acc : 0.4405000009387732\n",
      "\n",
      "Current : 37 Train Acc : 0.4668000006079674 Test Acc : 0.4417500004917383\n",
      "\n",
      "Current : 38 Train Acc : 0.46760000079870223 Test Acc : 0.4432500006258488\n",
      "\n",
      "Current : 39 Train Acc : 0.46920000106096266 Test Acc : 0.4447500009834766\n",
      "\n",
      "Current : 40 Train Acc : 0.4720000011920929 Test Acc : 0.4445000008866191\n",
      "\n",
      "Current : 41 Train Acc : 0.47600000101327894 Test Acc : 0.4445000008866191\n",
      "\n",
      "Current : 42 Train Acc : 0.47820000106096267 Test Acc : 0.44562500074505806\n",
      "\n",
      "Current : 43 Train Acc : 0.47980000108480453 Test Acc : 0.4481250012665987\n",
      "\n",
      "Current : 44 Train Acc : 0.48280000168085097 Test Acc : 0.44862500138580796\n",
      "\n",
      "Current : 45 Train Acc : 0.4832000021338463 Test Acc : 0.4487500013783574\n",
      "\n",
      "Current : 46 Train Acc : 0.4838000001311302 Test Acc : 0.4471250021457672\n",
      "\n",
      "Current : 47 Train Acc : 0.4820000010728836 Test Acc : 0.44100000120699406\n",
      "\n",
      "Current : 48 Train Acc : 0.47920000141859054 Test Acc : 0.44450000174343585\n",
      "\n",
      "Current : 49 Train Acc : 0.4854000012278557 Test Acc : 0.45250000193715095\n",
      "\n",
      "Current : 50 Train Acc : 0.4926000006198883 Test Acc : 0.45712500117719174\n",
      "\n",
      "Current : 51 Train Acc : 0.4954000017642975 Test Acc : 0.459375\n",
      "\n",
      "Current : 52 Train Acc : 0.4964000019431114 Test Acc : 0.4601250009983778\n",
      "\n",
      "Current : 53 Train Acc : 0.4956000012159347 Test Acc : 0.46100000068545344\n",
      "\n",
      "Current : 54 Train Acc : 0.5028000012040138 Test Acc : 0.4611250007525086\n",
      "\n",
      "Current : 55 Train Acc : 0.4966000004410744 Test Acc : 0.46350000075995923\n",
      "\n",
      "Current : 56 Train Acc : 0.5023999999165535 Test Acc : 0.466875000782311\n",
      "\n",
      "Current : 57 Train Acc : 0.5034000005125999 Test Acc : 0.46575000040233133\n",
      "\n",
      "Current : 58 Train Acc : 0.5088000009059906 Test Acc : 0.4665000012889504\n",
      "\n",
      "Current : 59 Train Acc : 0.5126000009775161 Test Acc : 0.4665000012889504\n",
      "\n",
      "Current : 60 Train Acc : 0.5166000013947487 Test Acc : 0.47075000062584876\n",
      "\n",
      "Current : 61 Train Acc : 0.5188000002503395 Test Acc : 0.47250000067055226\n",
      "\n",
      "Current : 62 Train Acc : 0.5216000011563301 Test Acc : 0.4700000008940697\n",
      "\n",
      "Current : 63 Train Acc : 0.5198000001907349 Test Acc : 0.46412500116974115\n",
      "\n",
      "Current : 64 Train Acc : 0.5192000009417533 Test Acc : 0.46850000057369473\n",
      "\n",
      "Current : 65 Train Acc : 0.523000001847744 Test Acc : 0.4708750009909272\n",
      "\n",
      "Current : 66 Train Acc : 0.5229999993443489 Test Acc : 0.46525000102818015\n",
      "\n",
      "Current : 67 Train Acc : 0.5218000002503395 Test Acc : 0.46062500067055223\n",
      "\n",
      "Current : 68 Train Acc : 0.5242000002264976 Test Acc : 0.4581250011175871\n",
      "\n",
      "Current : 69 Train Acc : 0.5265999996066093 Test Acc : 0.4562500009685755\n",
      "\n",
      "Current : 70 Train Acc : 0.5291999997496605 Test Acc : 0.45350000116974115\n",
      "\n",
      "Current : 71 Train Acc : 0.5333999997973442 Test Acc : 0.45637500084936616\n",
      "\n",
      "Current : 72 Train Acc : 0.537800000667572 Test Acc : 0.4556250009685755\n",
      "\n",
      "Current : 73 Train Acc : 0.539800000667572 Test Acc : 0.4571250006556511\n",
      "\n",
      "Current : 74 Train Acc : 0.5426000000238419 Test Acc : 0.4588750006258488\n",
      "\n",
      "Current : 75 Train Acc : 0.5454000000953674 Test Acc : 0.45575000140815974\n",
      "\n",
      "Current : 76 Train Acc : 0.5458000001907348 Test Acc : 0.4547500008717179\n",
      "\n",
      "Current : 77 Train Acc : 0.5474000002145767 Test Acc : 0.4587500008568168\n",
      "\n",
      "Current : 78 Train Acc : 0.5487999999523163 Test Acc : 0.4557500012591481\n",
      "\n",
      "Current : 79 Train Acc : 0.5524000005722046 Test Acc : 0.45325000148266553\n",
      "\n",
      "Current : 80 Train Acc : 0.552600000500679 Test Acc : 0.4580000011250377\n",
      "\n",
      "Current : 81 Train Acc : 0.5561999989748001 Test Acc : 0.4562500010803342\n",
      "\n",
      "Current : 82 Train Acc : 0.5580000004768372 Test Acc : 0.45850000079721215\n",
      "\n",
      "Current : 83 Train Acc : 0.5592000004053116 Test Acc : 0.45412500113248827\n",
      "\n",
      "Current : 84 Train Acc : 0.5608000010251999 Test Acc : 0.4590000014007092\n",
      "\n",
      "Current : 85 Train Acc : 0.5614000011682511 Test Acc : 0.4582500016316772\n",
      "\n",
      "Current : 86 Train Acc : 0.5640000009536743 Test Acc : 0.4593750014528632\n",
      "\n",
      "Current : 87 Train Acc : 0.5642000012397766 Test Acc : 0.45950000181794165\n",
      "\n",
      "Current : 88 Train Acc : 0.5688000005483628 Test Acc : 0.46650000136345626\n",
      "\n",
      "Current : 89 Train Acc : 0.5692000013589859 Test Acc : 0.46925000127404926\n",
      "\n",
      "Current : 90 Train Acc : 0.5716000019311905 Test Acc : 0.4695000011101365\n",
      "\n",
      "Current : 91 Train Acc : 0.5742000012397767 Test Acc : 0.47200000151991844\n",
      "\n",
      "Current : 92 Train Acc : 0.5750000020265579 Test Acc : 0.46937500208616256\n",
      "\n",
      "Current : 93 Train Acc : 0.5786000009775162 Test Acc : 0.46550000216811893\n",
      "\n",
      "Current : 94 Train Acc : 0.5782000012397767 Test Acc : 0.46412500225007536\n",
      "\n",
      "Current : 95 Train Acc : 0.5786000015735626 Test Acc : 0.4632500021904707\n",
      "\n",
      "Current : 96 Train Acc : 0.5812000013589859 Test Acc : 0.46850000232458117\n",
      "\n",
      "Current : 97 Train Acc : 0.5838000013828277 Test Acc : 0.46875000204890965\n",
      "\n",
      "Current : 98 Train Acc : 0.5848000017404557 Test Acc : 0.4678750017285347\n",
      "\n",
      "Current : 99 Train Acc : 0.5864000006914138 Test Acc : 0.4701250020414591\n",
      "\n",
      "Current : 100 Train Acc : 0.5874000015258789 Test Acc : 0.47025000259280203\n",
      "\n",
      "Current : 101 Train Acc : 0.5874000017642975 Test Acc : 0.4688750024139881\n",
      "\n",
      "Current : 102 Train Acc : 0.5896000012159347 Test Acc : 0.47237500254064796\n",
      "\n",
      "Current : 103 Train Acc : 0.5910000010728836 Test Acc : 0.47362500201910734\n",
      "\n",
      "Current : 104 Train Acc : 0.5932000005245208 Test Acc : 0.4768750014156103\n",
      "\n",
      "Current : 105 Train Acc : 0.5929999996423722 Test Acc : 0.476125001385808\n",
      "\n",
      "Current : 106 Train Acc : 0.5931999996900559 Test Acc : 0.478750001527369\n",
      "\n",
      "Current : 107 Train Acc : 0.593399999499321 Test Acc : 0.47587500113993886\n",
      "\n",
      "Current : 108 Train Acc : 0.5951999990940094 Test Acc : 0.47712500117719175\n",
      "\n",
      "Current : 109 Train Acc : 0.5949999994039535 Test Acc : 0.4768750010803342\n",
      "\n",
      "Current : 110 Train Acc : 0.598799998998642 Test Acc : 0.47712500136345626\n",
      "\n",
      "Current : 111 Train Acc : 0.5996000003814698 Test Acc : 0.4783750009909272\n",
      "\n",
      "Current : 112 Train Acc : 0.6013999998569488 Test Acc : 0.4791250006482005\n",
      "\n",
      "Current : 113 Train Acc : 0.6023999993801117 Test Acc : 0.48037500116974113\n",
      "\n",
      "Current : 114 Train Acc : 0.6053999999761581 Test Acc : 0.4810000007227063\n",
      "\n",
      "Current : 115 Train Acc : 0.6056000000238418 Test Acc : 0.4816250003501773\n",
      "\n",
      "Current : 116 Train Acc : 0.607 Test Acc : 0.4832500009611249\n",
      "\n",
      "Current : 117 Train Acc : 0.608800000667572 Test Acc : 0.4823749998956919\n",
      "\n",
      "Current : 118 Train Acc : 0.6116000007390976 Test Acc : 0.48474999975413086\n",
      "\n",
      "Current : 119 Train Acc : 0.6124000014066696 Test Acc : 0.48712500046938656\n",
      "\n",
      "Current : 120 Train Acc : 0.6146000007390976 Test Acc : 0.4878750003129244\n",
      "\n",
      "Current : 121 Train Acc : 0.6156000008583069 Test Acc : 0.49162500087171795\n",
      "\n",
      "Current : 122 Train Acc : 0.616000000834465 Test Acc : 0.4931250007078052\n",
      "\n",
      "Current : 123 Train Acc : 0.6184000006914139 Test Acc : 0.49150000091642143\n",
      "\n",
      "Current : 124 Train Acc : 0.6218000005483627 Test Acc : 0.4907500008493662\n",
      "\n",
      "Current : 125 Train Acc : 0.6210000005960464 Test Acc : 0.4868750006705522\n",
      "\n",
      "Current : 126 Train Acc : 0.6188000018596649 Test Acc : 0.48400000024586914\n",
      "\n",
      "Current : 127 Train Acc : 0.6224000002145768 Test Acc : 0.4862500005587935\n",
      "\n",
      "Current : 128 Train Acc : 0.6228000007867813 Test Acc : 0.48700000032782553\n",
      "\n",
      "Current : 129 Train Acc : 0.6240000013113022 Test Acc : 0.4895000008866191\n",
      "\n",
      "Current : 130 Train Acc : 0.6260000009536744 Test Acc : 0.49150000073015687\n",
      "\n",
      "Current : 131 Train Acc : 0.6288000000715256 Test Acc : 0.4900000008568168\n",
      "\n",
      "Current : 132 Train Acc : 0.6276000009775162 Test Acc : 0.4911250004544854\n",
      "\n",
      "Current : 133 Train Acc : 0.6290000005960464 Test Acc : 0.49025000039488076\n",
      "\n",
      "Current : 134 Train Acc : 0.630400000333786 Test Acc : 0.4917500003427267\n",
      "\n",
      "Current : 135 Train Acc : 0.6360000003576278 Test Acc : 0.49362500112503765\n",
      "\n",
      "Current : 136 Train Acc : 0.6367999993562699 Test Acc : 0.4922500010207295\n",
      "\n",
      "Current : 137 Train Acc : 0.6366000002622605 Test Acc : 0.49475000094622373\n",
      "\n",
      "Current : 138 Train Acc : 0.6362000008821488 Test Acc : 0.5012500010430813\n",
      "\n",
      "Current : 139 Train Acc : 0.6388000010251998 Test Acc : 0.5032500016689301\n",
      "\n",
      "Current : 140 Train Acc : 0.639600000500679 Test Acc : 0.5033750013262034\n",
      "\n",
      "Current : 141 Train Acc : 0.6410000003576278 Test Acc : 0.504000001847744\n",
      "\n",
      "Current : 142 Train Acc : 0.6426000010967254 Test Acc : 0.5057500015199184\n",
      "\n",
      "Current : 143 Train Acc : 0.6456000007390976 Test Acc : 0.5045000021159649\n",
      "\n",
      "Current : 144 Train Acc : 0.6486000013351441 Test Acc : 0.5035000019520521\n",
      "\n",
      "Current : 145 Train Acc : 0.6496000003814697 Test Acc : 0.5035000019520521\n",
      "\n",
      "Current : 146 Train Acc : 0.6510000004768371 Test Acc : 0.5041250017285347\n",
      "\n",
      "Current : 147 Train Acc : 0.653400000333786 Test Acc : 0.5048750015348196\n",
      "\n",
      "Current : 148 Train Acc : 0.6538000013828278 Test Acc : 0.504000001475215\n",
      "\n",
      "Current : 149 Train Acc : 0.6552000005245209 Test Acc : 0.5041250016540288\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# J\n",
    "current_exp_name = 'J';\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "# create layers\n",
    "l1 = CNN(3,3, 16,which_reg=current_exp_name); \n",
    "l2 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l3 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "\n",
    "l4 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l5 = CNN(3,16,16,which_reg=current_exp_name); \n",
    "l6 = CNN(3,16,10,which_reg=current_exp_name); \n",
    "\n",
    "# 2. graph \n",
    "x = tf.placeholder(tf.float32,(batch_size,96,96,3))\n",
    "y = tf.placeholder(tf.float32,(batch_size,10))\n",
    "\n",
    "layer1, layer1a = l1. feedforward(x,stride=2)\n",
    "layer2, layer2a = l2. feedforward(layer1a,stride=2)\n",
    "layer3, layer3a = l3. feedforward(layer2a,stride=2)\n",
    "layer4, layer4a = l4. feedforward(layer3a,stride=2)\n",
    "layer5, layer5a = l5. feedforward(layer4a)\n",
    "layer6, layer6a = l6. feedforward(layer5a)\n",
    "\n",
    "final_layer   = tf.reduce_mean(layer6a,(1,2))\n",
    "final_softmax = tf_softmax(final_layer)\n",
    "cost          = -tf.reduce_mean(y * tf.log(final_softmax + 1e-8))\n",
    "correct_prediction = tf.equal(tf.argmax(final_softmax, 1), tf.argmax(y, 1))\n",
    "accuracy           = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "gradient = tf.tile((final_softmax-y)[:,None,None,:],[1,6,6,1])/batch_size\n",
    "grad6p,grad6w,grad6_up = l6.backprop(gradient)\n",
    "grad5p,grad5w,grad5_up = l5.backprop(grad6p)\n",
    "grad4p,grad4w,grad4_up = l4.backprop(grad5p,stride=2)\n",
    "grad3p,grad3w,grad3_up = l3.backprop(grad4p,stride=2)\n",
    "grad2p,grad2w,grad2_up = l2.backprop(grad3p,stride=2)\n",
    "grad1p,grad1w,grad1_up = l1.backprop(grad2p,stride=2)\n",
    "\n",
    "gradient_update = grad6_up + grad5_up + grad4_up + grad3_up + grad2_up + grad1_up \n",
    "\n",
    "# train\n",
    "sess.run(tf.global_variables_initializer())\n",
    "avg_acc_train = 0; avg_acc_test  = 0; train_acc = [];test_acc = []\n",
    "\n",
    "# mean std skew kurt non-zero\n",
    "llayer1 = [[],[],[],[],[]]; llayer2 = [[],[],[],[],[]]; llayer3 = [[],[],[],[],[]]\n",
    "llayer4 = [[],[],[],[],[]]; llayer5 = [[],[],[],[],[]]; llayer6 = [[],[],[],[],[]]\n",
    "\n",
    "llayer1a = [[],[],[],[],[]]; llayer2a = [[],[],[],[],[]]; llayer3a = [[],[],[],[],[]]\n",
    "llayer4a = [[],[],[],[],[]]; llayer5a = [[],[],[],[],[]]; llayer6a = [[],[],[],[],[]]\n",
    "\n",
    "weight1 = [[],[],[],[],[]]; weight2 = [[],[],[],[],[]]; weight3 = [[],[],[],[],[]];\n",
    "weight4 = [[],[],[],[],[]]; weight5 = [[],[],[],[],[]]; weight6 = [[],[],[],[],[]];\n",
    "\n",
    "gradw1  = [[],[],[],[],[]]; gradw2  = [[],[],[],[],[]]; gradw3  = [[],[],[],[],[]];\n",
    "gradw4  = [[],[],[],[],[]]; gradw5  = [[],[],[],[],[]]; gradw6  = [[],[],[],[],[]];\n",
    "\n",
    "gradp1  = [[],[],[],[],[]]; gradp2  = [[],[],[],[],[]]; gradp3  = [[],[],[],[],[]];\n",
    "gradp4  = [[],[],[],[],[]]; gradp5  = [[],[],[],[],[]]; gradp6  = [[],[],[],[],[]];\n",
    "\n",
    "gradup1  = [[],[],[],[],[]]; gradup2  = [[],[],[],[],[]]; gradup3  = [[],[],[],[],[]];\n",
    "gradup4  = [[],[],[],[],[]]; gradup5  = [[],[],[],[],[]]; gradup6  = [[],[],[],[],[]];\n",
    "\n",
    "list_of_outputs = [\n",
    "    layer1,layer2,layer3,layer4,layer5,layer6,\n",
    "    layer1a,layer2a,layer3a,layer4a,layer5a,layer6a,\n",
    "    l1.getw(),l2.getw(),l3.getw(),l4.getw(),l5.getw(),l6.getw(),\n",
    "    grad1w,grad2w,grad3w,grad4w,grad5w,grad6w,\n",
    "    grad1p,grad2p,grad3p,grad4p,grad5p,grad6p,\n",
    "    grad1_up[0],grad2_up[0],grad3_up[0],grad4_up[0],grad5_up[0],grad6_up[0]\n",
    "]\n",
    "\n",
    "for iter in range(num_epoch):\n",
    "\n",
    "    # Training Accuracy    \n",
    "    for current_batch_index in range(0,len(train_images),batch_size):\n",
    "        current_data  = train_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = train_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy,gradient_update],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(train_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_train = avg_acc_train + sess_results[0]\n",
    "        \n",
    "    # get the results\n",
    "    mid_stat = sess.run(list_of_outputs,feed_dict={x:current_data,y:current_label})\n",
    "    \n",
    "    # Test Accuracy    \n",
    "    for current_batch_index in range(0,len(test_images), batch_size):\n",
    "        current_data  = test_images[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        current_label = test_labels[current_batch_index:current_batch_index+batch_size].astype(np.float32)\n",
    "        sess_results  = sess.run([accuracy],feed_dict={x:current_data,y:current_label})\n",
    "        sys.stdout.write('Current Iter : ' + str(iter) + '/'+ str(num_epoch)  + ' batch : ' + str(current_batch_index) + '/'+ str(len(test_images)) + ' acc : ' + str(sess_results[0]) + '\\r')\n",
    "        sys.stdout.flush(); avg_acc_test = avg_acc_test + sess_results[0]   \n",
    "        \n",
    "    # ======================== extract stats ========================\n",
    "    llayer1 = append_stat(llayer1,mid_stat,0);  llayer2 = append_stat(llayer2,mid_stat,1);  llayer3 = append_stat(llayer3,mid_stat,2);\n",
    "    llayer4 = append_stat(llayer4,mid_stat,3);  llayer5 = append_stat(llayer5,mid_stat,4);  llayer6 = append_stat(llayer6,mid_stat,5);\n",
    "\n",
    "    llayer1a = append_stat(llayer1a,mid_stat,6);  llayer2a = append_stat(llayer2a,mid_stat,7);  llayer3a = append_stat(llayer3a,mid_stat,8);\n",
    "    llayer4a = append_stat(llayer4a,mid_stat,9);  llayer5a = append_stat(llayer5a,mid_stat,10); llayer6a = append_stat(llayer6a,mid_stat,11);\n",
    "    \n",
    "    weight1 = append_stat(weight1,mid_stat,12);  weight2 = append_stat(weight2,mid_stat,13);  weight3 = append_stat(weight3,mid_stat,14);\n",
    "    weight4 = append_stat(weight4,mid_stat,15);  weight5 = append_stat(weight5,mid_stat,16);  weight6 = append_stat(weight6,mid_stat,17);\n",
    "    \n",
    "    gradw1 = append_stat(gradw1,mid_stat,18); gradw2 = append_stat(gradw2,mid_stat,19); gradw3 = append_stat(gradw3,mid_stat,20);\n",
    "    gradw4 = append_stat(gradw4,mid_stat,21); gradw5 = append_stat(gradw5,mid_stat,22); gradw6 = append_stat(gradw6,mid_stat,23);\n",
    "    \n",
    "    gradp1 = append_stat(gradp1,mid_stat,24); gradp2 = append_stat(gradp2,mid_stat,25); gradp3 = append_stat(gradp3,mid_stat,26);\n",
    "    gradp4 = append_stat(gradp4,mid_stat,27); gradp5 = append_stat(gradp5,mid_stat,28); gradp6 = append_stat(gradp6,mid_stat,29);\n",
    "\n",
    "    gradup1 = append_stat(gradup1,mid_stat,30); gradup2 = append_stat(gradup2,mid_stat,31); gradup3 = append_stat(gradup3,mid_stat,32);\n",
    "    gradup4 = append_stat(gradup4,mid_stat,33); gradup5 = append_stat(gradup5,mid_stat,34); gradup6 = append_stat(gradup6,mid_stat,35);\n",
    "\n",
    "    train_acc.append(avg_acc_train/(len(train_images)/batch_size))\n",
    "    test_acc .append(avg_acc_test / (len(test_images)/batch_size))\n",
    "    # ======================== extract stats ========================\n",
    "    \n",
    "    # ======================== save to image ========================\n",
    "    save_to_image(mid_stat[0:6]   ,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,\"layer\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[6:12]  ,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a,\"layera\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[12:18] ,weight1,weight2,weight3,weight4,weight5,weight6,\"weights\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[18:24] ,gradw1,gradw2,gradw3,gradw4,gradw5,gradw6,\"gradientw\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[24:30] ,gradp1,gradp2,gradp3,gradp4,gradp5,gradp6,\"gradientp\",train_acc,test_acc,current_exp_name,iter)\n",
    "    save_to_image(mid_stat[30:36] ,gradup1,gradup2,gradup3,gradup4,gradup5,gradup6,\"moment\",train_acc,test_acc,current_exp_name,iter)\n",
    "    # ======================== save to image ========================\n",
    "        \n",
    "    # ======================== print reset ========================\n",
    "    print(\"Current : \"+ str(iter) + \" Train Acc : \" + str(avg_acc_train/(len(train_images)/batch_size)) + \" Test Acc : \" + str(avg_acc_test/(len(test_images)/batch_size)) + '\\n')\n",
    "    avg_acc_train = 0 ; avg_acc_test  = 0\n",
    "    # ======================== print reset ========================\n",
    "\n",
    "np.save(current_exp_name+'/train_acc.npy',train_acc); np.save(current_exp_name+'/test_acc.npy', test_acc)    \n",
    "np.save(current_exp_name+'/llayer1.npy', llayer1);  np.save(current_exp_name+'/llayer2.npy', llayer2);  np.save(current_exp_name+'/llayer3.npy', llayer3); \n",
    "np.save(current_exp_name+'/llayer4.npy', llayer4);  np.save(current_exp_name+'/llayer5.npy', llayer5);  np.save(current_exp_name+'/llayer6.npy', llayer6); \n",
    "\n",
    "np.save(current_exp_name+'/llayer1a.npy', llayer1a);  np.save(current_exp_name+'/llayer2a.npy', llayer2a);  np.save(current_exp_name+'/llayer3a.npy', llayer3a); \n",
    "np.save(current_exp_name+'/llayer4a.npy', llayer4a);  np.save(current_exp_name+'/llayer5a.npy', llayer5a);  np.save(current_exp_name+'/llayer6a.npy', llayer6a); \n",
    "\n",
    "np.save(current_exp_name+'/weight1.npy', weight1);  np.save(current_exp_name+'/weight2.npy', weight2);  np.save(current_exp_name+'/weight3.npy', weight3);  \n",
    "np.save(current_exp_name+'/weight4.npy', weight4);  np.save(current_exp_name+'/weight5.npy', weight5);  np.save(current_exp_name+'/weight6.npy', weight6);  \n",
    "\n",
    "np.save(current_exp_name+'/gradw1.npy', gradw1); np.save(current_exp_name+'/gradw2.npy', gradw2); np.save(current_exp_name+'/gradw3.npy', gradw3);\n",
    "np.save(current_exp_name+'/gradw4.npy', gradw4); np.save(current_exp_name+'/gradw5.npy', gradw5); np.save(current_exp_name+'/gradw6.npy', gradw6);\n",
    "\n",
    "np.save(current_exp_name+'/gradp1.npy', gradp1); np.save(current_exp_name+'/gradp2.npy', gradp2); np.save(current_exp_name+'/gradp3.npy', gradp3);\n",
    "np.save(current_exp_name+'/gradp4.npy', gradp4); np.save(current_exp_name+'/gradp5.npy', gradp5); np.save(current_exp_name+'/gradp6.npy', gradp6);\n",
    "\n",
    "np.save(current_exp_name+'/gradup1.npy', gradup1); np.save(current_exp_name+'/gradup2.npy', gradup2); np.save(current_exp_name+'/gradup3.npy', gradup3);\n",
    "np.save(current_exp_name+'/gradup4.npy', gradup4); np.save(current_exp_name+'/gradup5.npy', gradup5); np.save(current_exp_name+'/gradup6.npy', gradup6);\n",
    "\n",
    "sess.close(); tf.reset_default_graph();\n",
    "\n",
    "%reset_selective -f l1,l2,l3,l4,l5,l6\n",
    "%reset_selective -f layer1,layer2,layer3,layer4,layer5,layer6\n",
    "%reset_selective -f layer1a,layer2a,layer3a,layer4a,layer5a,layer6a\n",
    "%reset_selective -f train_acc,test_acc,llayer1,llayer2,llayer3,llayer4,llayer5,llayer6,llayer1a,llayer2a,llayer3a,llayer4a,llayer5a,llayer6a\n",
    "%reset_selective -f weight1,weight2,weight3,weight4,weight5,weight6\n",
    "%reset_selective -f gradw1,gradw2,gradw3,gradw4,gradw5,gradw6\n",
    "%reset_selective -f gradp1,gradp2,gradp3,gradp4,gradp5,gradp6\n",
    "%reset_selective -f gradup1,gradup2,gradup3,gradup4,gradup5,gradup6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-05T10:32:13.164960Z",
     "start_time": "2019-01-05T10:32:12.622203Z"
    }
   },
   "outputs": [],
   "source": [
    "! start . "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}