homework/Day_021_HW.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 『本次練習內容』\n",
    "#### 使用Xception backbone做 Trnasfer Learning\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 『本次練習目的』\n",
    "  #### 了解如何使用Transfer Learning\n",
    "  #### 了解Transfer Learning的優點，可以觀察模型收斂速度"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 可以自行嘗試多種架構"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Convolution2D\n",
    "from keras.layers import MaxPooling2D\n",
    "from keras.layers import Flatten\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Dropout\n",
    "from keras.preprocessing import image\n",
    "from keras.models import Model\n",
    "from keras.layers import Dense, GlobalAveragePooling2D\n",
    "from keras import backend as K\n",
    "import keras\n",
    "from keras.layers import Input\n",
    " \n",
    "from keras.datasets import cifar10\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T06:14:29.689805Z",
     "start_time": "2020-01-17T06:14:23.629947Z"
    }
   },
   "outputs": [],
   "source": [
    "# use tensorflow 2.0\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Convolution2D\n",
    "from tensorflow.keras.layers import MaxPooling2D\n",
    "from tensorflow.keras.layers import Flatten\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.layers import Dropout\n",
    "from tensorflow.keras.preprocessing import image\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import Dense, GlobalAveragePooling2D\n",
    "from tensorflow.keras import backend as K\n",
    "import tensorflow.keras\n",
    "from tensorflow.keras.layers import Input\n",
    " \n",
    "from tensorflow.keras.datasets import cifar10\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from sklearn.preprocessing import OneHotEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T06:14:33.256125Z",
     "start_time": "2020-01-17T06:14:30.332525Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"xception\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 32, 32, 3)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv1 (Conv2D)           (None, 15, 15, 32)   864         input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv1_bn (BatchNormaliza (None, 15, 15, 32)   128         block1_conv1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv1_act (Activation)   (None, 15, 15, 32)   0           block1_conv1_bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv2 (Conv2D)           (None, 13, 13, 64)   18432       block1_conv1_act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv2_bn (BatchNormaliza (None, 13, 13, 64)   256         block1_conv2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv2_act (Activation)   (None, 13, 13, 64)   0           block1_conv2_bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block2_sepconv1 (SeparableConv2 (None, 13, 13, 128)  8768        block1_conv2_act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block2_sepconv1_bn (BatchNormal (None, 13, 13, 128)  512         block2_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block2_sepconv2_act (Activation (None, 13, 13, 128)  0           block2_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block2_sepconv2 (SeparableConv2 (None, 13, 13, 128)  17536       block2_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2_sepconv2_bn (BatchNormal (None, 13, 13, 128)  512         block2_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d (Conv2D)                 (None, 7, 7, 128)    8192        block1_conv2_act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block2_pool (MaxPooling2D)      (None, 7, 7, 128)    0           block2_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization (BatchNorma (None, 7, 7, 128)    512         conv2d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "add (Add)                       (None, 7, 7, 128)    0           block2_pool[0][0]                \n",
      "                                                                 batch_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv1_act (Activation (None, 7, 7, 128)    0           add[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv1 (SeparableConv2 (None, 7, 7, 256)    33920       block3_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv1_bn (BatchNormal (None, 7, 7, 256)    1024        block3_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv2_act (Activation (None, 7, 7, 256)    0           block3_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv2 (SeparableConv2 (None, 7, 7, 256)    67840       block3_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv2_bn (BatchNormal (None, 7, 7, 256)    1024        block3_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 4, 4, 256)    32768       add[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "block3_pool (MaxPooling2D)      (None, 4, 4, 256)    0           block3_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 4, 4, 256)    1024        conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 4, 4, 256)    0           block3_pool[0][0]                \n",
      "                                                                 batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv1_act (Activation (None, 4, 4, 256)    0           add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv1 (SeparableConv2 (None, 4, 4, 728)    188672      block4_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv1_bn (BatchNormal (None, 4, 4, 728)    2912        block4_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv2_act (Activation (None, 4, 4, 728)    0           block4_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv2 (SeparableConv2 (None, 4, 4, 728)    536536      block4_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv2_bn (BatchNormal (None, 4, 4, 728)    2912        block4_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 2, 2, 728)    186368      add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block4_pool (MaxPooling2D)      (None, 2, 2, 728)    0           block4_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 2, 2, 728)    2912        conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, 2, 2, 728)    0           block4_pool[0][0]                \n",
      "                                                                 batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv1_act (Activation (None, 2, 2, 728)    0           add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv1 (SeparableConv2 (None, 2, 2, 728)    536536      block5_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv1_bn (BatchNormal (None, 2, 2, 728)    2912        block5_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv2_act (Activation (None, 2, 2, 728)    0           block5_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv2 (SeparableConv2 (None, 2, 2, 728)    536536      block5_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv2_bn (BatchNormal (None, 2, 2, 728)    2912        block5_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv3_act (Activation (None, 2, 2, 728)    0           block5_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv3 (SeparableConv2 (None, 2, 2, 728)    536536      block5_sepconv3_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv3_bn (BatchNormal (None, 2, 2, 728)    2912        block5_sepconv3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "add_3 (Add)                     (None, 2, 2, 728)    0           block5_sepconv3_bn[0][0]         \n",
      "                                                                 add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv1_act (Activation (None, 2, 2, 728)    0           add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv1 (SeparableConv2 (None, 2, 2, 728)    536536      block6_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv1_bn (BatchNormal (None, 2, 2, 728)    2912        block6_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv2_act (Activation (None, 2, 2, 728)    0           block6_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv2 (SeparableConv2 (None, 2, 2, 728)    536536      block6_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv2_bn (BatchNormal (None, 2, 2, 728)    2912        block6_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv3_act (Activation (None, 2, 2, 728)    0           block6_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv3 (SeparableConv2 (None, 2, 2, 728)    536536      block6_sepconv3_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv3_bn (BatchNormal (None, 2, 2, 728)    2912        block6_sepconv3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, 2, 2, 728)    0           block6_sepconv3_bn[0][0]         \n",
      "                                                                 add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv1_act (Activation (None, 2, 2, 728)    0           add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv1 (SeparableConv2 (None, 2, 2, 728)    536536      block7_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv1_bn (BatchNormal (None, 2, 2, 728)    2912        block7_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv2_act (Activation (None, 2, 2, 728)    0           block7_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv2 (SeparableConv2 (None, 2, 2, 728)    536536      block7_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv2_bn (BatchNormal (None, 2, 2, 728)    2912        block7_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv3_act (Activation (None, 2, 2, 728)    0           block7_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv3 (SeparableConv2 (None, 2, 2, 728)    536536      block7_sepconv3_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv3_bn (BatchNormal (None, 2, 2, 728)    2912        block7_sepconv3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, 2, 2, 728)    0           block7_sepconv3_bn[0][0]         \n",
      "                                                                 add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv1_act (Activation (None, 2, 2, 728)    0           add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv1 (SeparableConv2 (None, 2, 2, 728)    536536      block8_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv1_bn (BatchNormal (None, 2, 2, 728)    2912        block8_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv2_act (Activation (None, 2, 2, 728)    0           block8_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv2 (SeparableConv2 (None, 2, 2, 728)    536536      block8_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv2_bn (BatchNormal (None, 2, 2, 728)    2912        block8_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv3_act (Activation (None, 2, 2, 728)    0           block8_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv3 (SeparableConv2 (None, 2, 2, 728)    536536      block8_sepconv3_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv3_bn (BatchNormal (None, 2, 2, 728)    2912        block8_sepconv3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "add_6 (Add)                     (None, 2, 2, 728)    0           block8_sepconv3_bn[0][0]         \n",
      "                                                                 add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv1_act (Activation (None, 2, 2, 728)    0           add_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv1 (SeparableConv2 (None, 2, 2, 728)    536536      block9_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv1_bn (BatchNormal (None, 2, 2, 728)    2912        block9_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv2_act (Activation (None, 2, 2, 728)    0           block9_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv2 (SeparableConv2 (None, 2, 2, 728)    536536      block9_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv2_bn (BatchNormal (None, 2, 2, 728)    2912        block9_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv3_act (Activation (None, 2, 2, 728)    0           block9_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv3 (SeparableConv2 (None, 2, 2, 728)    536536      block9_sepconv3_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv3_bn (BatchNormal (None, 2, 2, 728)    2912        block9_sepconv3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "add_7 (Add)                     (None, 2, 2, 728)    0           block9_sepconv3_bn[0][0]         \n",
      "                                                                 add_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv1_act (Activatio (None, 2, 2, 728)    0           add_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv1 (SeparableConv (None, 2, 2, 728)    536536      block10_sepconv1_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv1_bn (BatchNorma (None, 2, 2, 728)    2912        block10_sepconv1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv2_act (Activatio (None, 2, 2, 728)    0           block10_sepconv1_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv2 (SeparableConv (None, 2, 2, 728)    536536      block10_sepconv2_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv2_bn (BatchNorma (None, 2, 2, 728)    2912        block10_sepconv2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv3_act (Activatio (None, 2, 2, 728)    0           block10_sepconv2_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv3 (SeparableConv (None, 2, 2, 728)    536536      block10_sepconv3_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv3_bn (BatchNorma (None, 2, 2, 728)    2912        block10_sepconv3[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "add_8 (Add)                     (None, 2, 2, 728)    0           block10_sepconv3_bn[0][0]        \n",
      "                                                                 add_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv1_act (Activatio (None, 2, 2, 728)    0           add_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv1 (SeparableConv (None, 2, 2, 728)    536536      block11_sepconv1_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv1_bn (BatchNorma (None, 2, 2, 728)    2912        block11_sepconv1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv2_act (Activatio (None, 2, 2, 728)    0           block11_sepconv1_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv2 (SeparableConv (None, 2, 2, 728)    536536      block11_sepconv2_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv2_bn (BatchNorma (None, 2, 2, 728)    2912        block11_sepconv2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv3_act (Activatio (None, 2, 2, 728)    0           block11_sepconv2_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv3 (SeparableConv (None, 2, 2, 728)    536536      block11_sepconv3_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv3_bn (BatchNorma (None, 2, 2, 728)    2912        block11_sepconv3[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "add_9 (Add)                     (None, 2, 2, 728)    0           block11_sepconv3_bn[0][0]        \n",
      "                                                                 add_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv1_act (Activatio (None, 2, 2, 728)    0           add_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv1 (SeparableConv (None, 2, 2, 728)    536536      block12_sepconv1_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv1_bn (BatchNorma (None, 2, 2, 728)    2912        block12_sepconv1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv2_act (Activatio (None, 2, 2, 728)    0           block12_sepconv1_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv2 (SeparableConv (None, 2, 2, 728)    536536      block12_sepconv2_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv2_bn (BatchNorma (None, 2, 2, 728)    2912        block12_sepconv2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv3_act (Activatio (None, 2, 2, 728)    0           block12_sepconv2_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv3 (SeparableConv (None, 2, 2, 728)    536536      block12_sepconv3_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv3_bn (BatchNorma (None, 2, 2, 728)    2912        block12_sepconv3[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "add_10 (Add)                    (None, 2, 2, 728)    0           block12_sepconv3_bn[0][0]        \n",
      "                                                                 add_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv1_act (Activatio (None, 2, 2, 728)    0           add_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv1 (SeparableConv (None, 2, 2, 728)    536536      block13_sepconv1_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv1_bn (BatchNorma (None, 2, 2, 728)    2912        block13_sepconv1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv2_act (Activatio (None, 2, 2, 728)    0           block13_sepconv1_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv2 (SeparableConv (None, 2, 2, 1024)   752024      block13_sepconv2_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv2_bn (BatchNorma (None, 2, 2, 1024)   4096        block13_sepconv2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 1, 1, 1024)   745472      add_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block13_pool (MaxPooling2D)     (None, 1, 1, 1024)   0           block13_sepconv2_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 1, 1, 1024)   4096        conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_11 (Add)                    (None, 1, 1, 1024)   0           block13_pool[0][0]               \n",
      "                                                                 batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv1 (SeparableConv (None, 1, 1, 1536)   1582080     add_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv1_bn (BatchNorma (None, 1, 1, 1536)   6144        block14_sepconv1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv1_act (Activatio (None, 1, 1, 1536)   0           block14_sepconv1_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv2 (SeparableConv (None, 1, 1, 2048)   3159552     block14_sepconv1_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv2_bn (BatchNorma (None, 1, 1, 2048)   8192        block14_sepconv2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv2_act (Activatio (None, 1, 1, 2048)   0           block14_sepconv2_bn[0][0]        \n",
      "==================================================================================================\n",
      "Total params: 20,861,480\n",
      "Trainable params: 20,806,952\n",
      "Non-trainable params: 54,528\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "input_tensor = Input(shape = (32, 32, 3))\n",
    "#include top 決定要不要加入 fully Connected Layer\n",
    "\n",
    "'''Xception 架構'''\n",
    "model = tensorflow.keras.applications.Xception(include_top = False, weights = 'imagenet', \n",
    "                                             input_tensor = input_tensor,\n",
    "                                             pooling = None, classes = 10)\n",
    "\n",
    "'''Resnet 50 架構'''\n",
    "#model=keras.applications.ResNet50(include_top = False, weights = 'imagenet',\n",
    "                                    #input_tensor = input_tensor,\n",
    "                                    #pooling = None, classes = 10)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 添加層數"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T06:15:29.816305Z",
     "start_time": "2020-01-17T06:15:29.726432Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model深度： 138\n"
     ]
    }
   ],
   "source": [
    "x = model.output\n",
    "\n",
    "'''可以參考Cifar10實作章節'''\n",
    "x = Convolution2D(32, (1, 1) , activation = 'relu')(x)\n",
    "x = MaxPooling2D(pool_size = (1, 1))(x)\n",
    "x = Convolution2D(64, (1,1) , activation = 'relu')(x)\n",
    "x = Flatten()(x)\n",
    "x = Dense(units = 100, activation = 'relu')(x)\n",
    "\n",
    "#predictions = Dense(output_dim = 10, activation = 'softmax')(x)\n",
    "predictions = Dense(10, activation = 'softmax')(x)\n",
    "model = Model(inputs = model.input, outputs = predictions)\n",
    "print('Model深度：', len(model.layers))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 鎖定特定幾層不要更新權重"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T06:15:33.856517Z",
     "start_time": "2020-01-17T06:15:33.839441Z"
    }
   },
   "outputs": [],
   "source": [
    "for layer in model.layers[:100]:\n",
    "    layer.trainable = False\n",
    "    \n",
    "for layer in model.layers[100:]:\n",
    "    layer.trainable = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 準備 Cifar 10 資料"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T06:15:40.080907Z",
     "start_time": "2020-01-17T06:15:35.164867Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 32, 32, 3)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jasonch/anaconda3/envs/tf_cpu/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:371: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.\n",
      "If you want the future behaviour and silence this warning, you can specify \"categories='auto'\".\n",
      "In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "\n",
    "print(x_train.shape)      #(50000, 32, 32, 3)\n",
    "\n",
    "## Normalize Data\n",
    "def normalize(X_train,X_test):\n",
    "        mean = np.mean(X_train,axis = (0, 1, 2, 3))\n",
    "        std = np.std(X_train, axis = (0, 1, 2, 3))\n",
    "        X_train = (X_train - mean) / (std + 1e-7)\n",
    "        X_test = (X_test - mean) / (std + 1e-7)\n",
    "        \n",
    "        return X_train, X_test\n",
    "    \n",
    "    \n",
    "## Normalize Training and Testset    \n",
    "x_train, x_test = normalize(x_train, x_test) \n",
    "\n",
    "## OneHot Label 由(None, 1)-(None, 10)\n",
    "## ex. label=2,變成[0,0,1,0,0,0,0,0,0,0]\n",
    "one_hot = OneHotEncoder()\n",
    "y_train = one_hot.fit_transform(y_train).toarray()\n",
    "y_test = one_hot.transform(y_test).toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T06:52:15.480601Z",
     "start_time": "2020-01-17T06:15:41.927392Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 50000 samples\n",
      "Epoch 1/100\n",
      "50000/50000 [==============================] - 1134s 23ms/sample - loss: 1.4347 - accuracy: 0.4937\n",
      "Epoch 2/100\n",
      "40896/50000 [=======================>......] - ETA: 3:54 - loss: 1.0729 - accuracy: 0.6349"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-8cce2993b286>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# compile the model (should be done *after* setting layers to non-trainable)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'adam'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'categorical_crossentropy'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetrics\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mtrain_history\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m100\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/tf_cpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0mmax_queue_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_queue_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    727\u001b[0m         \u001b[0mworkers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mworkers\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 728\u001b[0;31m         use_multiprocessing=use_multiprocessing)\n\u001b[0m\u001b[1;32m    729\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    730\u001b[0m   def evaluate(self,\n",
      "\u001b[0;32m~/anaconda3/envs/tf_cpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, model, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, **kwargs)\u001b[0m\n\u001b[1;32m    322\u001b[0m                 \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    323\u001b[0m                 \u001b[0mtraining_context\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtraining_context\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 324\u001b[0;31m                 total_epochs=epochs)\n\u001b[0m\u001b[1;32m    325\u001b[0m             \u001b[0mcbks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake_logs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch_logs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining_result\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    326\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tf_cpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2.py\u001b[0m in \u001b[0;36mrun_one_epoch\u001b[0;34m(model, iterator, execution_function, dataset_size, batch_size, strategy, steps_per_epoch, num_samples, mode, training_context, total_epochs)\u001b[0m\n\u001b[1;32m    121\u001b[0m         step=step, mode=mode, size=current_batch_size) as batch_logs:\n\u001b[1;32m    122\u001b[0m       \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 123\u001b[0;31m         \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexecution_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    124\u001b[0m       \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mStopIteration\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOutOfRangeError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    125\u001b[0m         \u001b[0;31m# TODO(kaftan): File bug about tf function and errors.OutOfRangeError?\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tf_cpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py\u001b[0m in \u001b[0;36mexecution_function\u001b[0;34m(input_fn)\u001b[0m\n\u001b[1;32m     84\u001b[0m     \u001b[0;31m# `numpy` translates Tensors to values in Eager mode.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m     return nest.map_structure(_non_none_constant_value,\n\u001b[0;32m---> 86\u001b[0;31m                               distributed_function(input_fn))\n\u001b[0m\u001b[1;32m     87\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mexecution_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tf_cpu/lib/python3.6/site-packages/tensorflow_core/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    455\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    456\u001b[0m     \u001b[0mtracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 457\u001b[0;31m     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    458\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mtracing_count\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    459\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_counter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcalled_without_tracing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tf_cpu/lib/python3.6/site-packages/tensorflow_core/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    485\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    486\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 487\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    488\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    489\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tf_cpu/lib/python3.6/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1821\u001b[0m     \u001b[0;34m\"\"\"Calls a graph function specialized to the inputs.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1822\u001b[0m     \u001b[0mgraph_function\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1823\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1824\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1825\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tf_cpu/lib/python3.6/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[0;34m(self, args, kwargs)\u001b[0m\n\u001b[1;32m   1139\u001b[0m          if isinstance(t, (ops.Tensor,\n\u001b[1;32m   1140\u001b[0m                            resource_variable_ops.BaseResourceVariable))),\n\u001b[0;32m-> 1141\u001b[0;31m         self.captured_inputs)\n\u001b[0m\u001b[1;32m   1142\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1143\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_flat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tf_cpu/lib/python3.6/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1222\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mexecuting_eagerly\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1223\u001b[0m       flat_outputs = forward_function.call(\n\u001b[0;32m-> 1224\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager)\n\u001b[0m\u001b[1;32m   1225\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1226\u001b[0m       \u001b[0mgradient_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_delayed_rewrite_functions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mregister\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tf_cpu/lib/python3.6/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    509\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    510\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"executor_type\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexecutor_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"config_proto\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 511\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    512\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    513\u001b[0m           outputs = execute.execute_with_cancellation(\n",
      "\u001b[0;32m~/anaconda3/envs/tf_cpu/lib/python3.6/site-packages/tensorflow_core/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tensorflow.TFE_Py_Execute(ctx._handle, device_name,\n\u001b[1;32m     60\u001b[0m                                                \u001b[0mop_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m                                                num_outputs)\n\u001b[0m\u001b[1;32m     62\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# compile the model (should be done *after* setting layers to non-trainable)\n",
    "model.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy'])\n",
    "model.fit(x_train, y_train, batch_size = 32, epochs = 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}