每日作業/Day_021_HW.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 『本次練習內容』\n",
    "#### 使用Xception backbone做 Trnasfer Learning\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 『本次練習目的』\n",
    "  #### 了解如何使用Transfer Learning\n",
    "  #### 了解Transfer Learning的優點，可以觀察模型收斂速度"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 可以自行嘗試多種架構"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.datasets import cifar10\n",
    "from tensorflow.keras.preprocessing import image\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import backend as K\n",
    "from tensorflow.keras.datasets import cifar10\n",
    "from tensorflow.keras.models import Sequential, Model\n",
    "from tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, GlobalMaxPooling2D, GlobalAveragePooling2D, BatchNormalization, Flatten, Dense, Activation, Dropout"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cifar 10 資料( 參考 [Day_015_HW](https://github.com/YenLinWu/DL_CVMarathon/blob/master/%E6%AF%8F%E6%97%A5%E4%BD%9C%E6%A5%AD/Day_015_HW.ipynb) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of Training Data = (50000, 32, 32, 3)\n"
     ]
    }
   ],
   "source": [
    "( x_train, y_train ), ( x_test, y_test ) = cifar10.load_data()\n",
    "\n",
    "print( f'Shape of Training Data = {x_train.shape}' ) \n",
    "\n",
    "# Normalize Data\n",
    "def normalize( X_train, X_test ):\n",
    "    mean = np.mean( X_train, axis=(0,1,2,3) )\n",
    "    std = np.std( X_train, axis=(0,1,2,3) )\n",
    "    X_train = (X_train-mean)/(std+1e-7)\n",
    "    X_test = (X_test-mean)/(std+1e-7) \n",
    "    return X_train, X_test, mean, std\n",
    "    \n",
    "# Normalize Training and Testset    \n",
    "x_train, x_test, mean_train, std_train = normalize( x_train, x_test ) \n",
    "\n",
    "# OneHot Label\n",
    "one_hot = OneHotEncoder()\n",
    "y_train = one_hot.fit_transform( y_train ).toarray()\n",
    "y_test = one_hot.transform( y_test ).toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Xception 遷移式學習"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.applications.xception import Xception\n",
    "\n",
    "input_tensor = Input( shape=(32,32,3) )\n",
    "\n",
    "Base_Model = Xception( include_top=False, weights='imagenet', input_tensor=None, input_shape=None, pooling=None, classes=1000 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Numbers of Layer of Transfer Model =  136\n",
      "Model: \"Transfer_Model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            [(None, None, None,  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv1 (Conv2D)           (None, None, None, 3 864         input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv1_bn (BatchNormaliza (None, None, None, 3 128         block1_conv1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv1_act (Activation)   (None, None, None, 3 0           block1_conv1_bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv2 (Conv2D)           (None, None, None, 6 18432       block1_conv1_act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv2_bn (BatchNormaliza (None, None, None, 6 256         block1_conv2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv2_act (Activation)   (None, None, None, 6 0           block1_conv2_bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block2_sepconv1 (SeparableConv2 (None, None, None, 1 8768        block1_conv2_act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block2_sepconv1_bn (BatchNormal (None, None, None, 1 512         block2_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block2_sepconv2_act (Activation (None, None, None, 1 0           block2_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block2_sepconv2 (SeparableConv2 (None, None, None, 1 17536       block2_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2_sepconv2_bn (BatchNormal (None, None, None, 1 512         block2_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d (Conv2D)                 (None, None, None, 1 8192        block1_conv2_act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block2_pool (MaxPooling2D)      (None, None, None, 1 0           block2_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization (BatchNorma (None, None, None, 1 512         conv2d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "add (Add)                       (None, None, None, 1 0           block2_pool[0][0]                \n",
      "                                                                 batch_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv1_act (Activation (None, None, None, 1 0           add[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv1 (SeparableConv2 (None, None, None, 2 33920       block3_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv1_bn (BatchNormal (None, None, None, 2 1024        block3_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv2_act (Activation (None, None, None, 2 0           block3_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv2 (SeparableConv2 (None, None, None, 2 67840       block3_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv2_bn (BatchNormal (None, None, None, 2 1024        block3_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, None, None, 2 32768       add[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "block3_pool (MaxPooling2D)      (None, None, None, 2 0           block3_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, None, None, 2 1024        conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, None, None, 2 0           block3_pool[0][0]                \n",
      "                                                                 batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv1_act (Activation (None, None, None, 2 0           add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv1 (SeparableConv2 (None, None, None, 7 188672      block4_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv1_bn (BatchNormal (None, None, None, 7 2912        block4_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv2_act (Activation (None, None, None, 7 0           block4_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv2 (SeparableConv2 (None, None, None, 7 536536      block4_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv2_bn (BatchNormal (None, None, None, 7 2912        block4_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, None, None, 7 186368      add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block4_pool (MaxPooling2D)      (None, None, None, 7 0           block4_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, None, None, 7 2912        conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, None, None, 7 0           block4_pool[0][0]                \n",
      "                                                                 batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv1_act (Activation (None, None, None, 7 0           add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv1 (SeparableConv2 (None, None, None, 7 536536      block5_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv1_bn (BatchNormal (None, None, None, 7 2912        block5_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv2_act (Activation (None, None, None, 7 0           block5_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv2 (SeparableConv2 (None, None, None, 7 536536      block5_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv2_bn (BatchNormal (None, None, None, 7 2912        block5_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv3_act (Activation (None, None, None, 7 0           block5_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv3 (SeparableConv2 (None, None, None, 7 536536      block5_sepconv3_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv3_bn (BatchNormal (None, None, None, 7 2912        block5_sepconv3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "add_3 (Add)                     (None, None, None, 7 0           block5_sepconv3_bn[0][0]         \n",
      "                                                                 add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv1_act (Activation (None, None, None, 7 0           add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv1 (SeparableConv2 (None, None, None, 7 536536      block6_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv1_bn (BatchNormal (None, None, None, 7 2912        block6_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv2_act (Activation (None, None, None, 7 0           block6_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv2 (SeparableConv2 (None, None, None, 7 536536      block6_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv2_bn (BatchNormal (None, None, None, 7 2912        block6_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv3_act (Activation (None, None, None, 7 0           block6_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv3 (SeparableConv2 (None, None, None, 7 536536      block6_sepconv3_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv3_bn (BatchNormal (None, None, None, 7 2912        block6_sepconv3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, None, None, 7 0           block6_sepconv3_bn[0][0]         \n",
      "                                                                 add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv1_act (Activation (None, None, None, 7 0           add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv1 (SeparableConv2 (None, None, None, 7 536536      block7_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv1_bn (BatchNormal (None, None, None, 7 2912        block7_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv2_act (Activation (None, None, None, 7 0           block7_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv2 (SeparableConv2 (None, None, None, 7 536536      block7_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv2_bn (BatchNormal (None, None, None, 7 2912        block7_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv3_act (Activation (None, None, None, 7 0           block7_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv3 (SeparableConv2 (None, None, None, 7 536536      block7_sepconv3_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv3_bn (BatchNormal (None, None, None, 7 2912        block7_sepconv3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, None, None, 7 0           block7_sepconv3_bn[0][0]         \n",
      "                                                                 add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv1_act (Activation (None, None, None, 7 0           add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv1 (SeparableConv2 (None, None, None, 7 536536      block8_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv1_bn (BatchNormal (None, None, None, 7 2912        block8_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv2_act (Activation (None, None, None, 7 0           block8_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv2 (SeparableConv2 (None, None, None, 7 536536      block8_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv2_bn (BatchNormal (None, None, None, 7 2912        block8_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv3_act (Activation (None, None, None, 7 0           block8_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv3 (SeparableConv2 (None, None, None, 7 536536      block8_sepconv3_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv3_bn (BatchNormal (None, None, None, 7 2912        block8_sepconv3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "add_6 (Add)                     (None, None, None, 7 0           block8_sepconv3_bn[0][0]         \n",
      "                                                                 add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv1_act (Activation (None, None, None, 7 0           add_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv1 (SeparableConv2 (None, None, None, 7 536536      block9_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv1_bn (BatchNormal (None, None, None, 7 2912        block9_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv2_act (Activation (None, None, None, 7 0           block9_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv2 (SeparableConv2 (None, None, None, 7 536536      block9_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv2_bn (BatchNormal (None, None, None, 7 2912        block9_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv3_act (Activation (None, None, None, 7 0           block9_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv3 (SeparableConv2 (None, None, None, 7 536536      block9_sepconv3_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv3_bn (BatchNormal (None, None, None, 7 2912        block9_sepconv3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "add_7 (Add)                     (None, None, None, 7 0           block9_sepconv3_bn[0][0]         \n",
      "                                                                 add_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv1_act (Activatio (None, None, None, 7 0           add_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv1 (SeparableConv (None, None, None, 7 536536      block10_sepconv1_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv1_bn (BatchNorma (None, None, None, 7 2912        block10_sepconv1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv2_act (Activatio (None, None, None, 7 0           block10_sepconv1_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv2 (SeparableConv (None, None, None, 7 536536      block10_sepconv2_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv2_bn (BatchNorma (None, None, None, 7 2912        block10_sepconv2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv3_act (Activatio (None, None, None, 7 0           block10_sepconv2_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv3 (SeparableConv (None, None, None, 7 536536      block10_sepconv3_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv3_bn (BatchNorma (None, None, None, 7 2912        block10_sepconv3[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "add_8 (Add)                     (None, None, None, 7 0           block10_sepconv3_bn[0][0]        \n",
      "                                                                 add_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv1_act (Activatio (None, None, None, 7 0           add_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv1 (SeparableConv (None, None, None, 7 536536      block11_sepconv1_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv1_bn (BatchNorma (None, None, None, 7 2912        block11_sepconv1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv2_act (Activatio (None, None, None, 7 0           block11_sepconv1_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv2 (SeparableConv (None, None, None, 7 536536      block11_sepconv2_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv2_bn (BatchNorma (None, None, None, 7 2912        block11_sepconv2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv3_act (Activatio (None, None, None, 7 0           block11_sepconv2_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv3 (SeparableConv (None, None, None, 7 536536      block11_sepconv3_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv3_bn (BatchNorma (None, None, None, 7 2912        block11_sepconv3[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "add_9 (Add)                     (None, None, None, 7 0           block11_sepconv3_bn[0][0]        \n",
      "                                                                 add_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv1_act (Activatio (None, None, None, 7 0           add_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv1 (SeparableConv (None, None, None, 7 536536      block12_sepconv1_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv1_bn (BatchNorma (None, None, None, 7 2912        block12_sepconv1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv2_act (Activatio (None, None, None, 7 0           block12_sepconv1_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv2 (SeparableConv (None, None, None, 7 536536      block12_sepconv2_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv2_bn (BatchNorma (None, None, None, 7 2912        block12_sepconv2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv3_act (Activatio (None, None, None, 7 0           block12_sepconv2_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv3 (SeparableConv (None, None, None, 7 536536      block12_sepconv3_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv3_bn (BatchNorma (None, None, None, 7 2912        block12_sepconv3[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "add_10 (Add)                    (None, None, None, 7 0           block12_sepconv3_bn[0][0]        \n",
      "                                                                 add_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv1_act (Activatio (None, None, None, 7 0           add_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv1 (SeparableConv (None, None, None, 7 536536      block13_sepconv1_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv1_bn (BatchNorma (None, None, None, 7 2912        block13_sepconv1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv2_act (Activatio (None, None, None, 7 0           block13_sepconv1_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv2 (SeparableConv (None, None, None, 1 752024      block13_sepconv2_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv2_bn (BatchNorma (None, None, None, 1 4096        block13_sepconv2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, None, None, 1 745472      add_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block13_pool (MaxPooling2D)     (None, None, None, 1 0           block13_sepconv2_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, None, None, 1 4096        conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_11 (Add)                    (None, None, None, 1 0           block13_pool[0][0]               \n",
      "                                                                 batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv1 (SeparableConv (None, None, None, 1 1582080     add_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv1_bn (BatchNorma (None, None, None, 1 6144        block14_sepconv1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv1_act (Activatio (None, None, None, 1 0           block14_sepconv1_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv2 (SeparableConv (None, None, None, 2 3159552     block14_sepconv1_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv2_bn (BatchNorma (None, None, None, 2 8192        block14_sepconv2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv2_act (Activatio (None, None, None, 2 0           block14_sepconv2_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d (Globa (None, 2048)         0           block14_sepconv2_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "Dense (Dense)                   (None, 128)          262272      global_average_pooling2d[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "BN (BatchNormalization)         (None, 128)          512         Dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "Softmax (Dense)                 (None, 10)           1290        BN[0][0]                         \n",
      "==================================================================================================\n",
      "Total params: 21,125,554\n",
      "Trainable params: 21,070,770\n",
      "Non-trainable params: 54,784\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "x = Base_Model.output\n",
    "x = GlobalAveragePooling2D()(x)\n",
    "x = Dense( 128, activation='relu', name='Dense' )(x)\n",
    "x = BatchNormalization( name='BN' )(x)\n",
    "Output_Layer = Dense( 10, activation='softmax', name='Softmax' )(x)\n",
    "Transfer_Model = Model( inputs=Base_Model.input, outputs=Output_Layer, name='Transfer_Model' )\n",
    "\n",
    "print( 'Numbers of Layer of Transfer Model = ', len(Transfer_Model.layers) )\n",
    "Transfer_Model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 鎖定特定幾層不要更新權重"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 input_2 False\n",
      "1 block1_conv1 False\n",
      "2 block1_conv1_bn False\n",
      "3 block1_conv1_act False\n",
      "4 block1_conv2 False\n",
      "5 block1_conv2_bn False\n",
      "6 block1_conv2_act False\n",
      "7 block2_sepconv1 False\n",
      "8 block2_sepconv1_bn False\n",
      "9 block2_sepconv2_act False\n",
      "10 block2_sepconv2 False\n",
      "11 block2_sepconv2_bn False\n",
      "12 conv2d False\n",
      "13 block2_pool False\n",
      "14 batch_normalization False\n",
      "15 add False\n",
      "16 block3_sepconv1_act False\n",
      "17 block3_sepconv1 False\n",
      "18 block3_sepconv1_bn False\n",
      "19 block3_sepconv2_act False\n",
      "20 block3_sepconv2 False\n",
      "21 block3_sepconv2_bn False\n",
      "22 conv2d_1 False\n",
      "23 block3_pool False\n",
      "24 batch_normalization_1 False\n",
      "25 add_1 False\n",
      "26 block4_sepconv1_act False\n",
      "27 block4_sepconv1 False\n",
      "28 block4_sepconv1_bn False\n",
      "29 block4_sepconv2_act False\n",
      "30 block4_sepconv2 False\n",
      "31 block4_sepconv2_bn False\n",
      "32 conv2d_2 False\n",
      "33 block4_pool False\n",
      "34 batch_normalization_2 False\n",
      "35 add_2 False\n",
      "36 block5_sepconv1_act False\n",
      "37 block5_sepconv1 False\n",
      "38 block5_sepconv1_bn False\n",
      "39 block5_sepconv2_act False\n",
      "40 block5_sepconv2 False\n",
      "41 block5_sepconv2_bn False\n",
      "42 block5_sepconv3_act False\n",
      "43 block5_sepconv3 False\n",
      "44 block5_sepconv3_bn False\n",
      "45 add_3 False\n",
      "46 block6_sepconv1_act False\n",
      "47 block6_sepconv1 False\n",
      "48 block6_sepconv1_bn False\n",
      "49 block6_sepconv2_act False\n",
      "50 block6_sepconv2 False\n",
      "51 block6_sepconv2_bn False\n",
      "52 block6_sepconv3_act False\n",
      "53 block6_sepconv3 False\n",
      "54 block6_sepconv3_bn False\n",
      "55 add_4 False\n",
      "56 block7_sepconv1_act False\n",
      "57 block7_sepconv1 False\n",
      "58 block7_sepconv1_bn False\n",
      "59 block7_sepconv2_act False\n",
      "60 block7_sepconv2 False\n",
      "61 block7_sepconv2_bn False\n",
      "62 block7_sepconv3_act False\n",
      "63 block7_sepconv3 False\n",
      "64 block7_sepconv3_bn False\n",
      "65 add_5 False\n",
      "66 block8_sepconv1_act False\n",
      "67 block8_sepconv1 False\n",
      "68 block8_sepconv1_bn False\n",
      "69 block8_sepconv2_act False\n",
      "70 block8_sepconv2 False\n",
      "71 block8_sepconv2_bn False\n",
      "72 block8_sepconv3_act False\n",
      "73 block8_sepconv3 False\n",
      "74 block8_sepconv3_bn False\n",
      "75 add_6 False\n",
      "76 block9_sepconv1_act False\n",
      "77 block9_sepconv1 False\n",
      "78 block9_sepconv1_bn False\n",
      "79 block9_sepconv2_act False\n",
      "80 block9_sepconv2 False\n",
      "81 block9_sepconv2_bn False\n",
      "82 block9_sepconv3_act False\n",
      "83 block9_sepconv3 False\n",
      "84 block9_sepconv3_bn False\n",
      "85 add_7 False\n",
      "86 block10_sepconv1_act False\n",
      "87 block10_sepconv1 False\n",
      "88 block10_sepconv1_bn False\n",
      "89 block10_sepconv2_act False\n",
      "90 block10_sepconv2 False\n",
      "91 block10_sepconv2_bn False\n",
      "92 block10_sepconv3_act False\n",
      "93 block10_sepconv3 False\n",
      "94 block10_sepconv3_bn False\n",
      "95 add_8 False\n",
      "96 block11_sepconv1_act False\n",
      "97 block11_sepconv1 False\n",
      "98 block11_sepconv1_bn False\n",
      "99 block11_sepconv2_act False\n",
      "100 block11_sepconv2 True\n",
      "101 block11_sepconv2_bn True\n",
      "102 block11_sepconv3_act True\n",
      "103 block11_sepconv3 True\n",
      "104 block11_sepconv3_bn True\n",
      "105 add_9 True\n",
      "106 block12_sepconv1_act True\n",
      "107 block12_sepconv1 True\n",
      "108 block12_sepconv1_bn True\n",
      "109 block12_sepconv2_act True\n",
      "110 block12_sepconv2 True\n",
      "111 block12_sepconv2_bn True\n",
      "112 block12_sepconv3_act True\n",
      "113 block12_sepconv3 True\n",
      "114 block12_sepconv3_bn True\n",
      "115 add_10 True\n",
      "116 block13_sepconv1_act True\n",
      "117 block13_sepconv1 True\n",
      "118 block13_sepconv1_bn True\n",
      "119 block13_sepconv2_act True\n",
      "120 block13_sepconv2 True\n",
      "121 block13_sepconv2_bn True\n",
      "122 conv2d_3 True\n",
      "123 block13_pool True\n",
      "124 batch_normalization_3 True\n",
      "125 add_11 True\n",
      "126 block14_sepconv1 True\n",
      "127 block14_sepconv1_bn True\n",
      "128 block14_sepconv1_act True\n",
      "129 block14_sepconv2 True\n",
      "130 block14_sepconv2_bn True\n",
      "131 block14_sepconv2_act True\n",
      "132 global_average_pooling2d True\n",
      "133 Dense True\n",
      "134 BN True\n",
      "135 Softmax True\n"
     ]
    }
   ],
   "source": [
    "for layer in Transfer_Model.layers[:100]:\n",
    "    layer.trainable = False\n",
    "for layer in Transfer_Model.layers[100:]:\n",
    "    layer.trainable = True\n",
    "\n",
    "for i, layer in enumerate( Transfer_Model.layers ):\n",
    "    print( i, layer.name, layer.trainable )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 訓練模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 50000 samples\n",
      "Epoch 1/30\n",
      "50000/50000 [==============================] - 36s 718us/sample - loss: 1.5946 - accuracy: 0.4242\n",
      "Epoch 2/30\n",
      "50000/50000 [==============================] - 30s 590us/sample - loss: 1.0840 - accuracy: 0.6280\n",
      "Epoch 3/30\n",
      "50000/50000 [==============================] - 30s 591us/sample - loss: 0.9423 - accuracy: 0.6788\n",
      "Epoch 4/30\n",
      "50000/50000 [==============================] - 30s 594us/sample - loss: 0.8493 - accuracy: 0.7069\n",
      "Epoch 5/30\n",
      "50000/50000 [==============================] - 30s 592us/sample - loss: 0.7655 - accuracy: 0.7375\n",
      "Epoch 6/30\n",
      "50000/50000 [==============================] - 30s 592us/sample - loss: 0.6875 - accuracy: 0.7629\n",
      "Epoch 7/30\n",
      "50000/50000 [==============================] - 30s 603us/sample - loss: 0.6114 - accuracy: 0.7888\n",
      "Epoch 8/30\n",
      "50000/50000 [==============================] - 30s 594us/sample - loss: 0.5421 - accuracy: 0.8122\n",
      "Epoch 9/30\n",
      "50000/50000 [==============================] - 30s 595us/sample - loss: 0.4821 - accuracy: 0.8329\n",
      "Epoch 10/30\n",
      "50000/50000 [==============================] - 30s 595us/sample - loss: 0.4314 - accuracy: 0.8501\n",
      "Epoch 11/30\n",
      "50000/50000 [==============================] - 30s 596us/sample - loss: 0.3835 - accuracy: 0.8671\n",
      "Epoch 12/30\n",
      "50000/50000 [==============================] - 30s 595us/sample - loss: 0.3518 - accuracy: 0.8782\n",
      "Epoch 13/30\n",
      "50000/50000 [==============================] - 30s 594us/sample - loss: 0.3138 - accuracy: 0.8913\n",
      "Epoch 14/30\n",
      "50000/50000 [==============================] - 30s 593us/sample - loss: 0.2874 - accuracy: 0.9005\n",
      "Epoch 15/30\n",
      "50000/50000 [==============================] - 30s 593us/sample - loss: 0.2697 - accuracy: 0.9075\n",
      "Epoch 16/30\n",
      "50000/50000 [==============================] - 30s 594us/sample - loss: 0.2521 - accuracy: 0.9128\n",
      "Epoch 17/30\n",
      "50000/50000 [==============================] - 30s 594us/sample - loss: 0.2389 - accuracy: 0.9176\n",
      "Epoch 18/30\n",
      "50000/50000 [==============================] - 30s 600us/sample - loss: 0.2212 - accuracy: 0.9235\n",
      "Epoch 19/30\n",
      "50000/50000 [==============================] - 30s 597us/sample - loss: 0.2100 - accuracy: 0.9267\n",
      "Epoch 20/30\n",
      "50000/50000 [==============================] - 30s 598us/sample - loss: 0.1996 - accuracy: 0.9317\n",
      "Epoch 21/30\n",
      "50000/50000 [==============================] - 30s 599us/sample - loss: 0.1884 - accuracy: 0.9358\n",
      "Epoch 22/30\n",
      "50000/50000 [==============================] - 30s 597us/sample - loss: 0.1865 - accuracy: 0.9358\n",
      "Epoch 23/30\n",
      "50000/50000 [==============================] - 30s 595us/sample - loss: 0.1762 - accuracy: 0.9403\n",
      "Epoch 24/30\n",
      "50000/50000 [==============================] - 30s 597us/sample - loss: 0.1703 - accuracy: 0.9417\n",
      "Epoch 25/30\n",
      "50000/50000 [==============================] - 30s 597us/sample - loss: 0.1598 - accuracy: 0.9456\n",
      "Epoch 26/30\n",
      "50000/50000 [==============================] - 30s 596us/sample - loss: 0.1594 - accuracy: 0.9456\n",
      "Epoch 27/30\n",
      "50000/50000 [==============================] - 30s 599us/sample - loss: 0.1528 - accuracy: 0.9475\n",
      "Epoch 28/30\n",
      "50000/50000 [==============================] - 30s 602us/sample - loss: 0.1467 - accuracy: 0.9494\n",
      "Epoch 29/30\n",
      "50000/50000 [==============================] - 30s 597us/sample - loss: 0.1424 - accuracy: 0.9512\n",
      "Epoch 30/30\n",
      "50000/50000 [==============================] - 30s 599us/sample - loss: 0.1366 - accuracy: 0.9536\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x23f4d931bc8>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# compile the model (should be done *after* setting layers to non-trainable)\n",
    "Transfer_Model.compile( optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'] )\n",
    "Transfer_Model.fit( x_train, y_train, batch_size=32, epochs=30 )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}