homework/Day021_Transfer+Learning_HW.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 『本次練習內容』\n",
    "#### 使用Xception backbone做 Trnasfer Learning\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 『本次練習目的』\n",
    "  #### 了解如何使用Transfer Learning\n",
    "  #### 了解Transfer Learning的優點，可以觀察模型收斂速度"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 可以自行嘗試多種架構"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/anaconda3/lib/python3.7/site-packages/keras_applications/resnet50.py:265: UserWarning: The output shape of `ResNet50(include_top=False)` has been changed since Keras 2.2.0.\n",
      "  warnings.warn('The output shape of `ResNet50(include_top=False)` '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"resnet50\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 32, 32, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv1_pad (ZeroPadding2D)       (None, 38, 38, 3)    0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv1 (Conv2D)                  (None, 16, 16, 64)   9472        conv1_pad[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "bn_conv1 (BatchNormalization)   (None, 16, 16, 64)   256         conv1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 16, 16, 64)   0           bn_conv1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "pool1_pad (ZeroPadding2D)       (None, 18, 18, 64)   0           activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)  (None, 8, 8, 64)     0           pool1_pad[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "res2a_branch2a (Conv2D)         (None, 8, 8, 64)     4160        max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "bn2a_branch2a (BatchNormalizati (None, 8, 8, 64)     256         res2a_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 8, 8, 64)     0           bn2a_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res2a_branch2b (Conv2D)         (None, 8, 8, 64)     36928       activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bn2a_branch2b (BatchNormalizati (None, 8, 8, 64)     256         res2a_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 8, 8, 64)     0           bn2a_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res2a_branch2c (Conv2D)         (None, 8, 8, 256)    16640       activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "res2a_branch1 (Conv2D)          (None, 8, 8, 256)    16640       max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "bn2a_branch2c (BatchNormalizati (None, 8, 8, 256)    1024        res2a_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "bn2a_branch1 (BatchNormalizatio (None, 8, 8, 256)    1024        res2a_branch1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 8, 8, 256)    0           bn2a_branch2c[0][0]              \n",
      "                                                                 bn2a_branch1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 8, 8, 256)    0           add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res2b_branch2a (Conv2D)         (None, 8, 8, 64)     16448       activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bn2b_branch2a (BatchNormalizati (None, 8, 8, 64)     256         res2b_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, 8, 8, 64)     0           bn2b_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res2b_branch2b (Conv2D)         (None, 8, 8, 64)     36928       activation_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bn2b_branch2b (BatchNormalizati (None, 8, 8, 64)     256         res2b_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_6 (Activation)       (None, 8, 8, 64)     0           bn2b_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res2b_branch2c (Conv2D)         (None, 8, 8, 256)    16640       activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bn2b_branch2c (BatchNormalizati (None, 8, 8, 256)    1024        res2b_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, 8, 8, 256)    0           bn2b_branch2c[0][0]              \n",
      "                                                                 activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_7 (Activation)       (None, 8, 8, 256)    0           add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res2c_branch2a (Conv2D)         (None, 8, 8, 64)     16448       activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bn2c_branch2a (BatchNormalizati (None, 8, 8, 64)     256         res2c_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_8 (Activation)       (None, 8, 8, 64)     0           bn2c_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res2c_branch2b (Conv2D)         (None, 8, 8, 64)     36928       activation_8[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bn2c_branch2b (BatchNormalizati (None, 8, 8, 64)     256         res2c_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_9 (Activation)       (None, 8, 8, 64)     0           bn2c_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res2c_branch2c (Conv2D)         (None, 8, 8, 256)    16640       activation_9[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bn2c_branch2c (BatchNormalizati (None, 8, 8, 256)    1024        res2c_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_3 (Add)                     (None, 8, 8, 256)    0           bn2c_branch2c[0][0]              \n",
      "                                                                 activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_10 (Activation)      (None, 8, 8, 256)    0           add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res3a_branch2a (Conv2D)         (None, 4, 4, 128)    32896       activation_10[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3a_branch2a (BatchNormalizati (None, 4, 4, 128)    512         res3a_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_11 (Activation)      (None, 4, 4, 128)    0           bn3a_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3a_branch2b (Conv2D)         (None, 4, 4, 128)    147584      activation_11[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3a_branch2b (BatchNormalizati (None, 4, 4, 128)    512         res3a_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_12 (Activation)      (None, 4, 4, 128)    0           bn3a_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3a_branch2c (Conv2D)         (None, 4, 4, 512)    66048       activation_12[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3a_branch1 (Conv2D)          (None, 4, 4, 512)    131584      activation_10[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3a_branch2c (BatchNormalizati (None, 4, 4, 512)    2048        res3a_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "bn3a_branch1 (BatchNormalizatio (None, 4, 4, 512)    2048        res3a_branch1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, 4, 4, 512)    0           bn3a_branch2c[0][0]              \n",
      "                                                                 bn3a_branch1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_13 (Activation)      (None, 4, 4, 512)    0           add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res3b_branch2a (Conv2D)         (None, 4, 4, 128)    65664       activation_13[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3b_branch2a (BatchNormalizati (None, 4, 4, 128)    512         res3b_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_14 (Activation)      (None, 4, 4, 128)    0           bn3b_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3b_branch2b (Conv2D)         (None, 4, 4, 128)    147584      activation_14[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3b_branch2b (BatchNormalizati (None, 4, 4, 128)    512         res3b_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_15 (Activation)      (None, 4, 4, 128)    0           bn3b_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3b_branch2c (Conv2D)         (None, 4, 4, 512)    66048       activation_15[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3b_branch2c (BatchNormalizati (None, 4, 4, 512)    2048        res3b_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, 4, 4, 512)    0           bn3b_branch2c[0][0]              \n",
      "                                                                 activation_13[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_16 (Activation)      (None, 4, 4, 512)    0           add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res3c_branch2a (Conv2D)         (None, 4, 4, 128)    65664       activation_16[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3c_branch2a (BatchNormalizati (None, 4, 4, 128)    512         res3c_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_17 (Activation)      (None, 4, 4, 128)    0           bn3c_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3c_branch2b (Conv2D)         (None, 4, 4, 128)    147584      activation_17[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3c_branch2b (BatchNormalizati (None, 4, 4, 128)    512         res3c_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_18 (Activation)      (None, 4, 4, 128)    0           bn3c_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3c_branch2c (Conv2D)         (None, 4, 4, 512)    66048       activation_18[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3c_branch2c (BatchNormalizati (None, 4, 4, 512)    2048        res3c_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_6 (Add)                     (None, 4, 4, 512)    0           bn3c_branch2c[0][0]              \n",
      "                                                                 activation_16[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_19 (Activation)      (None, 4, 4, 512)    0           add_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res3d_branch2a (Conv2D)         (None, 4, 4, 128)    65664       activation_19[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3d_branch2a (BatchNormalizati (None, 4, 4, 128)    512         res3d_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_20 (Activation)      (None, 4, 4, 128)    0           bn3d_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3d_branch2b (Conv2D)         (None, 4, 4, 128)    147584      activation_20[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3d_branch2b (BatchNormalizati (None, 4, 4, 128)    512         res3d_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_21 (Activation)      (None, 4, 4, 128)    0           bn3d_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3d_branch2c (Conv2D)         (None, 4, 4, 512)    66048       activation_21[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3d_branch2c (BatchNormalizati (None, 4, 4, 512)    2048        res3d_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_7 (Add)                     (None, 4, 4, 512)    0           bn3d_branch2c[0][0]              \n",
      "                                                                 activation_19[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_22 (Activation)      (None, 4, 4, 512)    0           add_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res4a_branch2a (Conv2D)         (None, 2, 2, 256)    131328      activation_22[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4a_branch2a (BatchNormalizati (None, 2, 2, 256)    1024        res4a_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_23 (Activation)      (None, 2, 2, 256)    0           bn4a_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4a_branch2b (Conv2D)         (None, 2, 2, 256)    590080      activation_23[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4a_branch2b (BatchNormalizati (None, 2, 2, 256)    1024        res4a_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_24 (Activation)      (None, 2, 2, 256)    0           bn4a_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4a_branch2c (Conv2D)         (None, 2, 2, 1024)   263168      activation_24[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4a_branch1 (Conv2D)          (None, 2, 2, 1024)   525312      activation_22[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4a_branch2c (BatchNormalizati (None, 2, 2, 1024)   4096        res4a_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "bn4a_branch1 (BatchNormalizatio (None, 2, 2, 1024)   4096        res4a_branch1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_8 (Add)                     (None, 2, 2, 1024)   0           bn4a_branch2c[0][0]              \n",
      "                                                                 bn4a_branch1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_25 (Activation)      (None, 2, 2, 1024)   0           add_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res4b_branch2a (Conv2D)         (None, 2, 2, 256)    262400      activation_25[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4b_branch2a (BatchNormalizati (None, 2, 2, 256)    1024        res4b_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_26 (Activation)      (None, 2, 2, 256)    0           bn4b_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4b_branch2b (Conv2D)         (None, 2, 2, 256)    590080      activation_26[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4b_branch2b (BatchNormalizati (None, 2, 2, 256)    1024        res4b_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_27 (Activation)      (None, 2, 2, 256)    0           bn4b_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4b_branch2c (Conv2D)         (None, 2, 2, 1024)   263168      activation_27[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4b_branch2c (BatchNormalizati (None, 2, 2, 1024)   4096        res4b_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_9 (Add)                     (None, 2, 2, 1024)   0           bn4b_branch2c[0][0]              \n",
      "                                                                 activation_25[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_28 (Activation)      (None, 2, 2, 1024)   0           add_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res4c_branch2a (Conv2D)         (None, 2, 2, 256)    262400      activation_28[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4c_branch2a (BatchNormalizati (None, 2, 2, 256)    1024        res4c_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_29 (Activation)      (None, 2, 2, 256)    0           bn4c_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4c_branch2b (Conv2D)         (None, 2, 2, 256)    590080      activation_29[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4c_branch2b (BatchNormalizati (None, 2, 2, 256)    1024        res4c_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_30 (Activation)      (None, 2, 2, 256)    0           bn4c_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4c_branch2c (Conv2D)         (None, 2, 2, 1024)   263168      activation_30[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4c_branch2c (BatchNormalizati (None, 2, 2, 1024)   4096        res4c_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_10 (Add)                    (None, 2, 2, 1024)   0           bn4c_branch2c[0][0]              \n",
      "                                                                 activation_28[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_31 (Activation)      (None, 2, 2, 1024)   0           add_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "res4d_branch2a (Conv2D)         (None, 2, 2, 256)    262400      activation_31[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4d_branch2a (BatchNormalizati (None, 2, 2, 256)    1024        res4d_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_32 (Activation)      (None, 2, 2, 256)    0           bn4d_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4d_branch2b (Conv2D)         (None, 2, 2, 256)    590080      activation_32[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4d_branch2b (BatchNormalizati (None, 2, 2, 256)    1024        res4d_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_33 (Activation)      (None, 2, 2, 256)    0           bn4d_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4d_branch2c (Conv2D)         (None, 2, 2, 1024)   263168      activation_33[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4d_branch2c (BatchNormalizati (None, 2, 2, 1024)   4096        res4d_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_11 (Add)                    (None, 2, 2, 1024)   0           bn4d_branch2c[0][0]              \n",
      "                                                                 activation_31[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_34 (Activation)      (None, 2, 2, 1024)   0           add_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "res4e_branch2a (Conv2D)         (None, 2, 2, 256)    262400      activation_34[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4e_branch2a (BatchNormalizati (None, 2, 2, 256)    1024        res4e_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_35 (Activation)      (None, 2, 2, 256)    0           bn4e_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4e_branch2b (Conv2D)         (None, 2, 2, 256)    590080      activation_35[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4e_branch2b (BatchNormalizati (None, 2, 2, 256)    1024        res4e_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_36 (Activation)      (None, 2, 2, 256)    0           bn4e_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4e_branch2c (Conv2D)         (None, 2, 2, 1024)   263168      activation_36[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4e_branch2c (BatchNormalizati (None, 2, 2, 1024)   4096        res4e_branch2c[0][0]             \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "add_12 (Add)                    (None, 2, 2, 1024)   0           bn4e_branch2c[0][0]              \n",
      "                                                                 activation_34[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_37 (Activation)      (None, 2, 2, 1024)   0           add_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "res4f_branch2a (Conv2D)         (None, 2, 2, 256)    262400      activation_37[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4f_branch2a (BatchNormalizati (None, 2, 2, 256)    1024        res4f_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_38 (Activation)      (None, 2, 2, 256)    0           bn4f_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4f_branch2b (Conv2D)         (None, 2, 2, 256)    590080      activation_38[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4f_branch2b (BatchNormalizati (None, 2, 2, 256)    1024        res4f_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_39 (Activation)      (None, 2, 2, 256)    0           bn4f_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4f_branch2c (Conv2D)         (None, 2, 2, 1024)   263168      activation_39[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4f_branch2c (BatchNormalizati (None, 2, 2, 1024)   4096        res4f_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_13 (Add)                    (None, 2, 2, 1024)   0           bn4f_branch2c[0][0]              \n",
      "                                                                 activation_37[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_40 (Activation)      (None, 2, 2, 1024)   0           add_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "res5a_branch2a (Conv2D)         (None, 1, 1, 512)    524800      activation_40[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5a_branch2a (BatchNormalizati (None, 1, 1, 512)    2048        res5a_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_41 (Activation)      (None, 1, 1, 512)    0           bn5a_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res5a_branch2b (Conv2D)         (None, 1, 1, 512)    2359808     activation_41[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5a_branch2b (BatchNormalizati (None, 1, 1, 512)    2048        res5a_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_42 (Activation)      (None, 1, 1, 512)    0           bn5a_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res5a_branch2c (Conv2D)         (None, 1, 1, 2048)   1050624     activation_42[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res5a_branch1 (Conv2D)          (None, 1, 1, 2048)   2099200     activation_40[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5a_branch2c (BatchNormalizati (None, 1, 1, 2048)   8192        res5a_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "bn5a_branch1 (BatchNormalizatio (None, 1, 1, 2048)   8192        res5a_branch1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_14 (Add)                    (None, 1, 1, 2048)   0           bn5a_branch2c[0][0]              \n",
      "                                                                 bn5a_branch1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_43 (Activation)      (None, 1, 1, 2048)   0           add_14[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "res5b_branch2a (Conv2D)         (None, 1, 1, 512)    1049088     activation_43[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5b_branch2a (BatchNormalizati (None, 1, 1, 512)    2048        res5b_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_44 (Activation)      (None, 1, 1, 512)    0           bn5b_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res5b_branch2b (Conv2D)         (None, 1, 1, 512)    2359808     activation_44[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5b_branch2b (BatchNormalizati (None, 1, 1, 512)    2048        res5b_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_45 (Activation)      (None, 1, 1, 512)    0           bn5b_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res5b_branch2c (Conv2D)         (None, 1, 1, 2048)   1050624     activation_45[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5b_branch2c (BatchNormalizati (None, 1, 1, 2048)   8192        res5b_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_15 (Add)                    (None, 1, 1, 2048)   0           bn5b_branch2c[0][0]              \n",
      "                                                                 activation_43[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_46 (Activation)      (None, 1, 1, 2048)   0           add_15[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "res5c_branch2a (Conv2D)         (None, 1, 1, 512)    1049088     activation_46[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5c_branch2a (BatchNormalizati (None, 1, 1, 512)    2048        res5c_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_47 (Activation)      (None, 1, 1, 512)    0           bn5c_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res5c_branch2b (Conv2D)         (None, 1, 1, 512)    2359808     activation_47[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5c_branch2b (BatchNormalizati (None, 1, 1, 512)    2048        res5c_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_48 (Activation)      (None, 1, 1, 512)    0           bn5c_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res5c_branch2c (Conv2D)         (None, 1, 1, 2048)   1050624     activation_48[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5c_branch2c (BatchNormalizati (None, 1, 1, 2048)   8192        res5c_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_16 (Add)                    (None, 1, 1, 2048)   0           bn5c_branch2c[0][0]              \n",
      "                                                                 activation_46[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_49 (Activation)      (None, 1, 1, 2048)   0           add_16[0][0]                     \n",
      "==================================================================================================\n",
      "Total params: 23,587,712\n",
      "Trainable params: 23,534,592\n",
      "Non-trainable params: 53,120\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Convolution2D\n",
    "from keras.layers import MaxPooling2D\n",
    "from keras.layers import Flatten\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Dropout\n",
    "from keras.preprocessing import image\n",
    "from keras.models import Model\n",
    "from keras.layers import Dense, GlobalAveragePooling2D\n",
    "from keras import backend as K\n",
    "import keras\n",
    "from keras.layers import Input\n",
    " \n",
    "from keras.datasets import cifar10\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "\n",
    "input_tensor = Input(shape=(32, 32, 3))\n",
    "#include top 決定要不要加入 fully Connected Layer\n",
    "\n",
    "'''Xception 架構'''\n",
    "# 參考 https://keras.io/zh/applications/\n",
    "\n",
    "#model=keras.applications.xception.Xception(include_top=False, weights='imagenet',\\\n",
    "                                  #input_tensor=input_tensor,pooling=None, classes=10)\n",
    "\n",
    "\n",
    "'''Resnet 50 架構'''\n",
    "model=keras.applications.ResNet50(include_top=False, weights='imagenet',\\\n",
    "                                  input_tensor=input_tensor,pooling=None, classes=10)\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 添加層數"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model深度： 179\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:5: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(activation=\"relu\", units=128)`\n",
      "  \"\"\"\n",
      "/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:6: UserWarning: Update your `Dropout` call to the Keras 2 API: `Dropout(rate=0.1)`\n",
      "  \n",
      "/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:8: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(activation=\"softmax\", units=10)`\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "x = model.output\n",
    "\n",
    "'''可以參考Cifar10實作章節'''\n",
    "x = GlobalAveragePooling2D()(x)\n",
    "x = Dense(output_dim=128, activation='relu')(x)\n",
    "x=Dropout(p=0.1)(x)\n",
    "\n",
    "predictions = Dense(output_dim=10,activation='softmax')(x)\n",
    "model = Model(inputs=model.input, outputs=predictions)\n",
    "print('Model深度：', len(model.layers))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 鎖定特定幾層不要更新權重"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "for layer in model.layers[:100]:\n",
    "    layer.trainable = False\n",
    "for layer in model.layers[100:]:\n",
    "    layer.trainable = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 準備 Cifar 10 資料"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 32, 32, 3)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_encoders.py:415: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.\n",
      "If you want the future behaviour and silence this warning, you can specify \"categories='auto'\".\n",
      "In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "\n",
    "print(x_train.shape) #(50000, 32, 32, 3)\n",
    "\n",
    "## Normalize Data\n",
    "def normalize(X_train,X_test):\n",
    "        mean = np.mean(X_train,axis=(0,1,2,3))\n",
    "        std = np.std(X_train, axis=(0, 1, 2, 3))\n",
    "        X_train = (X_train-mean)/(std+1e-7)\n",
    "        X_test = (X_test-mean)/(std+1e-7)\n",
    "        return X_train, X_test\n",
    "    \n",
    "    \n",
    "## Normalize Training and Testset    \n",
    "x_train, x_test = normalize(x_train, x_test) \n",
    "\n",
    "## OneHot Label 由(None, 1)-(None, 10)\n",
    "## ex. label=2,變成[0,0,1,0,0,0,0,0,0,0]\n",
    "one_hot=OneHotEncoder()\n",
    "y_train=one_hot.fit_transform(y_train).toarray()\n",
    "y_test=one_hot.transform(y_test).toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "50000/50000 [==============================] - 7935s 159ms/step - loss: 1.2510 - accuracy: 0.5797\n",
      "Epoch 2/10\n",
      "50000/50000 [==============================] - 5442s 109ms/step - loss: 0.9604 - accuracy: 0.6761\n",
      "Epoch 3/10\n",
      "50000/50000 [==============================] - 5304s 106ms/step - loss: 0.8932 - accuracy: 0.6921\n",
      "Epoch 4/10\n",
      "50000/50000 [==============================] - 4423s 88ms/step - loss: 0.8015 - accuracy: 0.7246\n",
      "Epoch 5/10\n",
      "50000/50000 [==============================] - 3581s 72ms/step - loss: 0.7266 - accuracy: 0.7475\n",
      "Epoch 6/10\n",
      "50000/50000 [==============================] - 3284s 66ms/step - loss: 0.6834 - accuracy: 0.7618\n",
      "Epoch 7/10\n",
      "50000/50000 [==============================] - 3268s 65ms/step - loss: 0.5655 - accuracy: 0.8020\n",
      "Epoch 8/10\n",
      "50000/50000 [==============================] - 3392s 68ms/step - loss: 0.5061 - accuracy: 0.8224\n",
      "Epoch 9/10\n",
      "50000/50000 [==============================] - 3300s 66ms/step - loss: 0.4488 - accuracy: 0.8421\n",
      "Epoch 10/10\n",
      "50000/50000 [==============================] - 3241s 65ms/step - loss: 0.4040 - accuracy: 0.8601\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x1a5a64e48>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# compile the model (should be done *after* setting layers to non-trainable)\n",
    "model.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy'])\n",
    "model.fit(x_train,y_train,batch_size=32,epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}