Fusion_model.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "n9ojayfgkJhI"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import pydicom\n",
        "import os\n",
        "import matplotlib.pyplot as plt\n",
        "import collections\n",
        "from tqdm import tqdm_notebook as tqdm\n",
        "from datetime import datetime\n",
        "# from imgaug import augmenters as iaa\n",
        "from scipy import ndimage\n",
        "from math import ceil, floor, log\n",
        "import cv2\n",
        "import tensorflow as tf\n",
        "import tensorflow.keras as keras\n",
        "from tensorflow.keras.utils import to_categorical\n",
        "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
        "import sys\n",
        "import heapq\n",
        "import efficientnet.tfkeras as efn \n",
        "from sklearn.model_selection import ShuffleSplit\n",
        "\n",
        "# Set the log level of TensorFlow to suppress unnecessary output\n",
        "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'\n",
        "from tensorflow.python.util import deprecation\n",
        "deprecation._PRINT_DEPRECATION_WARNINGS = False\n",
        "tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)\n",
        "\n",
        "# Import necessary libraries for data preprocessing and modeling\n",
        "# numpy - for numerical operations on arrays\n",
        "# pandas - for data manipulation and analysis\n",
        "# pydicom - for reading DICOM files\n",
        "# os - for interacting with the operating system\n",
        "# matplotlib - for data visualization\n",
        "# collections - for specialized data structures\n",
        "# tqdm - for progress bars during loops\n",
        "# datetime - for working with dates and times\n",
        "# scipy - for scientific computing and image processing\n",
        "# math - for mathematical operations\n",
        "# cv2 - for computer vision tasks\n",
        "# tensorflow - for building and training machine learning models\n",
        "# sys - for system-specific parameters and functions\n",
        "# heapq - for heap-based data structures\n",
        "# efficientnet - a library of efficient neural network architectures\n",
        "# scikit-learn - for machine learning algorithms and model evaluation\n",
        "\n",
        "# Define functions for data preprocessing and modeling here (not shown)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "xeqxS-NAkJhI"
      },
      "outputs": [],
      "source": [
        "# preprocess the features  (train + val)\n",
        "tbl = pd.read_csv('# The path of the training clinical features')\n",
        "\n",
        "# combine_id + key\n",
        "tbl['ID'] = tbl.apply(lambda row: str(row.hashed_patient_ir_id) + '_' + row.Hash_Key, axis = 1)\n",
        "tbl = tbl.drop(['hashed_patient_ir_id', 'Hash_Key'], axis = 1)\n",
        "\n",
        "tbl.drop_duplicates(subset=['ID'], keep='first', inplace = True)\n",
        "\n",
        "tbl = tbl.set_index('ID')\n",
        "tbl.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "hcMQs8BEkJhJ"
      },
      "outputs": [],
      "source": [
        "# preprocess the features  (train + val)\n",
        "tbl_val = pd.read_csv('# The path of the validation clinical features')\n",
        "tbl_val.head(5)\n",
        "\n",
        "# combine_id + key\n",
        "tbl_val['ID'] = tbl_val.apply(lambda row: str(row.hashed_patient_ir_id) + '_' + row.Hash_Key, axis = 1)\n",
        "tbl_val = tbl_val.drop(['hashed_patient_ir_id', 'Hash_Key'], axis = 1)\n",
        "tbl_val.drop_duplicates(subset=['ID'], keep='first', inplace = True)\n",
        "\n",
        "tbl_val = tbl_val.set_index('ID')\n",
        "tbl_val.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "n7koeplXkJhK"
      },
      "outputs": [],
      "source": [
        "# preprocess testing dataset \n",
        "tbl_test = pd.read_csv('# The path of the testing clinical features')\n",
        "tbl_test.head(5)\n",
        "\n",
        "# combine_id + key\n",
        "tbl_test['ID'] = tbl_test.apply(lambda row: str(row.hashed_patient_ir_id) + '_' + row.Hash_Key, axis = 1)\n",
        "tbl_test = tbl_test.drop(['hashed_patient_ir_id', 'Hash_Key'], axis = 1)\n",
        "tbl_test.drop_duplicates(subset=['ID'], keep='first', inplace = True)\n",
        "\n",
        "tbl_test = tbl_test.set_index('ID')\n",
        "tbl_test.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "R8VTxkIxkJhK"
      },
      "outputs": [],
      "source": [
        "tbl_value = pd.read_csv('# The path of the risk level labels').loc[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "80feubkdkJhK"
      },
      "outputs": [],
      "source": [
        "def _read_tbl(df, ID, desired_size):    # here the input is \"int\"\n",
        "    try:\n",
        "        row = df.loc[ID,:]\n",
        "    except:\n",
        "        row = tbl_value\n",
        "    return row"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uYFIFQldkJhK"
      },
      "outputs": [],
      "source": [
        "from skimage.io import imread\n",
        "imagenet_mean = np.array([0.485, 0.456, 0.406])\n",
        "imagenet_std = np.array([0.229, 0.224, 0.225])\n",
        "\n",
        "def preprocess (img):\n",
        "    img = img/255.0\n",
        "    centered = np.subtract(img, imagenet_mean)\n",
        "    standardized = np.divide(centered, imagenet_std)\n",
        "    return standardized \n",
        "    return img  \n",
        "\n",
        "def _read_img(img_dir, ID):\n",
        "    img = imread(img_dir + ID)   \n",
        "    img = preprocess(img)\n",
        "    return img"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Jnkpdau3kJhL"
      },
      "outputs": [],
      "source": [
        "# Define the size of the input images\n",
        "input_size = (224, 224)\n",
        "\n",
        "# Define the paths to the directories containing the training, validation, and testing CXR features\n",
        "train_images_dir = '# The path of the training CXR features'\n",
        "val_images_dir = '# The path of the validation CXR features'\n",
        "test_images_dir = '# The path of the testing CXR features'\n",
        "\n",
        "# Define the training, validation, and testing clinical features DataFrames\n",
        "train_fs_dir = tbl\n",
        "valid_fs_dir = tbl_val\n",
        "test_fs_dir = tbl_test\n",
        "\n",
        "# Define the length of the clinical features vector\n",
        "length = 99\n",
        "\n",
        "class DataGenerator(keras.utils.Sequence):\n",
        "    'Generates data for Keras'\n",
        "    def __init__(self, list_IDs, labels, batch_size=16, dim=input_size, n_channels=3,\n",
        "                 img_dir=train_images_dir, fs_dir=train_fs_dir, fs_size=length,\n",
        "                 augment=False, shuffle=True):\n",
        "        'Initialization'\n",
        "        self.dim = dim\n",
        "        self.batch_size = batch_size\n",
        "        self.labels = labels\n",
        "        self.list_IDs = list_IDs\n",
        "        self.n_channels = n_channels\n",
        "        self.img_dir = img_dir\n",
        "        self.shuffle = shuffle\n",
        "        self.fs_dir = fs_dir\n",
        "        self.fs_size = fs_size\n",
        "        self.augment = augment\n",
        "        self.on_epoch_end()\n",
        "        \n",
        "    def __len__(self):\n",
        "        'Denotes the number of batches per epoch'\n",
        "        return int(np.floor(len(self.list_IDs) / self.batch_size))\n",
        "    \n",
        "    def __getitem__(self, index):\n",
        "        'Generate one batch of data'\n",
        "        # Generate indexes of the batch\n",
        "        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n",
        "\n",
        "        # Find list of IDs\n",
        "        list_IDs_temp = [self.list_IDs[k] for k in indexes]\n",
        "\n",
        "        # Generate data\n",
        "        X, X_fs, y = self.__data_generation(list_IDs_temp)\n",
        "\n",
        "        return [X, X_fs], y   \n",
        "    \n",
        "    def on_epoch_end(self):\n",
        "        'Updates indexes after each epoch'\n",
        "        self.indexes = np.arange(len(self.list_IDs))\n",
        "        if self.shuffle == True:\n",
        "            np.random.shuffle(self.indexes)\n",
        "            \n",
        "    def __data_generation(self, list_IDs_temp):\n",
        "        'Generates data containing batch_size samples'\n",
        "        # Initialization\n",
        "        X = np.empty((self.batch_size, *self.dim, self.n_channels))\n",
        "        X_fs = np.empty((self.batch_size, self.fs_size))\n",
        "        y = np.empty((self.batch_size, 3), dtype=np.float32)\n",
        "\n",
        "        # Generate data\n",
        "        for i, ID in enumerate(list_IDs_temp):\n",
        "            # Read in the CXR image and clinical features for the current sample\n",
        "            img = _read_img(self.img_dir, ID)\n",
        "            ID_split = ID.split('-')\n",
        "            X_fs[i,] = _read_tbl(self.fs_dir, ID_split[0], self.fs_size) \n",
        "\n",
        "            # Apply data augmentation if specified\n",
        "            if self.augment:\n",
        "                p = random.uniform(0,1)\n",
        "                X[i,] = Augmentation(img, p)\n",
        "            else:\n",
        "                X[i,] = img\n",
        "\n",
        "            # Get the label for the current sample and convert it to a one-hot encoded vector\n",
        "            y_temp = self.labels.loc[ID] - 1\n",
        "            y[i,:] = to_categorical(y_temp, num_classes=3)\n",
        "\n",
        "        return X, X_fs, y"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "PyCc9FgukJhL"
      },
      "outputs": [],
      "source": [
        "train_xr = '# The path of the risk level labels'\n",
        "val_xr = '# The path of the risk level labels'\n",
        "df = pd.read_csv(train_xr)\n",
        "df = df.set_index(['ID'])\n",
        "df.index\n",
        "\n",
        "df_val = pd.read_csv(val_xr)\n",
        "df_val = df_val.set_index(['ID'])\n",
        "df_val.index\n",
        "\n",
        "test_xr = '# The path of the risk level labels'\n",
        "\n",
        "df_test = pd.read_csv(test_xr)\n",
        "df_test = df_test.set_index(['ID'])\n",
        "df_test.index"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "S3_jYyGKkJhL",
        "outputId": "1481f950-2a19-4a73-934c-fb1319d19469"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "2    193\n",
              "3    145\n",
              "1    101\n",
              "Name: Level, dtype: int64"
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_test.Level.value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "49mRTpP-kJhL"
      },
      "outputs": [],
      "source": [
        "train_seq = DataGenerator(df.index, df, 16,input_size, 3,\n",
        "                         img_dir=train_images_dir,fs_dir = train_fs_dir, fs_size = length , shuffle=True)\n",
        "val_seq = DataGenerator(df_val.index, df_val, 16, input_size, 3,\n",
        "                         img_dir=val_images_dir,fs_dir = valid_fs_dir, fs_size = length, shuffle=False)\n",
        "\n",
        "test_seq = DataGenerator(df_test.index, df_test, 1, input_size, 3,\n",
        "                         img_dir=test_images_dir,fs_dir = test_fs_dir, fs_size = length, shuffle=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WsvBCq9FkJhM",
        "outputId": "27643802-dc67-460b-b9b4-f8954a4bf30b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            [(None, 224, 224, 3) 0                                            \n",
            "__________________________________________________________________________________________________\n",
            "zero_padding2d (ZeroPadding2D)  (None, 230, 230, 3)  0           input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "conv1/conv (Conv2D)             (None, 112, 112, 64) 9408        zero_padding2d[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "conv1/bn (BatchNormalization)   (None, 112, 112, 64) 256         conv1/conv[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv1/relu (Activation)         (None, 112, 112, 64) 0           conv1/bn[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "zero_padding2d_1 (ZeroPadding2D (None, 114, 114, 64) 0           conv1/relu[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "pool1 (MaxPooling2D)            (None, 56, 56, 64)   0           zero_padding2d_1[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block1_0_bn (BatchNormali (None, 56, 56, 64)   256         pool1[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block1_0_relu (Activation (None, 56, 56, 64)   0           conv2_block1_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block1_1_conv (Conv2D)    (None, 56, 56, 128)  8192        conv2_block1_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block1_1_bn (BatchNormali (None, 56, 56, 128)  512         conv2_block1_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block1_1_relu (Activation (None, 56, 56, 128)  0           conv2_block1_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block1_2_conv (Conv2D)    (None, 56, 56, 32)   36864       conv2_block1_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block1_concat (Concatenat (None, 56, 56, 96)   0           pool1[0][0]                      \n",
            "                                                                 conv2_block1_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block2_0_bn (BatchNormali (None, 56, 56, 96)   384         conv2_block1_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block2_0_relu (Activation (None, 56, 56, 96)   0           conv2_block2_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block2_1_conv (Conv2D)    (None, 56, 56, 128)  12288       conv2_block2_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block2_1_bn (BatchNormali (None, 56, 56, 128)  512         conv2_block2_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block2_1_relu (Activation (None, 56, 56, 128)  0           conv2_block2_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block2_2_conv (Conv2D)    (None, 56, 56, 32)   36864       conv2_block2_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block2_concat (Concatenat (None, 56, 56, 128)  0           conv2_block1_concat[0][0]        \n",
            "                                                                 conv2_block2_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block3_0_bn (BatchNormali (None, 56, 56, 128)  512         conv2_block2_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block3_0_relu (Activation (None, 56, 56, 128)  0           conv2_block3_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block3_1_conv (Conv2D)    (None, 56, 56, 128)  16384       conv2_block3_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block3_1_bn (BatchNormali (None, 56, 56, 128)  512         conv2_block3_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block3_1_relu (Activation (None, 56, 56, 128)  0           conv2_block3_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block3_2_conv (Conv2D)    (None, 56, 56, 32)   36864       conv2_block3_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block3_concat (Concatenat (None, 56, 56, 160)  0           conv2_block2_concat[0][0]        \n",
            "                                                                 conv2_block3_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block4_0_bn (BatchNormali (None, 56, 56, 160)  640         conv2_block3_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block4_0_relu (Activation (None, 56, 56, 160)  0           conv2_block4_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block4_1_conv (Conv2D)    (None, 56, 56, 128)  20480       conv2_block4_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block4_1_bn (BatchNormali (None, 56, 56, 128)  512         conv2_block4_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block4_1_relu (Activation (None, 56, 56, 128)  0           conv2_block4_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block4_2_conv (Conv2D)    (None, 56, 56, 32)   36864       conv2_block4_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block4_concat (Concatenat (None, 56, 56, 192)  0           conv2_block3_concat[0][0]        \n",
            "                                                                 conv2_block4_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block5_0_bn (BatchNormali (None, 56, 56, 192)  768         conv2_block4_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block5_0_relu (Activation (None, 56, 56, 192)  0           conv2_block5_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block5_1_conv (Conv2D)    (None, 56, 56, 128)  24576       conv2_block5_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block5_1_bn (BatchNormali (None, 56, 56, 128)  512         conv2_block5_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block5_1_relu (Activation (None, 56, 56, 128)  0           conv2_block5_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block5_2_conv (Conv2D)    (None, 56, 56, 32)   36864       conv2_block5_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block5_concat (Concatenat (None, 56, 56, 224)  0           conv2_block4_concat[0][0]        \n",
            "                                                                 conv2_block5_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block6_0_bn (BatchNormali (None, 56, 56, 224)  896         conv2_block5_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block6_0_relu (Activation (None, 56, 56, 224)  0           conv2_block6_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block6_1_conv (Conv2D)    (None, 56, 56, 128)  28672       conv2_block6_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block6_1_bn (BatchNormali (None, 56, 56, 128)  512         conv2_block6_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block6_1_relu (Activation (None, 56, 56, 128)  0           conv2_block6_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block6_2_conv (Conv2D)    (None, 56, 56, 32)   36864       conv2_block6_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block6_concat (Concatenat (None, 56, 56, 256)  0           conv2_block5_concat[0][0]        \n",
            "                                                                 conv2_block6_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "pool2_bn (BatchNormalization)   (None, 56, 56, 256)  1024        conv2_block6_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "pool2_relu (Activation)         (None, 56, 56, 256)  0           pool2_bn[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "pool2_conv (Conv2D)             (None, 56, 56, 128)  32768       pool2_relu[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "pool2_pool (AveragePooling2D)   (None, 28, 28, 128)  0           pool2_conv[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block1_0_bn (BatchNormali (None, 28, 28, 128)  512         pool2_pool[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block1_0_relu (Activation (None, 28, 28, 128)  0           conv3_block1_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block1_1_conv (Conv2D)    (None, 28, 28, 128)  16384       conv3_block1_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block1_1_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block1_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block1_1_relu (Activation (None, 28, 28, 128)  0           conv3_block1_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block1_2_conv (Conv2D)    (None, 28, 28, 32)   36864       conv3_block1_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block1_concat (Concatenat (None, 28, 28, 160)  0           pool2_pool[0][0]                 \n",
            "                                                                 conv3_block1_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block2_0_bn (BatchNormali (None, 28, 28, 160)  640         conv3_block1_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block2_0_relu (Activation (None, 28, 28, 160)  0           conv3_block2_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block2_1_conv (Conv2D)    (None, 28, 28, 128)  20480       conv3_block2_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block2_1_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block2_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block2_1_relu (Activation (None, 28, 28, 128)  0           conv3_block2_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block2_2_conv (Conv2D)    (None, 28, 28, 32)   36864       conv3_block2_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block2_concat (Concatenat (None, 28, 28, 192)  0           conv3_block1_concat[0][0]        \n",
            "                                                                 conv3_block2_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block3_0_bn (BatchNormali (None, 28, 28, 192)  768         conv3_block2_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block3_0_relu (Activation (None, 28, 28, 192)  0           conv3_block3_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block3_1_conv (Conv2D)    (None, 28, 28, 128)  24576       conv3_block3_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block3_1_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block3_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block3_1_relu (Activation (None, 28, 28, 128)  0           conv3_block3_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block3_2_conv (Conv2D)    (None, 28, 28, 32)   36864       conv3_block3_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block3_concat (Concatenat (None, 28, 28, 224)  0           conv3_block2_concat[0][0]        \n",
            "                                                                 conv3_block3_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block4_0_bn (BatchNormali (None, 28, 28, 224)  896         conv3_block3_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block4_0_relu (Activation (None, 28, 28, 224)  0           conv3_block4_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block4_1_conv (Conv2D)    (None, 28, 28, 128)  28672       conv3_block4_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block4_1_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block4_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block4_1_relu (Activation (None, 28, 28, 128)  0           conv3_block4_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block4_2_conv (Conv2D)    (None, 28, 28, 32)   36864       conv3_block4_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block4_concat (Concatenat (None, 28, 28, 256)  0           conv3_block3_concat[0][0]        \n",
            "                                                                 conv3_block4_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block5_0_bn (BatchNormali (None, 28, 28, 256)  1024        conv3_block4_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block5_0_relu (Activation (None, 28, 28, 256)  0           conv3_block5_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block5_1_conv (Conv2D)    (None, 28, 28, 128)  32768       conv3_block5_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block5_1_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block5_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block5_1_relu (Activation (None, 28, 28, 128)  0           conv3_block5_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block5_2_conv (Conv2D)    (None, 28, 28, 32)   36864       conv3_block5_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block5_concat (Concatenat (None, 28, 28, 288)  0           conv3_block4_concat[0][0]        \n",
            "                                                                 conv3_block5_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block6_0_bn (BatchNormali (None, 28, 28, 288)  1152        conv3_block5_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block6_0_relu (Activation (None, 28, 28, 288)  0           conv3_block6_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block6_1_conv (Conv2D)    (None, 28, 28, 128)  36864       conv3_block6_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block6_1_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block6_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block6_1_relu (Activation (None, 28, 28, 128)  0           conv3_block6_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block6_2_conv (Conv2D)    (None, 28, 28, 32)   36864       conv3_block6_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block6_concat (Concatenat (None, 28, 28, 320)  0           conv3_block5_concat[0][0]        \n",
            "                                                                 conv3_block6_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block7_0_bn (BatchNormali (None, 28, 28, 320)  1280        conv3_block6_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block7_0_relu (Activation (None, 28, 28, 320)  0           conv3_block7_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block7_1_conv (Conv2D)    (None, 28, 28, 128)  40960       conv3_block7_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block7_1_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block7_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block7_1_relu (Activation (None, 28, 28, 128)  0           conv3_block7_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block7_2_conv (Conv2D)    (None, 28, 28, 32)   36864       conv3_block7_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block7_concat (Concatenat (None, 28, 28, 352)  0           conv3_block6_concat[0][0]        \n",
            "                                                                 conv3_block7_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block8_0_bn (BatchNormali (None, 28, 28, 352)  1408        conv3_block7_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block8_0_relu (Activation (None, 28, 28, 352)  0           conv3_block8_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block8_1_conv (Conv2D)    (None, 28, 28, 128)  45056       conv3_block8_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block8_1_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block8_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block8_1_relu (Activation (None, 28, 28, 128)  0           conv3_block8_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block8_2_conv (Conv2D)    (None, 28, 28, 32)   36864       conv3_block8_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block8_concat (Concatenat (None, 28, 28, 384)  0           conv3_block7_concat[0][0]        \n",
            "                                                                 conv3_block8_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block9_0_bn (BatchNormali (None, 28, 28, 384)  1536        conv3_block8_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block9_0_relu (Activation (None, 28, 28, 384)  0           conv3_block9_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block9_1_conv (Conv2D)    (None, 28, 28, 128)  49152       conv3_block9_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block9_1_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block9_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block9_1_relu (Activation (None, 28, 28, 128)  0           conv3_block9_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block9_2_conv (Conv2D)    (None, 28, 28, 32)   36864       conv3_block9_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block9_concat (Concatenat (None, 28, 28, 416)  0           conv3_block8_concat[0][0]        \n",
            "                                                                 conv3_block9_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block10_0_bn (BatchNormal (None, 28, 28, 416)  1664        conv3_block9_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block10_0_relu (Activatio (None, 28, 28, 416)  0           conv3_block10_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block10_1_conv (Conv2D)   (None, 28, 28, 128)  53248       conv3_block10_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block10_1_bn (BatchNormal (None, 28, 28, 128)  512         conv3_block10_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block10_1_relu (Activatio (None, 28, 28, 128)  0           conv3_block10_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block10_2_conv (Conv2D)   (None, 28, 28, 32)   36864       conv3_block10_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block10_concat (Concatena (None, 28, 28, 448)  0           conv3_block9_concat[0][0]        \n",
            "                                                                 conv3_block10_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block11_0_bn (BatchNormal (None, 28, 28, 448)  1792        conv3_block10_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block11_0_relu (Activatio (None, 28, 28, 448)  0           conv3_block11_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block11_1_conv (Conv2D)   (None, 28, 28, 128)  57344       conv3_block11_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block11_1_bn (BatchNormal (None, 28, 28, 128)  512         conv3_block11_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block11_1_relu (Activatio (None, 28, 28, 128)  0           conv3_block11_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block11_2_conv (Conv2D)   (None, 28, 28, 32)   36864       conv3_block11_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block11_concat (Concatena (None, 28, 28, 480)  0           conv3_block10_concat[0][0]       \n",
            "                                                                 conv3_block11_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block12_0_bn (BatchNormal (None, 28, 28, 480)  1920        conv3_block11_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block12_0_relu (Activatio (None, 28, 28, 480)  0           conv3_block12_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block12_1_conv (Conv2D)   (None, 28, 28, 128)  61440       conv3_block12_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block12_1_bn (BatchNormal (None, 28, 28, 128)  512         conv3_block12_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block12_1_relu (Activatio (None, 28, 28, 128)  0           conv3_block12_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block12_2_conv (Conv2D)   (None, 28, 28, 32)   36864       conv3_block12_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block12_concat (Concatena (None, 28, 28, 512)  0           conv3_block11_concat[0][0]       \n",
            "                                                                 conv3_block12_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "pool3_bn (BatchNormalization)   (None, 28, 28, 512)  2048        conv3_block12_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "pool3_relu (Activation)         (None, 28, 28, 512)  0           pool3_bn[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "pool3_conv (Conv2D)             (None, 28, 28, 256)  131072      pool3_relu[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "pool3_pool (AveragePooling2D)   (None, 14, 14, 256)  0           pool3_conv[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block1_0_bn (BatchNormali (None, 14, 14, 256)  1024        pool3_pool[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block1_0_relu (Activation (None, 14, 14, 256)  0           conv4_block1_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block1_1_conv (Conv2D)    (None, 14, 14, 128)  32768       conv4_block1_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block1_1_bn (BatchNormali (None, 14, 14, 128)  512         conv4_block1_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block1_1_relu (Activation (None, 14, 14, 128)  0           conv4_block1_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block1_2_conv (Conv2D)    (None, 14, 14, 32)   36864       conv4_block1_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block1_concat (Concatenat (None, 14, 14, 288)  0           pool3_pool[0][0]                 \n",
            "                                                                 conv4_block1_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block2_0_bn (BatchNormali (None, 14, 14, 288)  1152        conv4_block1_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block2_0_relu (Activation (None, 14, 14, 288)  0           conv4_block2_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block2_1_conv (Conv2D)    (None, 14, 14, 128)  36864       conv4_block2_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block2_1_bn (BatchNormali (None, 14, 14, 128)  512         conv4_block2_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block2_1_relu (Activation (None, 14, 14, 128)  0           conv4_block2_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block2_2_conv (Conv2D)    (None, 14, 14, 32)   36864       conv4_block2_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block2_concat (Concatenat (None, 14, 14, 320)  0           conv4_block1_concat[0][0]        \n",
            "                                                                 conv4_block2_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block3_0_bn (BatchNormali (None, 14, 14, 320)  1280        conv4_block2_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block3_0_relu (Activation (None, 14, 14, 320)  0           conv4_block3_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block3_1_conv (Conv2D)    (None, 14, 14, 128)  40960       conv4_block3_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block3_1_bn (BatchNormali (None, 14, 14, 128)  512         conv4_block3_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block3_1_relu (Activation (None, 14, 14, 128)  0           conv4_block3_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block3_2_conv (Conv2D)    (None, 14, 14, 32)   36864       conv4_block3_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block3_concat (Concatenat (None, 14, 14, 352)  0           conv4_block2_concat[0][0]        \n",
            "                                                                 conv4_block3_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block4_0_bn (BatchNormali (None, 14, 14, 352)  1408        conv4_block3_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block4_0_relu (Activation (None, 14, 14, 352)  0           conv4_block4_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block4_1_conv (Conv2D)    (None, 14, 14, 128)  45056       conv4_block4_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block4_1_bn (BatchNormali (None, 14, 14, 128)  512         conv4_block4_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block4_1_relu (Activation (None, 14, 14, 128)  0           conv4_block4_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block4_2_conv (Conv2D)    (None, 14, 14, 32)   36864       conv4_block4_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block4_concat (Concatenat (None, 14, 14, 384)  0           conv4_block3_concat[0][0]        \n",
            "                                                                 conv4_block4_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block5_0_bn (BatchNormali (None, 14, 14, 384)  1536        conv4_block4_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block5_0_relu (Activation (None, 14, 14, 384)  0           conv4_block5_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block5_1_conv (Conv2D)    (None, 14, 14, 128)  49152       conv4_block5_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block5_1_bn (BatchNormali (None, 14, 14, 128)  512         conv4_block5_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block5_1_relu (Activation (None, 14, 14, 128)  0           conv4_block5_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block5_2_conv (Conv2D)    (None, 14, 14, 32)   36864       conv4_block5_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block5_concat (Concatenat (None, 14, 14, 416)  0           conv4_block4_concat[0][0]        \n",
            "                                                                 conv4_block5_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block6_0_bn (BatchNormali (None, 14, 14, 416)  1664        conv4_block5_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block6_0_relu (Activation (None, 14, 14, 416)  0           conv4_block6_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block6_1_conv (Conv2D)    (None, 14, 14, 128)  53248       conv4_block6_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block6_1_bn (BatchNormali (None, 14, 14, 128)  512         conv4_block6_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block6_1_relu (Activation (None, 14, 14, 128)  0           conv4_block6_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block6_2_conv (Conv2D)    (None, 14, 14, 32)   36864       conv4_block6_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block6_concat (Concatenat (None, 14, 14, 448)  0           conv4_block5_concat[0][0]        \n",
            "                                                                 conv4_block6_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block7_0_bn (BatchNormali (None, 14, 14, 448)  1792        conv4_block6_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block7_0_relu (Activation (None, 14, 14, 448)  0           conv4_block7_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block7_1_conv (Conv2D)    (None, 14, 14, 128)  57344       conv4_block7_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block7_1_bn (BatchNormali (None, 14, 14, 128)  512         conv4_block7_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block7_1_relu (Activation (None, 14, 14, 128)  0           conv4_block7_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block7_2_conv (Conv2D)    (None, 14, 14, 32)   36864       conv4_block7_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block7_concat (Concatenat (None, 14, 14, 480)  0           conv4_block6_concat[0][0]        \n",
            "                                                                 conv4_block7_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block8_0_bn (BatchNormali (None, 14, 14, 480)  1920        conv4_block7_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block8_0_relu (Activation (None, 14, 14, 480)  0           conv4_block8_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block8_1_conv (Conv2D)    (None, 14, 14, 128)  61440       conv4_block8_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block8_1_bn (BatchNormali (None, 14, 14, 128)  512         conv4_block8_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block8_1_relu (Activation (None, 14, 14, 128)  0           conv4_block8_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block8_2_conv (Conv2D)    (None, 14, 14, 32)   36864       conv4_block8_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block8_concat (Concatenat (None, 14, 14, 512)  0           conv4_block7_concat[0][0]        \n",
            "                                                                 conv4_block8_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block9_0_bn (BatchNormali (None, 14, 14, 512)  2048        conv4_block8_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block9_0_relu (Activation (None, 14, 14, 512)  0           conv4_block9_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block9_1_conv (Conv2D)    (None, 14, 14, 128)  65536       conv4_block9_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block9_1_bn (BatchNormali (None, 14, 14, 128)  512         conv4_block9_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block9_1_relu (Activation (None, 14, 14, 128)  0           conv4_block9_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block9_2_conv (Conv2D)    (None, 14, 14, 32)   36864       conv4_block9_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block9_concat (Concatenat (None, 14, 14, 544)  0           conv4_block8_concat[0][0]        \n",
            "                                                                 conv4_block9_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block10_0_bn (BatchNormal (None, 14, 14, 544)  2176        conv4_block9_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block10_0_relu (Activatio (None, 14, 14, 544)  0           conv4_block10_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block10_1_conv (Conv2D)   (None, 14, 14, 128)  69632       conv4_block10_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block10_1_bn (BatchNormal (None, 14, 14, 128)  512         conv4_block10_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block10_1_relu (Activatio (None, 14, 14, 128)  0           conv4_block10_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block10_2_conv (Conv2D)   (None, 14, 14, 32)   36864       conv4_block10_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block10_concat (Concatena (None, 14, 14, 576)  0           conv4_block9_concat[0][0]        \n",
            "                                                                 conv4_block10_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block11_0_bn (BatchNormal (None, 14, 14, 576)  2304        conv4_block10_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block11_0_relu (Activatio (None, 14, 14, 576)  0           conv4_block11_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block11_1_conv (Conv2D)   (None, 14, 14, 128)  73728       conv4_block11_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block11_1_bn (BatchNormal (None, 14, 14, 128)  512         conv4_block11_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block11_1_relu (Activatio (None, 14, 14, 128)  0           conv4_block11_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block11_2_conv (Conv2D)   (None, 14, 14, 32)   36864       conv4_block11_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block11_concat (Concatena (None, 14, 14, 608)  0           conv4_block10_concat[0][0]       \n",
            "                                                                 conv4_block11_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block12_0_bn (BatchNormal (None, 14, 14, 608)  2432        conv4_block11_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block12_0_relu (Activatio (None, 14, 14, 608)  0           conv4_block12_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block12_1_conv (Conv2D)   (None, 14, 14, 128)  77824       conv4_block12_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block12_1_bn (BatchNormal (None, 14, 14, 128)  512         conv4_block12_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block12_1_relu (Activatio (None, 14, 14, 128)  0           conv4_block12_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block12_2_conv (Conv2D)   (None, 14, 14, 32)   36864       conv4_block12_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block12_concat (Concatena (None, 14, 14, 640)  0           conv4_block11_concat[0][0]       \n",
            "                                                                 conv4_block12_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block13_0_bn (BatchNormal (None, 14, 14, 640)  2560        conv4_block12_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block13_0_relu (Activatio (None, 14, 14, 640)  0           conv4_block13_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block13_1_conv (Conv2D)   (None, 14, 14, 128)  81920       conv4_block13_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block13_1_bn (BatchNormal (None, 14, 14, 128)  512         conv4_block13_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block13_1_relu (Activatio (None, 14, 14, 128)  0           conv4_block13_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block13_2_conv (Conv2D)   (None, 14, 14, 32)   36864       conv4_block13_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block13_concat (Concatena (None, 14, 14, 672)  0           conv4_block12_concat[0][0]       \n",
            "                                                                 conv4_block13_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block14_0_bn (BatchNormal (None, 14, 14, 672)  2688        conv4_block13_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block14_0_relu (Activatio (None, 14, 14, 672)  0           conv4_block14_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block14_1_conv (Conv2D)   (None, 14, 14, 128)  86016       conv4_block14_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block14_1_bn (BatchNormal (None, 14, 14, 128)  512         conv4_block14_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block14_1_relu (Activatio (None, 14, 14, 128)  0           conv4_block14_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block14_2_conv (Conv2D)   (None, 14, 14, 32)   36864       conv4_block14_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block14_concat (Concatena (None, 14, 14, 704)  0           conv4_block13_concat[0][0]       \n",
            "                                                                 conv4_block14_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block15_0_bn (BatchNormal (None, 14, 14, 704)  2816        conv4_block14_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block15_0_relu (Activatio (None, 14, 14, 704)  0           conv4_block15_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block15_1_conv (Conv2D)   (None, 14, 14, 128)  90112       conv4_block15_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block15_1_bn (BatchNormal (None, 14, 14, 128)  512         conv4_block15_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block15_1_relu (Activatio (None, 14, 14, 128)  0           conv4_block15_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block15_2_conv (Conv2D)   (None, 14, 14, 32)   36864       conv4_block15_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block15_concat (Concatena (None, 14, 14, 736)  0           conv4_block14_concat[0][0]       \n",
            "                                                                 conv4_block15_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block16_0_bn (BatchNormal (None, 14, 14, 736)  2944        conv4_block15_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block16_0_relu (Activatio (None, 14, 14, 736)  0           conv4_block16_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block16_1_conv (Conv2D)   (None, 14, 14, 128)  94208       conv4_block16_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block16_1_bn (BatchNormal (None, 14, 14, 128)  512         conv4_block16_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block16_1_relu (Activatio (None, 14, 14, 128)  0           conv4_block16_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block16_2_conv (Conv2D)   (None, 14, 14, 32)   36864       conv4_block16_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block16_concat (Concatena (None, 14, 14, 768)  0           conv4_block15_concat[0][0]       \n",
            "                                                                 conv4_block16_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block17_0_bn (BatchNormal (None, 14, 14, 768)  3072        conv4_block16_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block17_0_relu (Activatio (None, 14, 14, 768)  0           conv4_block17_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block17_1_conv (Conv2D)   (None, 14, 14, 128)  98304       conv4_block17_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block17_1_bn (BatchNormal (None, 14, 14, 128)  512         conv4_block17_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block17_1_relu (Activatio (None, 14, 14, 128)  0           conv4_block17_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block17_2_conv (Conv2D)   (None, 14, 14, 32)   36864       conv4_block17_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block17_concat (Concatena (None, 14, 14, 800)  0           conv4_block16_concat[0][0]       \n",
            "                                                                 conv4_block17_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block18_0_bn (BatchNormal (None, 14, 14, 800)  3200        conv4_block17_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block18_0_relu (Activatio (None, 14, 14, 800)  0           conv4_block18_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block18_1_conv (Conv2D)   (None, 14, 14, 128)  102400      conv4_block18_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block18_1_bn (BatchNormal (None, 14, 14, 128)  512         conv4_block18_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block18_1_relu (Activatio (None, 14, 14, 128)  0           conv4_block18_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block18_2_conv (Conv2D)   (None, 14, 14, 32)   36864       conv4_block18_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block18_concat (Concatena (None, 14, 14, 832)  0           conv4_block17_concat[0][0]       \n",
            "                                                                 conv4_block18_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block19_0_bn (BatchNormal (None, 14, 14, 832)  3328        conv4_block18_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block19_0_relu (Activatio (None, 14, 14, 832)  0           conv4_block19_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block19_1_conv (Conv2D)   (None, 14, 14, 128)  106496      conv4_block19_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block19_1_bn (BatchNormal (None, 14, 14, 128)  512         conv4_block19_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block19_1_relu (Activatio (None, 14, 14, 128)  0           conv4_block19_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block19_2_conv (Conv2D)   (None, 14, 14, 32)   36864       conv4_block19_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block19_concat (Concatena (None, 14, 14, 864)  0           conv4_block18_concat[0][0]       \n",
            "                                                                 conv4_block19_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block20_0_bn (BatchNormal (None, 14, 14, 864)  3456        conv4_block19_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block20_0_relu (Activatio (None, 14, 14, 864)  0           conv4_block20_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block20_1_conv (Conv2D)   (None, 14, 14, 128)  110592      conv4_block20_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block20_1_bn (BatchNormal (None, 14, 14, 128)  512         conv4_block20_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block20_1_relu (Activatio (None, 14, 14, 128)  0           conv4_block20_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block20_2_conv (Conv2D)   (None, 14, 14, 32)   36864       conv4_block20_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block20_concat (Concatena (None, 14, 14, 896)  0           conv4_block19_concat[0][0]       \n",
            "                                                                 conv4_block20_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block21_0_bn (BatchNormal (None, 14, 14, 896)  3584        conv4_block20_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block21_0_relu (Activatio (None, 14, 14, 896)  0           conv4_block21_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block21_1_conv (Conv2D)   (None, 14, 14, 128)  114688      conv4_block21_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block21_1_bn (BatchNormal (None, 14, 14, 128)  512         conv4_block21_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block21_1_relu (Activatio (None, 14, 14, 128)  0           conv4_block21_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block21_2_conv (Conv2D)   (None, 14, 14, 32)   36864       conv4_block21_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block21_concat (Concatena (None, 14, 14, 928)  0           conv4_block20_concat[0][0]       \n",
            "                                                                 conv4_block21_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block22_0_bn (BatchNormal (None, 14, 14, 928)  3712        conv4_block21_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block22_0_relu (Activatio (None, 14, 14, 928)  0           conv4_block22_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block22_1_conv (Conv2D)   (None, 14, 14, 128)  118784      conv4_block22_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block22_1_bn (BatchNormal (None, 14, 14, 128)  512         conv4_block22_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block22_1_relu (Activatio (None, 14, 14, 128)  0           conv4_block22_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block22_2_conv (Conv2D)   (None, 14, 14, 32)   36864       conv4_block22_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block22_concat (Concatena (None, 14, 14, 960)  0           conv4_block21_concat[0][0]       \n",
            "                                                                 conv4_block22_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block23_0_bn (BatchNormal (None, 14, 14, 960)  3840        conv4_block22_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block23_0_relu (Activatio (None, 14, 14, 960)  0           conv4_block23_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block23_1_conv (Conv2D)   (None, 14, 14, 128)  122880      conv4_block23_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block23_1_bn (BatchNormal (None, 14, 14, 128)  512         conv4_block23_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block23_1_relu (Activatio (None, 14, 14, 128)  0           conv4_block23_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block23_2_conv (Conv2D)   (None, 14, 14, 32)   36864       conv4_block23_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block23_concat (Concatena (None, 14, 14, 992)  0           conv4_block22_concat[0][0]       \n",
            "                                                                 conv4_block23_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block24_0_bn (BatchNormal (None, 14, 14, 992)  3968        conv4_block23_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block24_0_relu (Activatio (None, 14, 14, 992)  0           conv4_block24_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block24_1_conv (Conv2D)   (None, 14, 14, 128)  126976      conv4_block24_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block24_1_bn (BatchNormal (None, 14, 14, 128)  512         conv4_block24_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block24_1_relu (Activatio (None, 14, 14, 128)  0           conv4_block24_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block24_2_conv (Conv2D)   (None, 14, 14, 32)   36864       conv4_block24_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block24_concat (Concatena (None, 14, 14, 1024) 0           conv4_block23_concat[0][0]       \n",
            "                                                                 conv4_block24_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "pool4_bn (BatchNormalization)   (None, 14, 14, 1024) 4096        conv4_block24_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "pool4_relu (Activation)         (None, 14, 14, 1024) 0           pool4_bn[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "pool4_conv (Conv2D)             (None, 14, 14, 512)  524288      pool4_relu[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "pool4_pool (AveragePooling2D)   (None, 7, 7, 512)    0           pool4_conv[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block1_0_bn (BatchNormali (None, 7, 7, 512)    2048        pool4_pool[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block1_0_relu (Activation (None, 7, 7, 512)    0           conv5_block1_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block1_1_conv (Conv2D)    (None, 7, 7, 128)    65536       conv5_block1_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block1_1_bn (BatchNormali (None, 7, 7, 128)    512         conv5_block1_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block1_1_relu (Activation (None, 7, 7, 128)    0           conv5_block1_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block1_2_conv (Conv2D)    (None, 7, 7, 32)     36864       conv5_block1_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block1_concat (Concatenat (None, 7, 7, 544)    0           pool4_pool[0][0]                 \n",
            "                                                                 conv5_block1_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block2_0_bn (BatchNormali (None, 7, 7, 544)    2176        conv5_block1_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block2_0_relu (Activation (None, 7, 7, 544)    0           conv5_block2_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block2_1_conv (Conv2D)    (None, 7, 7, 128)    69632       conv5_block2_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block2_1_bn (BatchNormali (None, 7, 7, 128)    512         conv5_block2_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block2_1_relu (Activation (None, 7, 7, 128)    0           conv5_block2_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block2_2_conv (Conv2D)    (None, 7, 7, 32)     36864       conv5_block2_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block2_concat (Concatenat (None, 7, 7, 576)    0           conv5_block1_concat[0][0]        \n",
            "                                                                 conv5_block2_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block3_0_bn (BatchNormali (None, 7, 7, 576)    2304        conv5_block2_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block3_0_relu (Activation (None, 7, 7, 576)    0           conv5_block3_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block3_1_conv (Conv2D)    (None, 7, 7, 128)    73728       conv5_block3_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block3_1_bn (BatchNormali (None, 7, 7, 128)    512         conv5_block3_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block3_1_relu (Activation (None, 7, 7, 128)    0           conv5_block3_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block3_2_conv (Conv2D)    (None, 7, 7, 32)     36864       conv5_block3_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block3_concat (Concatenat (None, 7, 7, 608)    0           conv5_block2_concat[0][0]        \n",
            "                                                                 conv5_block3_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block4_0_bn (BatchNormali (None, 7, 7, 608)    2432        conv5_block3_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block4_0_relu (Activation (None, 7, 7, 608)    0           conv5_block4_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block4_1_conv (Conv2D)    (None, 7, 7, 128)    77824       conv5_block4_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block4_1_bn (BatchNormali (None, 7, 7, 128)    512         conv5_block4_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block4_1_relu (Activation (None, 7, 7, 128)    0           conv5_block4_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block4_2_conv (Conv2D)    (None, 7, 7, 32)     36864       conv5_block4_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block4_concat (Concatenat (None, 7, 7, 640)    0           conv5_block3_concat[0][0]        \n",
            "                                                                 conv5_block4_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block5_0_bn (BatchNormali (None, 7, 7, 640)    2560        conv5_block4_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block5_0_relu (Activation (None, 7, 7, 640)    0           conv5_block5_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block5_1_conv (Conv2D)    (None, 7, 7, 128)    81920       conv5_block5_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block5_1_bn (BatchNormali (None, 7, 7, 128)    512         conv5_block5_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block5_1_relu (Activation (None, 7, 7, 128)    0           conv5_block5_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block5_2_conv (Conv2D)    (None, 7, 7, 32)     36864       conv5_block5_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block5_concat (Concatenat (None, 7, 7, 672)    0           conv5_block4_concat[0][0]        \n",
            "                                                                 conv5_block5_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block6_0_bn (BatchNormali (None, 7, 7, 672)    2688        conv5_block5_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block6_0_relu (Activation (None, 7, 7, 672)    0           conv5_block6_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block6_1_conv (Conv2D)    (None, 7, 7, 128)    86016       conv5_block6_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block6_1_bn (BatchNormali (None, 7, 7, 128)    512         conv5_block6_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block6_1_relu (Activation (None, 7, 7, 128)    0           conv5_block6_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block6_2_conv (Conv2D)    (None, 7, 7, 32)     36864       conv5_block6_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block6_concat (Concatenat (None, 7, 7, 704)    0           conv5_block5_concat[0][0]        \n",
            "                                                                 conv5_block6_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block7_0_bn (BatchNormali (None, 7, 7, 704)    2816        conv5_block6_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block7_0_relu (Activation (None, 7, 7, 704)    0           conv5_block7_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block7_1_conv (Conv2D)    (None, 7, 7, 128)    90112       conv5_block7_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block7_1_bn (BatchNormali (None, 7, 7, 128)    512         conv5_block7_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block7_1_relu (Activation (None, 7, 7, 128)    0           conv5_block7_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block7_2_conv (Conv2D)    (None, 7, 7, 32)     36864       conv5_block7_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block7_concat (Concatenat (None, 7, 7, 736)    0           conv5_block6_concat[0][0]        \n",
            "                                                                 conv5_block7_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block8_0_bn (BatchNormali (None, 7, 7, 736)    2944        conv5_block7_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block8_0_relu (Activation (None, 7, 7, 736)    0           conv5_block8_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block8_1_conv (Conv2D)    (None, 7, 7, 128)    94208       conv5_block8_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block8_1_bn (BatchNormali (None, 7, 7, 128)    512         conv5_block8_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block8_1_relu (Activation (None, 7, 7, 128)    0           conv5_block8_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block8_2_conv (Conv2D)    (None, 7, 7, 32)     36864       conv5_block8_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block8_concat (Concatenat (None, 7, 7, 768)    0           conv5_block7_concat[0][0]        \n",
            "                                                                 conv5_block8_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block9_0_bn (BatchNormali (None, 7, 7, 768)    3072        conv5_block8_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block9_0_relu (Activation (None, 7, 7, 768)    0           conv5_block9_0_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block9_1_conv (Conv2D)    (None, 7, 7, 128)    98304       conv5_block9_0_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block9_1_bn (BatchNormali (None, 7, 7, 128)    512         conv5_block9_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block9_1_relu (Activation (None, 7, 7, 128)    0           conv5_block9_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block9_2_conv (Conv2D)    (None, 7, 7, 32)     36864       conv5_block9_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block9_concat (Concatenat (None, 7, 7, 800)    0           conv5_block8_concat[0][0]        \n",
            "                                                                 conv5_block9_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block10_0_bn (BatchNormal (None, 7, 7, 800)    3200        conv5_block9_concat[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block10_0_relu (Activatio (None, 7, 7, 800)    0           conv5_block10_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block10_1_conv (Conv2D)   (None, 7, 7, 128)    102400      conv5_block10_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block10_1_bn (BatchNormal (None, 7, 7, 128)    512         conv5_block10_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block10_1_relu (Activatio (None, 7, 7, 128)    0           conv5_block10_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block10_2_conv (Conv2D)   (None, 7, 7, 32)     36864       conv5_block10_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block10_concat (Concatena (None, 7, 7, 832)    0           conv5_block9_concat[0][0]        \n",
            "                                                                 conv5_block10_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block11_0_bn (BatchNormal (None, 7, 7, 832)    3328        conv5_block10_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block11_0_relu (Activatio (None, 7, 7, 832)    0           conv5_block11_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block11_1_conv (Conv2D)   (None, 7, 7, 128)    106496      conv5_block11_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block11_1_bn (BatchNormal (None, 7, 7, 128)    512         conv5_block11_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block11_1_relu (Activatio (None, 7, 7, 128)    0           conv5_block11_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block11_2_conv (Conv2D)   (None, 7, 7, 32)     36864       conv5_block11_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block11_concat (Concatena (None, 7, 7, 864)    0           conv5_block10_concat[0][0]       \n",
            "                                                                 conv5_block11_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block12_0_bn (BatchNormal (None, 7, 7, 864)    3456        conv5_block11_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block12_0_relu (Activatio (None, 7, 7, 864)    0           conv5_block12_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block12_1_conv (Conv2D)   (None, 7, 7, 128)    110592      conv5_block12_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block12_1_bn (BatchNormal (None, 7, 7, 128)    512         conv5_block12_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block12_1_relu (Activatio (None, 7, 7, 128)    0           conv5_block12_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block12_2_conv (Conv2D)   (None, 7, 7, 32)     36864       conv5_block12_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block12_concat (Concatena (None, 7, 7, 896)    0           conv5_block11_concat[0][0]       \n",
            "                                                                 conv5_block12_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block13_0_bn (BatchNormal (None, 7, 7, 896)    3584        conv5_block12_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block13_0_relu (Activatio (None, 7, 7, 896)    0           conv5_block13_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block13_1_conv (Conv2D)   (None, 7, 7, 128)    114688      conv5_block13_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block13_1_bn (BatchNormal (None, 7, 7, 128)    512         conv5_block13_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block13_1_relu (Activatio (None, 7, 7, 128)    0           conv5_block13_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block13_2_conv (Conv2D)   (None, 7, 7, 32)     36864       conv5_block13_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block13_concat (Concatena (None, 7, 7, 928)    0           conv5_block12_concat[0][0]       \n",
            "                                                                 conv5_block13_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block14_0_bn (BatchNormal (None, 7, 7, 928)    3712        conv5_block13_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block14_0_relu (Activatio (None, 7, 7, 928)    0           conv5_block14_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block14_1_conv (Conv2D)   (None, 7, 7, 128)    118784      conv5_block14_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block14_1_bn (BatchNormal (None, 7, 7, 128)    512         conv5_block14_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block14_1_relu (Activatio (None, 7, 7, 128)    0           conv5_block14_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block14_2_conv (Conv2D)   (None, 7, 7, 32)     36864       conv5_block14_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block14_concat (Concatena (None, 7, 7, 960)    0           conv5_block13_concat[0][0]       \n",
            "                                                                 conv5_block14_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block15_0_bn (BatchNormal (None, 7, 7, 960)    3840        conv5_block14_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block15_0_relu (Activatio (None, 7, 7, 960)    0           conv5_block15_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block15_1_conv (Conv2D)   (None, 7, 7, 128)    122880      conv5_block15_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block15_1_bn (BatchNormal (None, 7, 7, 128)    512         conv5_block15_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block15_1_relu (Activatio (None, 7, 7, 128)    0           conv5_block15_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block15_2_conv (Conv2D)   (None, 7, 7, 32)     36864       conv5_block15_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block15_concat (Concatena (None, 7, 7, 992)    0           conv5_block14_concat[0][0]       \n",
            "                                                                 conv5_block15_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block16_0_bn (BatchNormal (None, 7, 7, 992)    3968        conv5_block15_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block16_0_relu (Activatio (None, 7, 7, 992)    0           conv5_block16_0_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block16_1_conv (Conv2D)   (None, 7, 7, 128)    126976      conv5_block16_0_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block16_1_bn (BatchNormal (None, 7, 7, 128)    512         conv5_block16_1_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block16_1_relu (Activatio (None, 7, 7, 128)    0           conv5_block16_1_bn[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block16_2_conv (Conv2D)   (None, 7, 7, 32)     36864       conv5_block16_1_relu[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block16_concat (Concatena (None, 7, 7, 1024)   0           conv5_block15_concat[0][0]       \n",
            "                                                                 conv5_block16_2_conv[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "bn (BatchNormalization)         (None, 7, 7, 1024)   4096        conv5_block16_concat[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "relu (Activation)               (None, 7, 7, 1024)   0           bn[0][0]                         \n",
            "__________________________________________________________________________________________________\n",
            "input_2 (InputLayer)            [(None, 99)]         0                                            \n",
            "__________________________________________________________________________________________________\n",
            "avg_pool (GlobalAveragePooling2 (None, 1024)         0           relu[0][0]                       \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 128)          12800       input_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 64)           65600       avg_pool[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_Relu_1 (TensorFlowO [(None, 128)]        0           dense_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dropout (Dropout)               (None, 64)           0           dense[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "dropout_1 (Dropout)             (None, 128)          0           tf_op_layer_Relu_1[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "concatenate (Concatenate)       (None, 192)          0           dropout[0][0]                    \n",
            "                                                                 dropout_1[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "dense_2 (Dense)                 (None, 128)          24704       concatenate[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "activation (Activation)         (None, 128)          0           dense_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "fusion_last (Dense)             (None, 3)            387         activation[0][0]                 \n",
            "==================================================================================================\n",
            "Total params: 7,140,995\n",
            "Trainable params: 7,057,347\n",
            "Non-trainable params: 83,648\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "from tensorflow.keras.initializers import glorot_normal\n",
        "\n",
        "# Build the base model\n",
        "base_model = DenseNet121(include_top=False, weights=None, input_shape=(224,224,3))\n",
        "\n",
        "# Add a global average pooling layer\n",
        "x = keras.layers.GlobalAveragePooling2D(name='avg_pool')(base_model.output)\n",
        "\n",
        "# Add a dense layer\n",
        "x = keras.layers.Dense(64, kernel_initializer=glorot_normal())(x)\n",
        "\n",
        "# Apply dropout to reduce overfitting\n",
        "xray = keras.layers.Dropout(0.2)(x)\n",
        "\n",
        "# Create input layer for the clinical features\n",
        "fs_in = keras.layers.Input(shape=(length,))\n",
        "\n",
        "# Add a dense layer to process the clinical features\n",
        "feature = keras.layers.Dense(128, kernel_initializer=glorot_normal())(fs_in)\n",
        "feature = keras.activations.relu(feature)\n",
        "feature = keras.layers.Dropout(0.2)(feature)\n",
        "\n",
        "# Concatenate image and clinical features\n",
        "fusion = keras.layers.Concatenate()([xray, feature])\n",
        "\n",
        "# Add another dense layer to process the fusion features\n",
        "fusion = keras.layers.Dense(128, kernel_initializer=glorot_normal())(fusion)\n",
        "fusion = keras.layers.Activation('relu')(fusion)\n",
        "\n",
        "# Output layer\n",
        "predictions = keras.layers.Dense(3, activation=\"softmax\", name='fusion_last')(fusion)\n",
        "\n",
        "# Define the model\n",
        "model = keras.models.Model(inputs=[base_model.input, fs_in], outputs=predictions)\n",
        "\n",
        "# Print the model summary\n",
        "model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "y_biyWwpkJhM"
      },
      "outputs": [],
      "source": [
        "for i in range(5):\n",
        "    \n",
        "    Fold = i\n",
        "\n",
        "    # load the initial weights by name\n",
        "    model.load_weights('../DenseNet_224_up_uncrop.h5', by_name = True)\n",
        "\n",
        "    len(base_model.layers)\n",
        "\n",
        "    for layer in model.layers[:428]:\n",
        "        layer.trainable = False\n",
        "    for layer in model.layers[428:]:\n",
        "        layer.trainable = True\n",
        "\n",
        "\n",
        "    model.compile(loss='categorical_crossentropy', \n",
        "                  optimizer=keras.optimizers.SGD(lr=0.0002, momentum=0.9,nesterov=True),\n",
        "                  metrics=['acc',                       \n",
        "                         keras.metrics.AUC(),\n",
        "                         keras.metrics.Precision(name='precision'),\n",
        "                         keras.metrics.Recall(name='recall')])\n",
        "\n",
        "\n",
        "    from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n",
        "    weightpath = '# The path to save the move'.format(Fold)\n",
        "    es = EarlyStopping(monitor='val_acc', \n",
        "                       verbose=1, \n",
        "                       patience=8, \n",
        "                       min_delta=0.000001, \n",
        "                       mode='max')\n",
        "    mc = ModelCheckpoint(weightpath, \n",
        "                         monitor='val_acc', \n",
        "                         verbose=1, \n",
        "                         save_best_only=True, \n",
        "                         mode='max')\n",
        "    rlr = ReduceLROnPlateau(monitor='val_acc',\n",
        "                            mode='max',\n",
        "                            factor=0.1,\n",
        "                            patience=3)\n",
        "    \n",
        "    # class weight\n",
        "    classes = pd.value_counts(df.Level)\n",
        "    print(classes)\n",
        "    # # classes\n",
        "    class_weight = [(classes[3] + classes[1] + classes[2])/classes[1], (classes[3] + classes[1] + classes[2])/classes[2],(classes[3] + classes[1] + classes[2])/classes[3]] \n",
        "    print(class_weight)\n",
        "    \n",
        "    \n",
        "    hist = model.fit_generator(train_seq, epochs=100, verbose=1, max_queue_size=1, \n",
        "                           workers=1, validation_data=val_seq, \n",
        "                           callbacks = [es, mc,rlr], use_multiprocessing=False)  #,  class_weight = class_weight)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "QA4jT1T5kJhN"
      },
      "outputs": [],
      "source": [
        "plt.plot(hist.history['acc'])\n",
        "plt.plot(hist.history['val_acc'])\n",
        "plt.title('Model accuracy')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train', 'Val'], loc='upper left')\n",
        "plt.savefig('insert_path')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C449hqKXkJhN"
      },
      "source": [
        "# second stage"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "SjxyTgEIkJhO"
      },
      "outputs": [],
      "source": [
        "for i in range(5):\n",
        "    model.load_weights('The path you saved the model above.hdf5'.format(i))\n",
        "\n",
        "    for layer in model.layers[:]:\n",
        "        layer.trainable = True\n",
        "\n",
        "\n",
        "    es = EarlyStopping(monitor='val_acc', \n",
        "                       verbose=1, \n",
        "                       patience=10, \n",
        "                       min_delta=0.0000001, \n",
        "                       mode='max')\n",
        "    mc = ModelCheckpoint('# The path to .hdf5'.format(i), \n",
        "                         monitor='val_acc', \n",
        "                         verbose=1, \n",
        "                         save_best_only=True, \n",
        "                         mode='max')\n",
        "    rlr = ReduceLROnPlateau(monitor='val_acc',\n",
        "                            mode='max',\n",
        "                            factor=0.1,\n",
        "                            patience=5)\n",
        "\n",
        "    model.compile(loss='binary_crossentropy', optimizer=keras.optimizers.SGD(learning_rate=0.00002, momentum=0.9,nesterov=True), \n",
        "                  metrics=['acc', \n",
        "                           tf.keras.metrics.AUC(name='auc'), \n",
        "                           tf.keras.metrics.Precision(name='precision'), \n",
        "                           tf.keras.metrics.Recall(name='recall')])\n",
        "\n",
        "\n",
        "    hist = model.fit_generator(train_seq, epochs=100, verbose=1, max_queue_size=1, \n",
        "                               workers=1, validation_data=val_seq, \n",
        "                               callbacks = [es, mc,rlr], use_multiprocessing=False) #, class_weight = class_weight)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "eaXZldzTkJhO"
      },
      "outputs": [],
      "source": [
        "plt.plot(hist.history['acc'])\n",
        "plt.plot(hist.history['val_acc'])\n",
        "plt.title('Model accuracy')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train', 'Val'], loc='upper left')\n",
        "plt.savefig('insert_path')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pXNE9tEEkJhO"
      },
      "outputs": [],
      "source": [
        "model.load_weights('# The path where you saved the best weights')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4TXxmb6VkJhP"
      },
      "outputs": [],
      "source": [
        "val_seq = DataGenerator(df_val.index, df_val, 1, input_size, 3,\n",
        "                         img_dir=val_images_dir,fs_dir = valid_fs_dir, fs_size = length, shuffle=False)\n",
        "Y_pred = model.predict_generator(val_seq,)\n",
        "y_true = val_seq.labels.Level.values\n",
        "\n",
        "Y_pred = np.argmax(Y_pred, axis = 1)\n",
        "Y_pred = Y_pred + 1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2xOdIEOjkJhP"
      },
      "outputs": [],
      "source": [
        "from sklearn.metrics import classification_report, confusion_matrix\n",
        "y_pred = np.round(Y_pred)\n",
        "print('Confusion Matrix')\n",
        "cm = confusion_matrix(y_true, y_pred)\n",
        "target_names = ['0', '1', '2']\n",
        "print(cm)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "URGFfGOWkJhP"
      },
      "outputs": [],
      "source": [
        "from sklearn.metrics import roc_curve, auc\n",
        "fpr, tpr, thresholds = roc_curve(y_true, Y_pred)\n",
        "auc = auc(fpr, tpr)\n",
        "plt.figure(1)\n",
        "plt.plot([0, 1], [0, 1], 'k--')\n",
        "plt.plot(fpr, tpr, label='AUC = {:.3f}'.format(auc))\n",
        "plt.xlabel('1-Specificity')\n",
        "plt.ylabel('Sensitivity')\n",
        "plt.title('ROC curve')\n",
        "plt.legend(loc='lower right')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2dfpoRfCkJhP"
      },
      "source": [
        "# Testing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "O5la4Rm6kJhP"
      },
      "outputs": [],
      "source": [
        "for i in range(5):\n",
        "    for j in range(1):\n",
        "        if j == 0:\n",
        "            model.load_weights('The path of the best model at Stage 2'.format(i))\n",
        "        else:\n",
        "            model.load_weights('The path of the best model at Stage 2'.format(i))\n",
        "\n",
        "        Y_pred = model.predict_generator(test_seq)\n",
        "        yy_pred = Y_pred\n",
        "        Y_pred = np.argmax(Y_pred, axis = 1)\n",
        "        Y_pred = Y_pred + 1\n",
        "\n",
        "        y_true = test_seq.labels.Level.values\n",
        "\n",
        "        from sklearn.metrics import classification_report, confusion_matrix, accuracy_score, recall_score, f1_score, roc_auc_score, precision_score\n",
        "        y_pred = np.round(Y_pred)\n",
        "\n",
        "        print('i = ', i, 'j = ', j+1)\n",
        "        print(accuracy_score(y_true, y_pred))\n",
        "        print(recall_score(y_true, y_pred, average = 'weighted'))\n",
        "        print(precision_score(y_true, y_pred, average = 'weighted'))\n",
        "        print(f1_score(y_true, y_pred, average = 'weighted'))\n",
        "        print(roc_auc_score(y_true, yy_pred, average = 'weighted', multi_class = 'ovo'))\n",
        "        print('**********')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cX1vjW8HkJhQ"
      },
      "source": [
        "# Grad-cam"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MlP8TJ56kJhQ",
        "outputId": "b81e32e8-e94e-4013-9f6d-7476e9096f60"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(1, 224, 224, 3)\n"
          ]
        }
      ],
      "source": [
        "from tensorflow.keras.preprocessing import image\n",
        "sample = '# one sample image'  \n",
        "input_path = '# path of the image' + sample\n",
        "input_img = image.load_img(input_path, target_size=(224, 224),\n",
        "                           color_mode='rgb', interpolation='lanczos')\n",
        "img_array = np.asarray(input_img, dtype='float64')\n",
        "img = preprocess(img_array)\n",
        "img = np.expand_dims(img, axis=0)\n",
        "print(img.shape)\n",
        "#tabular\n",
        "fea = _read_tbl(tbl_test,sample.split('-')[0], (99,))\n",
        "fea = np.expand_dims(fea, axis=0)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WaRVe2x9kJhQ",
        "outputId": "bdcdcf77-a0bb-40ef-e613-4705cb4fe9e3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Run it without eager mode. Paste those commands at the beginning of your script:\n",
            "> import tensorflow as tf\n",
            "> tf.compat.v1.disable_eager_execution()\n"
          ]
        }
      ],
      "source": [
        "from keract import get_activations\n",
        "import keract\n",
        "activations = get_activations(model, [img, fea], layer_names = 'bn', auto_compile=True)\n",
        "\n",
        "print(model.predict([img, fea]))\n",
        "\n",
        "lab = df_test.loc[sample,:]\n",
        "print(lab)\n",
        "\n",
        "yy = np.zeros((1,3))\n",
        "yy[:,lab-1] = 1\n",
        "print(yy)\n",
        "print(yy.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ILi7ZiuRkJhR",
        "outputId": "bfc57123-428c-4d14-e004-ecc65b0b1d3b"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(1, 7, 7, 1024)"
            ]
          },
          "execution_count": 133,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "res = keract.get_gradients_of_activations(model,[img, fea] , yy, layer_names = 'avg_pool ', output_format='simple')\n",
        "res['avg_pool'].shape\n",
        "activations['bn'].shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3V3tO3G0kJhR"
      },
      "outputs": [],
      "source": [
        "ACT = np.zeros((7,7))\n",
        "for i in range(res['avg_pool'].shape[1]):\n",
        "    ACT = ACT + res['avg_pool'][0, i] * activations['bn'][0, :,:,i]\n",
        "ACT[ACT < 0] = 0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BUPj6ZoFkJhR"
      },
      "outputs": [],
      "source": [
        "def NormalizeData(data):\n",
        "    return (data - np.min(data)) / (np.max(data) - np.min(data))\n",
        "\n",
        "ACT2 = NormalizeData(ACT)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "MzHANyZXkJhS"
      },
      "outputs": [],
      "source": [
        "import matplotlib.cm as cm\n",
        "from IPython.display import Image, display\n",
        "def save_and_display_gradcam(img_path, heatmap, cam_path=\"cam.jpg\", alpha=0.4):\n",
        "    # Load the original image\n",
        "    img = tf.keras.preprocessing.image.load_img(img_path)\n",
        "    img = tf.keras.preprocessing.image.img_to_array(img)\n",
        "\n",
        "    # Rescale heatmap to a range 0-255\n",
        "    heatmap = np.uint8(255 * heatmap)\n",
        "\n",
        "    # Use jet colormap to colorize heatmap\n",
        "    jet = cm.get_cmap(\"jet\")\n",
        "\n",
        "    # Use RGB values of the colormap\n",
        "    jet_colors = jet(np.arange(256))[:, :3]\n",
        "    jet_heatmap = jet_colors[heatmap]\n",
        "\n",
        "    # Create an image with RGB colorized heatmap\n",
        "    jet_heatmap = tf.keras.preprocessing.image.array_to_img(jet_heatmap)\n",
        "    jet_heatmap = jet_heatmap.resize((img.shape[1], img.shape[0]))\n",
        "    jet_heatmap = tf.keras.preprocessing.image.img_to_array(jet_heatmap)\n",
        "\n",
        "    # Superimpose the heatmap on original image\n",
        "    superimposed_img = jet_heatmap * alpha + img\n",
        "    superimposed_img = tf.keras.preprocessing.image.array_to_img(superimposed_img)\n",
        "\n",
        "    # Save the superimposed image\n",
        "    superimposed_img.save(cam_path)\n",
        "\n",
        "    # Display Grad CAM\n",
        "    display(Image(cam_path))\n",
        "\n",
        "\n",
        "save_and_display_gradcam(input_path, ACT2, cam_path = 'img_both.png')"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.8"
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}