model_and_callback.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import time\n",
    "os.environ['KERAS_BACKEND'] = 'tensorflow'\n",
    "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "# import tensorflow as tf\n",
    "# from keras.backend.tensorflow_backend import set_session\n",
    "# config = tf.ConfigProto()\n",
    "# config.gpu_options.allow_growth = True\n",
    "# config.gpu_options.visible_device_list = \"0\"\n",
    "# set_session(tf.Session(config=config))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "from keras.callbacks import Callback\n",
    "from keras.engine import Layer\n",
    "from keras import layers\n",
    "from keras.layers import Input, LSTM, ZeroPadding2D, AveragePooling2D\n",
    "from keras.layers.core import Activation, Dense, Dropout, Flatten, Lambda, Reshape\n",
    "from keras.layers.convolutional import Conv2D, MaxPooling2D\n",
    "from keras.layers.merge import concatenate\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.layers.recurrent import LSTM\n",
    "from keras.layers.wrappers import TimeDistributed, Bidirectional\n",
    "from keras.models import Model\n",
    "from keras.utils import plot_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def identity_block(input_tensor, kernel_size, filters, stage, block):\n",
    "    \"\"\"The identity block is the block that has no conv layer at shortcut.\n",
    "    # Arguments\n",
    "        input_tensor: input tensor\n",
    "        kernel_size: defualt 3, the kernel size of middle conv layer at main path\n",
    "        filters: list of integers, the filterss of 3 conv layer at main path\n",
    "        stage: integer, current stage label, used for generating layer names\n",
    "        block: 'a','b'..., current block label, used for generating layer names\n",
    "    # Returns\n",
    "        Output tensor for the block.\n",
    "    \"\"\"\n",
    "    filters1, filters2, filters3 = filters\n",
    "    if K.image_data_format() == 'channels_last':\n",
    "        bn_axis = 3\n",
    "    else:\n",
    "        bn_axis = 1\n",
    "    conv_name_base = 'res' + str(stage) + block + '_branch'\n",
    "    bn_name_base = 'bn' + str(stage) + block + '_branch'\n",
    "\n",
    "    x = Conv2D(filters1, (1, 1), name=conv_name_base + '2a')(input_tensor)\n",
    "    x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)\n",
    "    x = Activation('relu')(x)\n",
    "\n",
    "    x = Conv2D(filters2, kernel_size,\n",
    "               padding='same', name=conv_name_base + '2b')(x)\n",
    "    x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)\n",
    "    x = Activation('relu')(x)\n",
    "\n",
    "    x = Conv2D(filters3, (1, 1), name=conv_name_base + '2c')(x)\n",
    "    x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)\n",
    "\n",
    "    x = layers.add([x, input_tensor])\n",
    "    x = Activation('relu')(x)\n",
    "    return x\n",
    "\n",
    "\n",
    "def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)):\n",
    "    \"\"\"conv_block is the block that has a conv layer at shortcut\n",
    "    # Arguments\n",
    "        input_tensor: input tensor\n",
    "        kernel_size: defualt 3, the kernel size of middle conv layer at main path\n",
    "        filters: list of integers, the filterss of 3 conv layer at main path\n",
    "        stage: integer, current stage label, used for generating layer names\n",
    "        block: 'a','b'..., current block label, used for generating layer names\n",
    "    # Returns\n",
    "        Output tensor for the block.\n",
    "    Note that from stage 3, the first conv layer at main path is with strides=(2,2)\n",
    "    And the shortcut should have strides=(2,2) as well\n",
    "    \"\"\"\n",
    "    filters1, filters2, filters3 = filters\n",
    "    if K.image_data_format() == 'channels_last':\n",
    "        bn_axis = 3\n",
    "    else:\n",
    "        bn_axis = 1\n",
    "    conv_name_base = 'res' + str(stage) + block + '_branch'\n",
    "    bn_name_base = 'bn' + str(stage) + block + '_branch'\n",
    "\n",
    "    x = Conv2D(filters1, (1, 1), strides=strides,\n",
    "               name=conv_name_base + '2a')(input_tensor)\n",
    "    x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)\n",
    "    x = Activation('relu')(x)\n",
    "\n",
    "    x = Conv2D(filters2, kernel_size, padding='same',\n",
    "               name=conv_name_base + '2b')(x)\n",
    "    x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)\n",
    "    x = Activation('relu')(x)\n",
    "\n",
    "    x = Conv2D(filters3, (1, 1), name=conv_name_base + '2c')(x)\n",
    "    x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)\n",
    "\n",
    "    shortcut = Conv2D(filters3, (1, 1), strides=strides,\n",
    "                      name=conv_name_base + '1')(input_tensor)\n",
    "    shortcut = BatchNormalization(axis=bn_axis, name=bn_name_base + '1')(shortcut)\n",
    "\n",
    "    x = layers.add([x, shortcut])\n",
    "    x = Activation('relu')(x)\n",
    "    return x\n",
    "\n",
    "def ResNet50(input_shape=(224,224,3), bottom_identity_layer=True):\n",
    "    \n",
    "    img_input = Input(shape=input_shape)\n",
    "\n",
    "    x = ZeroPadding2D((3, 3))(img_input)\n",
    "    x = Conv2D(64, (7, 7), strides=(2, 2), name='conv1')(x)\n",
    "    x = BatchNormalization(axis=3, name='bn_conv1')(x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = MaxPooling2D((3, 3), strides=(2, 2))(x)\n",
    "\n",
    "    x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))\n",
    "    x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')\n",
    "    x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')\n",
    "\n",
    "    x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')\n",
    "    x = identity_block(x, 3, [128, 128, 512], stage=3, block='b')\n",
    "    x = identity_block(x, 3, [128, 128, 512], stage=3, block='c')\n",
    "    x = identity_block(x, 3, [128, 128, 512], stage=3, block='d')\n",
    "\n",
    "    x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')\n",
    "    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b')\n",
    "    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c')\n",
    "    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d')\n",
    "    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e')\n",
    "    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f')\n",
    "\n",
    "    if bottom_identity_layer:\n",
    "        x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')\n",
    "        x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')\n",
    "        x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')\n",
    "\n",
    "    x = AveragePooling2D((x._keras_shape[-2], x._keras_shape[-2]), name='avg_pool')(x)\n",
    "\n",
    "    inputs = img_input\n",
    "    model = Model(inputs, x, name='resnet50')\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LRN2D(Layer):\n",
    "    def __init__(self, alpha=1e-4, k=2, beta=0.75, n=5, **kwargs):\n",
    "        if n % 2 == 0:\n",
    "            raise NotImplementedError(\"LRN2D only works with odd n. n provided: \" + str(n))\n",
    "        super(LRN2D, self).__init__(**kwargs)\n",
    "        #super(LRN2D, self).all(*args, **kwargs)\n",
    "        self.alpha = alpha\n",
    "        self.k = k\n",
    "        self.beta = beta\n",
    "        self.n = n\n",
    "\n",
    "    def get_output(self, train):\n",
    "        X = self.get_input(train)\n",
    "        b, ch, r, c = K.shape(X)\n",
    "        half_n = self.n // 2\n",
    "        input_sqr = K.square(X)\n",
    "\n",
    "        extra_channels = K.zeros((b, ch + 2 * half_n, r, c))\n",
    "        input_sqr = K.concatenate([extra_channels[:, :half_n, :, :],\n",
    "                                   input_sqr,\n",
    "                                   extra_channels[:, half_n + ch:, :, :]],\n",
    "                                  axis=1)\n",
    "        scale = self.k\n",
    "\n",
    "        for i in range(self.n):\n",
    "            scale += self.alpha * input_sqr[:, i:i + ch, :, :]\n",
    "        scale = scale ** self.beta\n",
    "\n",
    "        return X / scale\n",
    "\n",
    "    def get_config(self):\n",
    "        config = {\"name\": self.__class__.__name__,\n",
    "                  \"alpha\": self.alpha,\n",
    "                  \"k\": self.k,\n",
    "                  \"beta\": self.beta,\n",
    "                  \"n\": self.n}\n",
    "        base_config = super(LRN2D, self).get_config()\n",
    "        return dict(list(base_config.items()) + list(config.items()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def custom_lstm_layers(model,\n",
    "                       lstm_layers,\n",
    "                       many_to_many=False,\n",
    "                       lstm_name='lstm_'):\n",
    "    '''\n",
    "    Defines custom lstm layers\n",
    "\n",
    "    #Arguments\n",
    "\n",
    "    model:\n",
    "        Previous model to which lstm layers are to be attached\n",
    "\n",
    "    lstm_layers:\n",
    "        A list of tuples: (x, y, z)\n",
    "            x - Units in lstm layers\n",
    "            y - Bidirectional status\n",
    "            z - Merge mode, TODO: None mode not supported\n",
    "        Example [512] (Only 1 LSTM layer with 512 units)\n",
    "        TODO: Add GRU as well\n",
    "\n",
    "    many_to_many:\n",
    "        True - many_to_may\n",
    "        False - many_to_one\n",
    "        Last LSTM layer output sequence\n",
    "    '''\n",
    "\n",
    "    # Add LSTM layers stacked on top of each other\n",
    "    for index, cell in enumerate(lstm_layers):\n",
    "        units, bi, bi_mode = cell\n",
    "        lstm_layer_name = lstm_name + str(index + 1) + '_' + str(units)\n",
    "        # None bi_mode not supported\n",
    "        if not bi_mode: bi_mode = 'concat'\n",
    "        \n",
    "        if bi:\n",
    "            model = Bidirectional(LSTM(units,\n",
    "                                       return_sequences=many_to_many or (len(lstm_layers) != (index + 1)),\n",
    "                                       name=lstm_layer_name),\n",
    "                                  merge_mode=bi_mode,\n",
    "                                  name='bi_' + lstm_layer_name)(model)\n",
    "        else:\n",
    "            model = LSTM(units,\n",
    "                             return_sequences=many_to_many or (len(lstm_layers) != index + 1),\n",
    "                             name=lstm_layer_name)(model)\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lstm_model(input_shape,\n",
    "               resnet = False,\n",
    "               dense_feature_to_compare=1024,\n",
    "               drop_feature=0.5,\n",
    "               compare_layer=False,\n",
    "               compare_layer_dense=1024,\n",
    "               drop_compare_dense=0.5,\n",
    "               lstm_cells=[(512, False, 'concat')],\n",
    "               many_to_many=False,\n",
    "               dense_layer_after_lstm=128,\n",
    "               multi_output=False,\n",
    "               number_of_classes=12,\n",
    "               model_name='lstm_model',\n",
    "               test_mode=False):\n",
    "    '''\n",
    "    #Arguments\n",
    "\n",
    "    input_shape:\n",
    "        The input for the CNN\n",
    "        \n",
    "    resnet:\n",
    "        Use resnet or caffe model for image feature extraction\n",
    "\n",
    "    dense_feature_to_compare:\n",
    "        Add a dense layer after image features, specifying a 0 won't add one\n",
    "\n",
    "    drop_feature:\n",
    "        drop layer after feature map, 0 or 0.0 will cancel the layer\n",
    "\n",
    "    compare_layer:\n",
    "        True - In a tensor v(:, dim, :) compares layer (dim, dim+1)\n",
    "        TODO: Only works for dim=4\n",
    "\n",
    "    compare_layer_dense:\n",
    "        Adds a dense layer after concat step of compare_layer, specifying a 0\n",
    "        won't add one\n",
    "\n",
    "    drop_compare_dense:\n",
    "        Add a dense layer for compare dense layer, specifying a 0 won't add one\n",
    "\n",
    "    lstm_cells[\n",
    "    \n",
    "    ]\n",
    "        A list of tuples: (x, y, z)\n",
    "            x - Units in lstm layers\n",
    "            y - Bidirectional status\n",
    "            z - Merge mode, TODO: None mode not supported\n",
    "        Example [512] (Only 1 LSTM layer with 512 units)\n",
    "        TODO: Add GRU as well\n",
    "    \n",
    "    many_to_many:\n",
    "        True - many_to_may\n",
    "        False - many_to_one\n",
    "        Last LSTM layer output sequence\n",
    "\n",
    "    dense_layer_after_lstm:\n",
    "        If Integer given adds a Dense layer on top of the output of LSTM\n",
    "        If 0 is given no layer is added\n",
    "\n",
    "    multi_output:\n",
    "        True - Have multiple outputs\n",
    "\n",
    "    number_of_classes:\n",
    "        Number of output classes\n",
    "\n",
    "    #NamingFormat - Caffe Net TimeDistributed\n",
    "    <layer><depth>_<filter>_<kernel>_<stride>\n",
    "        \n",
    "    -------------------- *** Caffe Net - TimeDistributed *** ------------------\n",
    "    \n",
    "    Original CaffeNet model, modified as below\n",
    "    Changes Made:   (Original Copy)\n",
    "    conv1_96_11_4   kernel_size=(11, 11), strides=(4, 4)\n",
    "    flatten6        after this layer customized for problem\n",
    "\n",
    "    INPUT (None, TD, W, H, C)\n",
    "    =========================\n",
    "    conv1_96_7_3\n",
    "    mpool1_3_2\n",
    "    lrnorm1\n",
    "    conv2_256_3_1\n",
    "    mpool2_3_2\n",
    "    lrnorm2\n",
    "    conv3_384_3_1\n",
    "    bn3\n",
    "    relu3\n",
    "    conv4_384_3_1\n",
    "    bn4\n",
    "    relu4\n",
    "    conv5_256_3_1\n",
    "    bn5\n",
    "    relu5\n",
    "    mpool5_3_2\n",
    "    flatten6\n",
    "    =========================\n",
    "\n",
    "    -------------------- *** Caffe Net - TimeDistributed *** ------------------\n",
    "\n",
    "    #NamingFormat - LSTM Configurable Model\n",
    "    [] Means a stack of layers, one after another (depth/height)\n",
    "    ? Means the parameter is configurable by the value specified in if condition\n",
    "    {TD} Means the output of that layer may or may not be in time dimension\n",
    "\n",
    "    ---------------------- *** LSTM Configurable Model *** --------------------\n",
    "    \n",
    "    INPUT (None, TD, C)\n",
    "    ===================\n",
    "    if dense_feature_to_compare:\n",
    "        dense6_?\n",
    "        bn6\n",
    "        relu6\n",
    "    if drop_feature:\n",
    "        drop6_?\n",
    "    if compare_layer:\n",
    "        concat_01\n",
    "        concat_12\n",
    "        concat_23\n",
    "        concat_03\n",
    "        if compare_layer_dense:\n",
    "            dense_01_?\n",
    "            bn_01\n",
    "            relu_01\n",
    "            dense_12_?\n",
    "            bn_12\n",
    "            relu_12\n",
    "            dense_23_?\n",
    "            bn_23\n",
    "            relu_23\n",
    "            if drop_compare_dense:\n",
    "                dropout_01_?\n",
    "                dropout_12_?\n",
    "                dropout_23_?\n",
    "    [lstm_sort_?]\n",
    "    if dense_layer_after_lstm:\n",
    "        dense7_lstm\n",
    "    if many_to_many and not multi_output:\n",
    "        flatten7\n",
    "    lstm_catg{TD}\n",
    "    lstm_output{TD}\n",
    "    ===================\n",
    "\n",
    "    ---------------------- *** LSTM Configurable Model *** --------------------\n",
    "    '''\n",
    "    print \"Model Creation Started\"\n",
    "    \n",
    "    # Some sanity checks\n",
    "    if multi_output: many_to_many=True\n",
    "    \n",
    "    # Input variables used in model\n",
    "    compare_layer_image_dim = None\n",
    "    \n",
    "    input_layer = Input((input_shape), name='input')\n",
    "    \n",
    "    # Resnet or Caffe Model for feature extraction\n",
    "    if resnet:\n",
    "        resNet = ResNet50(input_shape=input_shape[1:], bottom_identity_layer=False)\n",
    "        model = TimeDistributed(resNet)(input_layer)\n",
    "    \n",
    "    else:\n",
    "        # conv1_96_7_3\n",
    "        model = TimeDistributed(Conv2D(96, \n",
    "                                       kernel_size=(11, 11),\n",
    "                                       strides=(3, 3),\n",
    "                                       activation='relu',\n",
    "                                       padding='valid',\n",
    "                                       name='conv1_96_7_3'),\n",
    "                                name='TD_conv1_96_7_3')(input_layer)\n",
    "\n",
    "        # mpool1_3_2\n",
    "        model = TimeDistributed(MaxPooling2D(pool_size=(3, 3), \n",
    "                                             strides=(2, 2),\n",
    "                                             padding='valid',\n",
    "                                             name='mpool1_3_2'),\n",
    "                                name='TD_mpool1_3_2')(model)\n",
    "\n",
    "        # lrnorm1\n",
    "        model = TimeDistributed(LRN2D(name='lrnorm1'), name='TD_lrnorm1')(model)\n",
    "\n",
    "        # conv2_256_3_1\n",
    "        model = TimeDistributed(Conv2D(256,\n",
    "                                       kernel_size=(5, 5),\n",
    "                                       strides=(1, 1),\n",
    "                                       padding='same',\n",
    "                                       activation='relu',\n",
    "                                       name='conv2_256_3_1'),\n",
    "                                name='TD_conv2_256_3_1')(model)\n",
    "\n",
    "        # mpool2_3_2\n",
    "        model = TimeDistributed(MaxPooling2D(pool_size=(3, 3),\n",
    "                                             strides=(2, 2),\n",
    "                                             padding='valid',\n",
    "                                             name='mpool2_3_2'),\n",
    "                                name='TD_mpool2_3_2')(model)\n",
    "\n",
    "        # lrnorm2\n",
    "        model = TimeDistributed(LRN2D(name='lrnorm2'), name='TD_lrnorm2')(model)\n",
    "\n",
    "        # conv3_, bn, relu <3,4,5>\n",
    "        filters = [384, 384, 256]\n",
    "        for layer in xrange(3, 6, 1):\n",
    "            conv_step_name = 'conv' + str(layer) + '_' + str(filters[layer - 3]) + '_3_1'\n",
    "\n",
    "            model = TimeDistributed(Conv2D(filters[layer - 3],\n",
    "                                           kernel_size=(3, 3),\n",
    "                                           strides=(1, 1),\n",
    "                                           padding='same',\n",
    "                                           name=conv_step_name),\n",
    "                                    name='TD_' + conv_step_name)(model)\n",
    "\n",
    "            model = TimeDistributed(BatchNormalization(name='bn_' + str(layer)), \n",
    "                                    name='TD_bn_' + str(layer))(model)\n",
    "\n",
    "            model = TimeDistributed(Activation('relu', name='relu' + str(layer)),\n",
    "                                    name='TD_relu_' + str(layer))(model)\n",
    "\n",
    "\n",
    "        # mpool5_3_2\n",
    "        model = TimeDistributed(MaxPooling2D(pool_size=(3, 3),\n",
    "                                             strides=(2, 2),\n",
    "                                             padding='valid',\n",
    "                                             name='mpool5_3_2'),\n",
    "                                name='TD_mpool5_3_2')(model)\n",
    "\n",
    "    # flatten6, feature map from CNN\n",
    "    model = TimeDistributed(Flatten(name='flatten6'), name='TD_flatten6')(model)\n",
    "\n",
    "    # If no dense layer specified below on feature map get this layer dimension\n",
    "    if not dense_feature_to_compare: compare_layer_image_dim = model._keras_shape[-1]\n",
    "\n",
    "    print \"Model Base Created\"\n",
    "    \n",
    "    # Add a dense layer to feature of images\n",
    "    if dense_feature_to_compare:\n",
    "        # Update compare_layer_image_dim, if using compare_layer in model\n",
    "        compare_layer_image_dim = dense_feature_to_compare\n",
    "        model = TimeDistributed(Dense(dense_feature_to_compare,\n",
    "                                      name='dense6_' + str(dense_feature_to_compare)),\n",
    "                                name='TD_dense6_' + str(dense_feature_to_compare))(model)\n",
    "        \n",
    "        # bn6\n",
    "        model = TimeDistributed(BatchNormalization(name='bn6'),\n",
    "                                name='TD_bn6')(model)\n",
    "\n",
    "        # relu6\n",
    "        model = TimeDistributed(Activation('relu', name='relu6'),\n",
    "                                name='TD_relu6')(model)\n",
    "\n",
    "    if drop_feature:\n",
    "        # drop6\n",
    "        model = TimeDistributed(Dropout(drop_feature, name='drop6'),\n",
    "                                name='TD_drop6' + str(drop_feature))(model)\n",
    "\n",
    "    # Introducing LSTM compare layer\n",
    "    if compare_layer:\n",
    "        # image_<0,1,2,3>\n",
    "        image_f = [Lambda(lambda model : model[:, frame, :], \n",
    "                          output_shape=(compare_layer_image_dim,),\n",
    "                          name='image_' + str(frame))(model) \n",
    "                   for frame in xrange(4)]\n",
    "\n",
    "        # compare <01,12,23> \n",
    "        concat_01 = concatenate([\n",
    "                                    image_f[0],\n",
    "                                    image_f[1]\n",
    "                                ],\n",
    "                                name='compare_01',\n",
    "                                axis=-1)\n",
    "        \n",
    "        concat_12 = concatenate([\n",
    "                                    image_f[1],\n",
    "                                    image_f[2]\n",
    "                                ],\n",
    "                                name='compare_12',\n",
    "                                axis=-1)\n",
    "\n",
    "        concat_23 = concatenate([\n",
    "                                    image_f[2],\n",
    "                                    image_f[3]\n",
    "                                ], \n",
    "                                name='compare_23',\n",
    "                                axis=-1)\n",
    "\n",
    "        concat_03 = concatenate([\n",
    "                                    image_f[0],\n",
    "                                    image_f[3]\n",
    "                                ], \n",
    "                                name='compare_03',\n",
    "                                axis=-1)\n",
    "\n",
    "        if compare_layer_dense:\n",
    "            # dense<01,12,23>_7\n",
    "            concat_01 = Dense(compare_layer_dense,\n",
    "                              name=\"dense_01_7\")(concat_01)\n",
    "            concat_12 = Dense(compare_layer_dense,\n",
    "                              name=\"dense_12_7\")(concat_12)\n",
    "            concat_23 = Dense(compare_layer_dense,\n",
    "                              name=\"dense_23_7\")(concat_23)\n",
    "            concat_03 = Dense(compare_layer_dense,\n",
    "                              name=\"dense_03_7\")(concat_03)\n",
    "            \n",
    "            concat_01 = BatchNormalization(name='bn_01_7')(concat_01)\n",
    "            concat_12 = BatchNormalization(name='bn_12_7')(concat_12)\n",
    "            concat_23 = BatchNormalization(name='bn_23_7')(concat_23)\n",
    "            concat_03 = BatchNormalization(name='bn_03_7')(concat_03)\n",
    "\n",
    "            concat_01 = Activation('relu',\n",
    "                                   name='relu_01_7')(concat_01)\n",
    "            concat_12 = Activation('relu',\n",
    "                                  name='relu_12_7')(concat_12)\n",
    "            concat_23 = Activation('relu',\n",
    "                                  name='relu_23_7')(concat_23)\n",
    "            concat_03 = Activation('relu',\n",
    "                                  name='relu_03_7')(concat_03)\n",
    "            \n",
    "            if drop_compare_dense:\n",
    "                concat_01 = Dropout(drop_compare_dense, \n",
    "                                    name=\"dropout_01_7_\" + str(drop_compare_dense))(concat_01)\n",
    "                concat_12 = Dropout(drop_compare_dense,\n",
    "                                    name=\"dropout_12_7_\" + str(drop_compare_dense))(concat_12)\n",
    "                concat_23 = Dropout(drop_compare_dense,\n",
    "                                    name=\"dropout_23_7_\" + str(drop_compare_dense))(concat_23)\n",
    "                concat_03 = Dropout(drop_compare_dense,\n",
    "                                    name=\"dropout_03_7_\" + str(drop_compare_dense))(concat_23)\n",
    "\n",
    "        # Concat the layers into 3 transitive units\n",
    "        concat_0123 = concatenate([\n",
    "                                      concat_01,\n",
    "                                      concat_12,\n",
    "                                      concat_23,\n",
    "                                      concat_03,\n",
    "                                  ],\n",
    "                                  name='concat_0123',\n",
    "                                  axis=-1)\n",
    "        \n",
    "        # If LSTM layer exits reshape it for lstm input\n",
    "        if lstm_cells[0][0]:\n",
    "            if compare_layer_dense:\n",
    "                model = Reshape(target_shape=(4, compare_layer_dense),\n",
    "                                name='reshape_' + str(compare_layer_dense))(concat_0123)\n",
    "            else:\n",
    "                model = Reshape(target_shape=(4, concat_01._keras_shape),\n",
    "                                name='reshape_' + str(concat_01._keras_shape))(concat_0123)\n",
    "        else: model = concat_0123\n",
    "\n",
    "    print \"Model Compare Unit Finished\"\n",
    "    \n",
    "    if lstm_cells[0][0]:\n",
    "        # Add LSTM layers stacked on top of each other\n",
    "        model = custom_lstm_layers(model, lstm_cells, many_to_many, lstm_name='lstm_sort_')\n",
    "\n",
    "    print \"Model LSTM Unit Finished\"\n",
    "    \n",
    "    # Add dense layer after lstm\n",
    "    if dense_layer_after_lstm:\n",
    "        if many_to_many:\n",
    "            # Add dense on many to many\n",
    "            model = TimeDistributed(Dense(dense_layer_after_lstm, name='dense7_lstm'), name='TD_dense7_lstm')(model)\n",
    "        else:\n",
    "            # Add dense on many to many\n",
    "            model = Dense(dense_layer_after_lstm, name='dense7_lstm')(model)\n",
    "\n",
    "    # Many output of LSTM enabled and no multi output then flatten layer\n",
    "    if many_to_many and not multi_output: model = Flatten(name='flatten7')(model)\n",
    "    \n",
    "    print \"Model Output Layers Finished\"\n",
    "    \n",
    "    # Class Output with respect to multiple outputs\n",
    "    if not multi_output:    \n",
    "        model = Dense(number_of_classes, name='lstm_catg')(model)\n",
    "        model = Activation('sigmoid', name='output')(model)\n",
    "    else:\n",
    "        model = TimeDistributed(Dense(number_of_classes, name='lstm_catg'), name='TD_lstm_catg')(model)\n",
    "        model = TimeDistributed(Activation('softmax', name='lstm_output'), name='TD_output')(model)\n",
    "    \n",
    "    if test_mode:\n",
    "        model = Model(inputs=[input_layer], outputs=[model])\n",
    "        #plot_model(model, to_file=model_name + '.png')\n",
    "        print model.summary()\n",
    "        \n",
    "    print \"Model Creation Finished\"\n",
    "\n",
    "    return input_layer, model\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "*For Testing the model*\n",
    "```\n",
    "model = lstm_model(\n",
    "                input_shape=(4, 80, 80, 3),\n",
    "                resnet=False,\n",
    "                dense_feature_to_compare=0,\n",
    "                drop_feature=0,\n",
    "                compare_layer=True,\n",
    "                compare_layer_dense=512,\n",
    "                drop_compare_dense=0.25,\n",
    "                lstm_cells=[(256, True, 'mul')],\n",
    "                many_to_many=False,\n",
    "                dense_layer_after_lstm=0,\n",
    "                multi_output=False,\n",
    "                number_of_classes=2,\n",
    "                model_name='chrono_lstm',\n",
    "                test_mode=True\n",
    "          )\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EpochCallback(Callback):\n",
    "    def __init__(self, model_name, save_dir_path, epoch_size, initial_iteration=0):\n",
    "        self.name = model_name\n",
    "        self.epoch_run_time = time.time()\n",
    "        self.total_batch_run_time = 0.0\n",
    "        self.batch_run_time = 0\n",
    "        self.dir = os.path.join(save_dir_path, 'weights')\n",
    "        self.initial_iteration = initial_iteration\n",
    "        self.epoch_size = epoch_size\n",
    "        if not os.path.exists(self.dir):\n",
    "            os.makedirs(self.dir)\n",
    "            \n",
    "    def on_train_begin(self, logs=None):\n",
    "        print \"\\n->TRAINING STARTING\"\n",
    "        if self.initial_iteration:\n",
    "            K.set_value(self.model.optimizer.iterations, self.initial_iteration)\n",
    "            print \"Initial Iteration Set To\", self.initial_iteration, \"\\n\"\n",
    "            \n",
    "    def on_epoch_begin(self, epoch, logs={}):\n",
    "        self.epoch_run_time = time.time()\n",
    "        \n",
    "    def on_batch_begin(self, batch, logs={}):\n",
    "        self.batch_run_time = time.time()\n",
    "        \n",
    "    def on_batch_end(self, batch, logs={}):\n",
    "        self.total_batch_run_time += time.time() - self.batch_run_time\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        self.model.save_weights(os.path.join(self.dir, \"EPOCH_\" + str(epoch + 1) + \".h5\"))\n",
    "        optimizer = self.model.optimizer\n",
    "        lr = K.eval(tf.to_float(optimizer.lr) * (1. / (1. + tf.to_float(optimizer.decay) * tf.to_float(optimizer.iterations))))\n",
    "        print \"\\n\", self.name, \"| EPOCH\", str(epoch + 1), \"ENDS\"\n",
    "        print \"Epoch weights saved\", os.path.join(self.dir, \"EPOCH_\" + str(epoch + 1) + \".h5\")\n",
    "        print \"AVG BATCH GPU TIME | Batches\", str(self.epoch_size), \"| processed with time : \", str(self.total_batch_run_time / float(self.epoch_size))\n",
    "        print \"Epoch\", str(epoch + 1), \"Completed | Time taken\", str(time.time() - self.epoch_run_time)\n",
    "        print 'LEARNING RATE: {:.6f}'.format(lr), \"| ITERATIONS:\", K.eval(optimizer.iterations), \"| DECAY:\", K.eval(optimizer.decay), \"\\n\\n\"\n",
    "        self.total_batch_run_time = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from keras.applications.resnet50 import ResNet50\n",
    "\n",
    "# resNet = ResNet50(weights=None, include_top=False)\n",
    "\n",
    "# input_layer = Input(shape=(4, 80, 80, 3))\n",
    "# curr_layer = TimeDistributed(resNet)(input_layer)\n",
    "# # curr_layer = Reshape(target_shape=(4, 2048))(curr_layer)\n",
    "# # curr_layer = LSTM(384, return_sequences=False)(curr_layer)\n",
    "# # curr_layer = Dense(1)(curr_layer)\n",
    "# model = Model(inputs=[input_layer], outputs=[curr_layer])\n",
    "# print model.summary()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:py_tensor]",
   "language": "python",
   "name": "conda-env-py_tensor-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}