14-full-res-model-1920-1080.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": "true"
   },
   "source": [
    " # Table of Contents\n",
    "<div class=\"toc\" style=\"margin-top: 1em;\"><ul class=\"toc-item\" id=\"toc-level0\"><li><span><a href=\"http://localhost:8889/notebooks/14-full-res-model-v2.ipynb#Load-libraries\" data-toc-modified-id=\"Load-libraries-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Load libraries</a></span></li><li><span><a href=\"http://localhost:8889/notebooks/14-full-res-model-v2.ipynb#Define-loss-functions\" data-toc-modified-id=\"Define-loss-functions-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Define loss functions</a></span></li><li><span><a href=\"http://localhost:8889/notebooks/14-full-res-model-v2.ipynb#Define-models\" data-toc-modified-id=\"Define-models-3\"><span class=\"toc-item-num\">3&nbsp;&nbsp;</span>Define models</a></span></li><li><span><a href=\"http://localhost:8889/notebooks/14-full-res-model-v2.ipynb#Save-predictions-of-512-model\" data-toc-modified-id=\"Save-predictions-of-512-model-4\"><span class=\"toc-item-num\">4&nbsp;&nbsp;</span>Save predictions of 512 model</a></span></li><li><span><a href=\"http://localhost:8889/notebooks/14-full-res-model-v2.ipynb#Full-res-model\" data-toc-modified-id=\"Full-res-model-5\"><span class=\"toc-item-num\">5&nbsp;&nbsp;</span>Full res model</a></span></li><li><span><a href=\"http://localhost:8889/notebooks/14-full-res-model-v2.ipynb#Training\" data-toc-modified-id=\"Training-6\"><span class=\"toc-item-num\">6&nbsp;&nbsp;</span>Training</a></span></li><li><span><a href=\"http://localhost:8889/notebooks/14-full-res-model-v2.ipynb#Full-res-model-2\" data-toc-modified-id=\"Full-res-model-2-7\"><span class=\"toc-item-num\">7&nbsp;&nbsp;</span>Full res model 2</a></span></li></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-07T07:32:24.349721Z",
     "start_time": "2017-09-07T07:32:10.527862Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import cv2\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint, TensorBoard\n",
    "from keras.models import Model\n",
    "from keras.layers import Input, concatenate, Conv2D, MaxPooling2D, Activation, UpSampling2D, BatchNormalization\n",
    "from keras.optimizers import RMSprop\n",
    "from keras.losses import binary_crossentropy\n",
    "import keras.backend as K\n",
    "\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-07T07:32:24.480919Z",
     "start_time": "2017-09-07T07:32:24.358896Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import random\n",
    "import gzip\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define loss functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-07T07:32:24.497301Z",
     "start_time": "2017-09-07T07:32:24.489981Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def dice_coeff(y_true, y_pred):\n",
    "    smooth = 1.\n",
    "    y_true_f = K.flatten(y_true)\n",
    "    y_pred_f = K.flatten(y_pred)\n",
    "    intersection = K.sum(y_true_f * y_pred_f)\n",
    "    score = (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n",
    "    return score\n",
    "\n",
    "\n",
    "def dice_loss(y_true, y_pred):\n",
    "    loss = 1 - dice_coeff(y_true, y_pred)\n",
    "    return loss\n",
    "\n",
    "def bce_dice_loss(y_true, y_pred):\n",
    "    loss = binary_crossentropy(y_true, y_pred) + dice_loss(y_true, y_pred)\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-07T07:32:24.554423Z",
     "start_time": "2017-09-07T07:32:24.506618Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def unet_down_one_block(inputs, num_filters):\n",
    "    x = Conv2D(num_filters, (3, 3), padding='same')(inputs)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = Conv2D(num_filters, (3, 3), padding='same')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('relu')(x)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-07T07:32:24.598503Z",
     "start_time": "2017-09-07T07:32:24.563838Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def unet_max_pool(inputs):\n",
    "    x = MaxPooling2D((2, 2), strides=(2, 2))(inputs)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-07T07:32:24.679604Z",
     "start_time": "2017-09-07T07:32:24.672023Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def unet_up_one_block(up_input, down_input, num_filters):\n",
    "    x = UpSampling2D((2,2))(up_input)\n",
    "    x = concatenate([down_input, x], axis=3)\n",
    "    x = Conv2D(num_filters, (3,3), padding='same')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = Conv2D(num_filters, (3,3), padding='same')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = Conv2D(num_filters, (3,3), padding='same')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('relu')(x)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-07T07:32:24.724212Z",
     "start_time": "2017-09-07T07:32:24.690133Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_unet(input_shape = (256, 256, 3),\n",
    "             num_classes = 1,\n",
    "             initial_filters = 32,\n",
    "             central_filters = 1024):\n",
    "    \n",
    "    num_filters = initial_filters\n",
    "    \n",
    "    out_list    = [Input(shape=input_shape)]\n",
    "    down_interim_list = []\n",
    "    \n",
    "    while num_filters <= central_filters/2:\n",
    "        x = unet_down_one_block(out_list[-1], num_filters)\n",
    "        down_interim_list.append(x)\n",
    "        num_filters = num_filters * 2\n",
    "        y = unet_max_pool(x)\n",
    "        out_list.append(y)\n",
    "    \n",
    "    x = unet_down_one_block(out_list[-1], num_filters)\n",
    "    out_list.append(x)\n",
    "    num_filters = int(num_filters / 2)\n",
    "    \n",
    "    while num_filters >= initial_filters:\n",
    "        x = unet_up_one_block(out_list[-1], down_interim_list.pop(), num_filters)\n",
    "        out_list.append(x)\n",
    "        num_filters = int(num_filters / 2)\n",
    "    \n",
    "    classify = Conv2D(num_classes, (1,1), activation = 'sigmoid')(out_list[-1])\n",
    "    \n",
    "    model = Model(inputs=out_list[0], outputs=classify)\n",
    "    \n",
    "    model.compile(optimizer=RMSprop(lr=0.0001),\n",
    "                  loss=bce_dice_loss,\n",
    "                  metrics=[dice_coeff])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Save predictions of 512 model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-06T13:22:39.784295Z",
     "start_time": "2017-09-06T13:22:38.379307Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "model_512 = get_unet(input_shape=(512,512,3), initial_filters=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-06T13:22:55.553577Z",
     "start_time": "2017-09-06T13:22:39.789832Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "model_512.load_weights('./weights/best_weights_512_2.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-06T13:22:55.561627Z",
     "start_time": "2017-09-06T13:22:55.559581Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "input_size=512"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-06T13:22:56.021875Z",
     "start_time": "2017-09-06T13:22:55.566960Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from os import listdir\n",
    "ids_list = [x.split('.')[0] for x in listdir('data/train')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-06T13:26:51.547810Z",
     "start_time": "2017-09-06T13:23:01.352780Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "all_imgs=[]\n",
    "for id in ids_list:\n",
    "    img = cv2.imread('data/train/{}.jpg'.format(id))\n",
    "    img = cv2.resize(img, (input_size, input_size))\n",
    "    all_imgs.append(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-06T13:33:43.087236Z",
     "start_time": "2017-09-06T13:27:19.506601Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "preds = model_512.predict(np.array(all_imgs, np.float32)/255, batch_size=16)\n",
    "preds = np.squeeze(preds, axis=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-06T13:34:18.348603Z",
     "start_time": "2017-09-06T13:34:18.344023Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5088, 512, 512)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-06T13:51:43.878252Z",
     "start_time": "2017-09-06T13:34:25.313172Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "for i in range(len(ids_list)):\n",
    "    this_id = ids_list[i]\n",
    "    this_pred = np.copy(preds[i,:,:])\n",
    "    f = gzip.open('./data/preds512/' + this_id+ '.pklz','wb')\n",
    "    pickle.dump(this_pred, f)\n",
    "    f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Full res model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-06T13:59:23.503502Z",
     "start_time": "2017-09-06T13:59:22.761703Z"
    },
    "collapsed": true,
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model_fullres =  get_unet(input_shape=(1280,1920,4), initial_filters=8, central_filters=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-06T13:59:23.541192Z",
     "start_time": "2017-09-06T13:59:23.516915Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_1 (InputLayer)             (None, 1280, 1920, 4) 0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)                (None, 1280, 1920, 8) 296         input_1[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNorm (None, 1280, 1920, 8) 32          conv2d_1[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "activation_1 (Activation)        (None, 1280, 1920, 8) 0           batch_normalization_1[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)                (None, 1280, 1920, 8) 584         activation_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNorm (None, 1280, 1920, 8) 32          conv2d_2[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "activation_2 (Activation)        (None, 1280, 1920, 8) 0           batch_normalization_2[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)   (None, 640, 960, 8)   0           activation_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)                (None, 640, 960, 16)  1168        max_pooling2d_1[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNorm (None, 640, 960, 16)  64          conv2d_3[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "activation_3 (Activation)        (None, 640, 960, 16)  0           batch_normalization_3[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)                (None, 640, 960, 16)  2320        activation_3[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNorm (None, 640, 960, 16)  64          conv2d_4[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "activation_4 (Activation)        (None, 640, 960, 16)  0           batch_normalization_4[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2D)   (None, 320, 480, 16)  0           activation_4[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)                (None, 320, 480, 32)  4640        max_pooling2d_2[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNorm (None, 320, 480, 32)  128         conv2d_5[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "activation_5 (Activation)        (None, 320, 480, 32)  0           batch_normalization_5[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)                (None, 320, 480, 32)  9248        activation_5[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNorm (None, 320, 480, 32)  128         conv2d_6[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "activation_6 (Activation)        (None, 320, 480, 32)  0           batch_normalization_6[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2D)   (None, 160, 240, 32)  0           activation_6[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_7 (Conv2D)                (None, 160, 240, 64)  18496       max_pooling2d_3[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNorm (None, 160, 240, 64)  256         conv2d_7[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "activation_7 (Activation)        (None, 160, 240, 64)  0           batch_normalization_7[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_8 (Conv2D)                (None, 160, 240, 64)  36928       activation_7[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNorm (None, 160, 240, 64)  256         conv2d_8[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "activation_8 (Activation)        (None, 160, 240, 64)  0           batch_normalization_8[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "up_sampling2d_1 (UpSampling2D)   (None, 320, 480, 64)  0           activation_8[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)      (None, 320, 480, 96)  0           activation_6[0][0]               \n",
      "                                                                   up_sampling2d_1[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_9 (Conv2D)                (None, 320, 480, 32)  27680       concatenate_1[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_9 (BatchNorm (None, 320, 480, 32)  128         conv2d_9[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "activation_9 (Activation)        (None, 320, 480, 32)  0           batch_normalization_9[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_10 (Conv2D)               (None, 320, 480, 32)  9248        activation_9[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_10 (BatchNor (None, 320, 480, 32)  128         conv2d_10[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_10 (Activation)       (None, 320, 480, 32)  0           batch_normalization_10[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_11 (Conv2D)               (None, 320, 480, 32)  9248        activation_10[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_11 (BatchNor (None, 320, 480, 32)  128         conv2d_11[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_11 (Activation)       (None, 320, 480, 32)  0           batch_normalization_11[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "up_sampling2d_2 (UpSampling2D)   (None, 640, 960, 32)  0           activation_11[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)      (None, 640, 960, 48)  0           activation_4[0][0]               \n",
      "                                                                   up_sampling2d_2[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_12 (Conv2D)               (None, 640, 960, 16)  6928        concatenate_2[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_12 (BatchNor (None, 640, 960, 16)  64          conv2d_12[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_12 (Activation)       (None, 640, 960, 16)  0           batch_normalization_12[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_13 (Conv2D)               (None, 640, 960, 16)  2320        activation_12[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_13 (BatchNor (None, 640, 960, 16)  64          conv2d_13[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_13 (Activation)       (None, 640, 960, 16)  0           batch_normalization_13[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_14 (Conv2D)               (None, 640, 960, 16)  2320        activation_13[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_14 (BatchNor (None, 640, 960, 16)  64          conv2d_14[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_14 (Activation)       (None, 640, 960, 16)  0           batch_normalization_14[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "up_sampling2d_3 (UpSampling2D)   (None, 1280, 1920, 16 0           activation_14[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)      (None, 1280, 1920, 24 0           activation_2[0][0]               \n",
      "                                                                   up_sampling2d_3[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_15 (Conv2D)               (None, 1280, 1920, 8) 1736        concatenate_3[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_15 (BatchNor (None, 1280, 1920, 8) 32          conv2d_15[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_15 (Activation)       (None, 1280, 1920, 8) 0           batch_normalization_15[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_16 (Conv2D)               (None, 1280, 1920, 8) 584         activation_15[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_16 (BatchNor (None, 1280, 1920, 8) 32          conv2d_16[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_16 (Activation)       (None, 1280, 1920, 8) 0           batch_normalization_16[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_17 (Conv2D)               (None, 1280, 1920, 8) 584         activation_16[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_17 (BatchNor (None, 1280, 1920, 8) 32          conv2d_17[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_17 (Activation)       (None, 1280, 1920, 8) 0           batch_normalization_17[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_18 (Conv2D)               (None, 1280, 1920, 1) 9           activation_17[0][0]              \n",
      "====================================================================================================\n",
      "Total params: 135,969\n",
      "Trainable params: 135,153\n",
      "Non-trainable params: 816\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_fullres.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-06T13:59:24.534242Z",
     "start_time": "2017-09-06T13:59:23.554377Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href='model-fullres-v2.png' target='_blank'>model-fullres-v2.png</a><br>"
      ],
      "text/plain": [
       "/home/sainath/extvol/kaggle-carvana-image-masking-challenge/model-fullres-v2.png"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.utils import plot_model  \n",
    "plot_model(model_fullres, to_file='model-fullres-v2.png', show_shapes=True, show_layer_names=True)  \n",
    "from IPython.display import FileLink\n",
    "FileLink('model-fullres-v2.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-07T07:33:06.210115Z",
     "start_time": "2017-09-07T07:33:06.207688Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "threshold  = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-07T07:33:08.011812Z",
     "start_time": "2017-09-07T07:33:06.543313Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('data/train_masks.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-07T07:33:08.053843Z",
     "start_time": "2017-09-07T07:33:08.046673Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "ids_train = df_train['img'].map(lambda s: s.split('.')[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-07T07:33:08.111105Z",
     "start_time": "2017-09-07T07:33:08.090424Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "ids_train_split, ids_valid_split = train_test_split(ids_train, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-07T07:33:08.157209Z",
     "start_time": "2017-09-07T07:33:08.146094Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def randomHueSaturationValue(image, hue_shift_limit=(-180, 180),\n",
    "                             sat_shift_limit=(-255, 255),\n",
    "                             val_shift_limit=(-255, 255), u=0.5):\n",
    "    if np.random.random() < u:\n",
    "        image = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)\n",
    "        h, s, v = cv2.split(image)\n",
    "        hue_shift = np.random.uniform(hue_shift_limit[0], hue_shift_limit[1])\n",
    "        h = cv2.add(h, hue_shift)\n",
    "        sat_shift = np.random.uniform(sat_shift_limit[0], sat_shift_limit[1])\n",
    "        s = cv2.add(s, sat_shift)\n",
    "        val_shift = np.random.uniform(val_shift_limit[0], val_shift_limit[1])\n",
    "        v = cv2.add(v, val_shift)\n",
    "        image = cv2.merge((h, s, v))\n",
    "        image = cv2.cvtColor(image, cv2.COLOR_HSV2BGR)\n",
    "\n",
    "    return image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-07T07:33:08.465491Z",
     "start_time": "2017-09-07T07:33:08.438040Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def randomShiftScaleRotate(image, mask,\n",
    "                           shift_limit=(-0.0625, 0.0625),\n",
    "                           scale_limit=(-0.1, 0.1),\n",
    "                           rotate_limit=(-45, 45), aspect_limit=(0, 0),\n",
    "                           borderMode=cv2.BORDER_CONSTANT, u=0.5):\n",
    "    if np.random.random() < u:\n",
    "        height, width, channel = image.shape\n",
    "\n",
    "        angle = np.random.uniform(rotate_limit[0], rotate_limit[1])  # degree\n",
    "        scale = np.random.uniform(1 + scale_limit[0], 1 + scale_limit[1])\n",
    "        aspect = np.random.uniform(1 + aspect_limit[0], 1 + aspect_limit[1])\n",
    "        sx = scale * aspect / (aspect ** 0.5)\n",
    "        sy = scale / (aspect ** 0.5)\n",
    "        dx = round(np.random.uniform(shift_limit[0], shift_limit[1]) * width)\n",
    "        dy = round(np.random.uniform(shift_limit[0], shift_limit[1]) * height)\n",
    "\n",
    "        cc = np.math.cos(angle / 180 * np.math.pi) * sx\n",
    "        ss = np.math.sin(angle / 180 * np.math.pi) * sy\n",
    "        rotate_matrix = np.array([[cc, -ss], [ss, cc]])\n",
    "\n",
    "        box0 = np.array([[0, 0], [width, 0], [width, height], [0, height], ])\n",
    "        box1 = box0 - np.array([width / 2, height / 2])\n",
    "        box1 = np.dot(box1, rotate_matrix.T) + np.array([width / 2 + dx, height / 2 + dy])\n",
    "\n",
    "        box0 = box0.astype(np.float32)\n",
    "        box1 = box1.astype(np.float32)\n",
    "        mat = cv2.getPerspectiveTransform(box0, box1)\n",
    "        image = cv2.warpPerspective(image, mat, (width, height), flags=cv2.INTER_LINEAR, borderMode=borderMode,\n",
    "                                    borderValue=(\n",
    "                                        0, 0,\n",
    "                                        0,))\n",
    "        mask = cv2.warpPerspective(mask, mat, (width, height), flags=cv2.INTER_LINEAR, borderMode=borderMode,\n",
    "                                   borderValue=(\n",
    "                                       0, 0,\n",
    "                                       0,))\n",
    "\n",
    "    return image, mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-07T07:33:09.021220Z",
     "start_time": "2017-09-07T07:33:09.017598Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def randomHorizontalFlip(image, mask, u=0.5):\n",
    "    if np.random.random() < u:\n",
    "        image = cv2.flip(image, 1)\n",
    "        mask = cv2.flip(mask, 1)\n",
    "\n",
    "    return image, mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-07T07:33:09.517756Z",
     "start_time": "2017-09-07T07:33:09.511808Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def m512_stuff(ids_list):\n",
    "    final_512_preds=[]\n",
    "    for this_id in ids_list:\n",
    "        \n",
    "        f = gzip.open('./data/preds512/' + this_id + '.pklz','rb')\n",
    "        mask = pickle.load(f)\n",
    "        f.close()\n",
    "        \n",
    "        resized_mask = cv2.resize(mask, (1920, 1280))\n",
    "        resized_mask = np.expand_dims(resized_mask, axis=3)*255\n",
    "        final_512_preds.append(resized_mask)\n",
    "    \n",
    "    return final_512_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-07T07:33:10.125172Z",
     "start_time": "2017-09-07T07:33:10.096712Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def train_generator(train_batch_size):\n",
    "    while True:\n",
    "        this_ids_train_split = random.sample(list(ids_train_split), len(list(ids_train_split)))\n",
    "        for start in range(0, len(ids_train_split), train_batch_size):\n",
    "            x_batch = []\n",
    "            y_batch = []\n",
    "            \n",
    "            start=0\n",
    "            end = min(start + train_batch_size, len(ids_train_split))\n",
    "            ids_train_batch = this_ids_train_split[start:end]\n",
    "            \n",
    "            all_imgs = []\n",
    "            for id in ids_train_batch:\n",
    "                img = cv2.imread('data/train/{}.jpg'.format(id))\n",
    "                all_imgs.append(img)\n",
    "            \n",
    "            final_512_preds = m512_stuff(ids_train_batch)\n",
    "            \n",
    "            for i in range(len(all_imgs)):\n",
    "                img = cv2.resize(all_imgs[i], (1920,1280))\n",
    "                img = randomHueSaturationValue(img,\n",
    "                                               hue_shift_limit=(-50, 50),\n",
    "                                               sat_shift_limit=(-5, 5),\n",
    "                                               val_shift_limit=(-15, 15))\n",
    "                img = np.concatenate([img, final_512_preds[i]], axis=2)\n",
    "                \n",
    "                mask = cv2.imread('data/train_masks/{}_mask.png'.format(ids_train_batch[i]), cv2.IMREAD_GRAYSCALE)\n",
    "                mask = cv2.resize(mask, (1920,1280))\n",
    "                \n",
    "                img, mask = randomShiftScaleRotate(img, mask,\n",
    "                                                   shift_limit=(-0.0625, 0.0625),\n",
    "                                                   scale_limit=(-0.1, 0.1),\n",
    "                                                   rotate_limit=(-0, 0))\n",
    "                img, mask = randomHorizontalFlip(img, mask)\n",
    "                mask = np.expand_dims(mask, axis=2)\n",
    "                x_batch.append(img)\n",
    "                y_batch.append(mask)\n",
    "            \n",
    "            x_batch = np.array(x_batch, np.float32) / 255\n",
    "            y_batch = np.array(y_batch, np.float32) / 255\n",
    "            yield x_batch, y_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-07T07:33:10.952169Z",
     "start_time": "2017-09-07T07:33:10.934413Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def valid_generator(val_batch_size):\n",
    "    while True:\n",
    "        for start in range(0, len(ids_valid_split), val_batch_size):\n",
    "            \n",
    "            x_batch = []\n",
    "            y_batch = []\n",
    "            \n",
    "            end = min(start + val_batch_size, len(ids_valid_split))\n",
    "            ids_valid_batch = ids_valid_split[start:end]\n",
    "            \n",
    "            all_imgs = []\n",
    "            for id in ids_valid_batch.values:\n",
    "                img = cv2.imread('data/train/{}.jpg'.format(id))\n",
    "                all_imgs.append(img)\n",
    "            \n",
    "            final_512_preds = m512_stuff(ids_valid_batch.values)\n",
    "            \n",
    "            for i in range(len(all_imgs)):\n",
    "                img = cv2.resize(all_imgs[i], (1920,1280))\n",
    "                img = np.concatenate([img, final_512_preds[i]], axis=2)\n",
    "                mask = cv2.imread('data/train_masks/{}_mask.png'.format(ids_valid_batch.values[i]), cv2.IMREAD_GRAYSCALE)\n",
    "                mask = cv2.resize(mask, (1920,1280))\n",
    "                mask = np.expand_dims(mask, axis=2)\n",
    "                x_batch.append(img)\n",
    "                y_batch.append(mask)\n",
    "            \n",
    "            x_batch = np.array(x_batch, np.float32) / 255\n",
    "            y_batch = np.array(y_batch, np.float32) / 255\n",
    "            yield x_batch, y_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-07T07:33:15.965706Z",
     "start_time": "2017-09-07T07:33:15.953898Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "max_epochs = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-07T07:33:16.200426Z",
     "start_time": "2017-09-07T07:33:16.198126Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "train_batch_size = 3\n",
    "val_batch_size   = 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2017-09-06T14:00:22.313Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "4466s - loss: 0.7622 - dice_coeff: 0.5814 - val_loss: 0.6156 - val_dice_coeff: 0.6557\n",
      "Epoch 2/50\n",
      "4459s - loss: 0.4551 - dice_coeff: 0.7366 - val_loss: 0.3884 - val_dice_coeff: 0.7606\n",
      "Epoch 3/50\n",
      "4459s - loss: 0.2361 - dice_coeff: 0.8573 - val_loss: 0.2059 - val_dice_coeff: 0.8662\n",
      "Epoch 4/50\n",
      "4460s - loss: 0.2182 - dice_coeff: 0.8479 - val_loss: 0.1380 - val_dice_coeff: 0.9107\n",
      "Epoch 5/50\n",
      "4460s - loss: 0.0832 - dice_coeff: 0.9425 - val_loss: 0.0594 - val_dice_coeff: 0.9633\n",
      "Epoch 6/50\n",
      "4460s - loss: 0.0290 - dice_coeff: 0.9808 - val_loss: 0.0294 - val_dice_coeff: 0.9848\n",
      "Epoch 7/50\n",
      "4459s - loss: 0.0129 - dice_coeff: 0.9913 - val_loss: 0.0186 - val_dice_coeff: 0.9920\n",
      "Epoch 8/50\n",
      "4458s - loss: 0.0083 - dice_coeff: 0.9943 - val_loss: 0.0174 - val_dice_coeff: 0.9937\n",
      "Epoch 9/50\n",
      "4460s - loss: 0.0049 - dice_coeff: 0.9973 - val_loss: 0.0141 - val_dice_coeff: 0.9953\n",
      "Epoch 10/50\n",
      "4461s - loss: 0.0045 - dice_coeff: 0.9975 - val_loss: 0.0148 - val_dice_coeff: 0.9953\n",
      "Epoch 11/50\n",
      "4461s - loss: 0.0031 - dice_coeff: 0.9984 - val_loss: 0.0141 - val_dice_coeff: 0.9957\n",
      "Epoch 12/50\n",
      "4462s - loss: 0.0029 - dice_coeff: 0.9985 - val_loss: 0.0152 - val_dice_coeff: 0.9955\n",
      "Epoch 13/50\n",
      "4461s - loss: 0.0028 - dice_coeff: 0.9985 - val_loss: 0.0144 - val_dice_coeff: 0.9956\n",
      "Epoch 14/50\n",
      "\n",
      "Epoch 00013: reducing learning rate to 9.999999747378752e-06.\n",
      "4461s - loss: 0.0032 - dice_coeff: 0.9983 - val_loss: 0.0155 - val_dice_coeff: 0.9954\n",
      "Epoch 15/50\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-25-59878f25d0c3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     19\u001b[0m                             \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m                             \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalid_generator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_batch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m                             validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size)))\n\u001b[0m",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     85\u001b[0m                 warnings.warn('Update your `' + object_name +\n\u001b[1;32m     86\u001b[0m                               '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 87\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m   2040\u001b[0m                     outs = self.train_on_batch(x, y,\n\u001b[1;32m   2041\u001b[0m                                                \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2042\u001b[0;31m                                                class_weight=class_weight)\n\u001b[0m\u001b[1;32m   2043\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2044\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[1;32m   1760\u001b[0m             \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1761\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1762\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1763\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1764\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2271\u001b[0m         updated = session.run(self.outputs + [self.updates_op],\n\u001b[1;32m   2272\u001b[0m                               \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2273\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2274\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2275\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    893\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    894\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 895\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    896\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    897\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1122\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1123\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1124\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1125\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1126\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1319\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1320\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1321\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1322\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1323\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1325\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1326\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1327\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1328\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1329\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1304\u001b[0m           return tf_session.TF_Run(session, options,\n\u001b[1;32m   1305\u001b[0m                                    \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1306\u001b[0;31m                                    status, run_metadata)\n\u001b[0m\u001b[1;32m   1307\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1308\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "callbacks = [EarlyStopping(monitor='val_loss',\n",
    "                           patience=8,\n",
    "                           verbose=1,\n",
    "                           min_delta=1e-4),\n",
    "             ReduceLROnPlateau(monitor='val_loss',\n",
    "                               factor=0.1,\n",
    "                               patience=4,\n",
    "                               verbose=1,\n",
    "                               epsilon=1e-4),\n",
    "             ModelCheckpoint(monitor='val_loss',\n",
    "                             filepath='weights/best_weights_fullresv2.hdf5',\n",
    "                             save_best_only=True,\n",
    "                             save_weights_only=True)]\n",
    "\n",
    "model_fullres.fit_generator(generator=train_generator(train_batch_size),\n",
    "                            steps_per_epoch=np.ceil(float(len(ids_train_split)) / float(train_batch_size)),\n",
    "                            epochs=max_epochs,\n",
    "                            verbose=2,\n",
    "                            callbacks=callbacks,\n",
    "                            validation_data=valid_generator(val_batch_size),\n",
    "                            validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Full res model 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-07T07:32:43.966341Z",
     "start_time": "2017-09-07T07:32:42.342318Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_fullres =  get_unet(input_shape=(1280,1920,4), initial_filters=8, central_filters=1024)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-07T07:32:53.560629Z",
     "start_time": "2017-09-07T07:32:53.524081Z"
    },
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_1 (InputLayer)             (None, 1280, 1920, 4) 0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)                (None, 1280, 1920, 8) 296         input_1[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNorm (None, 1280, 1920, 8) 32          conv2d_1[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "activation_1 (Activation)        (None, 1280, 1920, 8) 0           batch_normalization_1[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)                (None, 1280, 1920, 8) 584         activation_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNorm (None, 1280, 1920, 8) 32          conv2d_2[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "activation_2 (Activation)        (None, 1280, 1920, 8) 0           batch_normalization_2[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)   (None, 640, 960, 8)   0           activation_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)                (None, 640, 960, 16)  1168        max_pooling2d_1[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNorm (None, 640, 960, 16)  64          conv2d_3[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "activation_3 (Activation)        (None, 640, 960, 16)  0           batch_normalization_3[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)                (None, 640, 960, 16)  2320        activation_3[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNorm (None, 640, 960, 16)  64          conv2d_4[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "activation_4 (Activation)        (None, 640, 960, 16)  0           batch_normalization_4[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2D)   (None, 320, 480, 16)  0           activation_4[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)                (None, 320, 480, 32)  4640        max_pooling2d_2[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNorm (None, 320, 480, 32)  128         conv2d_5[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "activation_5 (Activation)        (None, 320, 480, 32)  0           batch_normalization_5[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)                (None, 320, 480, 32)  9248        activation_5[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNorm (None, 320, 480, 32)  128         conv2d_6[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "activation_6 (Activation)        (None, 320, 480, 32)  0           batch_normalization_6[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2D)   (None, 160, 240, 32)  0           activation_6[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_7 (Conv2D)                (None, 160, 240, 64)  18496       max_pooling2d_3[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNorm (None, 160, 240, 64)  256         conv2d_7[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "activation_7 (Activation)        (None, 160, 240, 64)  0           batch_normalization_7[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_8 (Conv2D)                (None, 160, 240, 64)  36928       activation_7[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNorm (None, 160, 240, 64)  256         conv2d_8[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "activation_8 (Activation)        (None, 160, 240, 64)  0           batch_normalization_8[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2D)   (None, 80, 120, 64)   0           activation_8[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_9 (Conv2D)                (None, 80, 120, 128)  73856       max_pooling2d_4[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_9 (BatchNorm (None, 80, 120, 128)  512         conv2d_9[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "activation_9 (Activation)        (None, 80, 120, 128)  0           batch_normalization_9[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_10 (Conv2D)               (None, 80, 120, 128)  147584      activation_9[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_10 (BatchNor (None, 80, 120, 128)  512         conv2d_10[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_10 (Activation)       (None, 80, 120, 128)  0           batch_normalization_10[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_5 (MaxPooling2D)   (None, 40, 60, 128)   0           activation_10[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_11 (Conv2D)               (None, 40, 60, 256)   295168      max_pooling2d_5[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_11 (BatchNor (None, 40, 60, 256)   1024        conv2d_11[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_11 (Activation)       (None, 40, 60, 256)   0           batch_normalization_11[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_12 (Conv2D)               (None, 40, 60, 256)   590080      activation_11[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_12 (BatchNor (None, 40, 60, 256)   1024        conv2d_12[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_12 (Activation)       (None, 40, 60, 256)   0           batch_normalization_12[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_6 (MaxPooling2D)   (None, 20, 30, 256)   0           activation_12[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_13 (Conv2D)               (None, 20, 30, 512)   1180160     max_pooling2d_6[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_13 (BatchNor (None, 20, 30, 512)   2048        conv2d_13[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_13 (Activation)       (None, 20, 30, 512)   0           batch_normalization_13[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_14 (Conv2D)               (None, 20, 30, 512)   2359808     activation_13[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_14 (BatchNor (None, 20, 30, 512)   2048        conv2d_14[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_14 (Activation)       (None, 20, 30, 512)   0           batch_normalization_14[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_7 (MaxPooling2D)   (None, 10, 15, 512)   0           activation_14[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_15 (Conv2D)               (None, 10, 15, 1024)  4719616     max_pooling2d_7[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_15 (BatchNor (None, 10, 15, 1024)  4096        conv2d_15[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_15 (Activation)       (None, 10, 15, 1024)  0           batch_normalization_15[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_16 (Conv2D)               (None, 10, 15, 1024)  9438208     activation_15[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_16 (BatchNor (None, 10, 15, 1024)  4096        conv2d_16[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_16 (Activation)       (None, 10, 15, 1024)  0           batch_normalization_16[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "up_sampling2d_1 (UpSampling2D)   (None, 20, 30, 1024)  0           activation_16[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)      (None, 20, 30, 1536)  0           activation_14[0][0]              \n",
      "                                                                   up_sampling2d_1[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_17 (Conv2D)               (None, 20, 30, 512)   7078400     concatenate_1[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_17 (BatchNor (None, 20, 30, 512)   2048        conv2d_17[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_17 (Activation)       (None, 20, 30, 512)   0           batch_normalization_17[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_18 (Conv2D)               (None, 20, 30, 512)   2359808     activation_17[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_18 (BatchNor (None, 20, 30, 512)   2048        conv2d_18[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_18 (Activation)       (None, 20, 30, 512)   0           batch_normalization_18[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_19 (Conv2D)               (None, 20, 30, 512)   2359808     activation_18[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_19 (BatchNor (None, 20, 30, 512)   2048        conv2d_19[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_19 (Activation)       (None, 20, 30, 512)   0           batch_normalization_19[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "up_sampling2d_2 (UpSampling2D)   (None, 40, 60, 512)   0           activation_19[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)      (None, 40, 60, 768)   0           activation_12[0][0]              \n",
      "                                                                   up_sampling2d_2[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_20 (Conv2D)               (None, 40, 60, 256)   1769728     concatenate_2[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_20 (BatchNor (None, 40, 60, 256)   1024        conv2d_20[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_20 (Activation)       (None, 40, 60, 256)   0           batch_normalization_20[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_21 (Conv2D)               (None, 40, 60, 256)   590080      activation_20[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_21 (BatchNor (None, 40, 60, 256)   1024        conv2d_21[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_21 (Activation)       (None, 40, 60, 256)   0           batch_normalization_21[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_22 (Conv2D)               (None, 40, 60, 256)   590080      activation_21[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_22 (BatchNor (None, 40, 60, 256)   1024        conv2d_22[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_22 (Activation)       (None, 40, 60, 256)   0           batch_normalization_22[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "up_sampling2d_3 (UpSampling2D)   (None, 80, 120, 256)  0           activation_22[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)      (None, 80, 120, 384)  0           activation_10[0][0]              \n",
      "                                                                   up_sampling2d_3[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_23 (Conv2D)               (None, 80, 120, 128)  442496      concatenate_3[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_23 (BatchNor (None, 80, 120, 128)  512         conv2d_23[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_23 (Activation)       (None, 80, 120, 128)  0           batch_normalization_23[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_24 (Conv2D)               (None, 80, 120, 128)  147584      activation_23[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_24 (BatchNor (None, 80, 120, 128)  512         conv2d_24[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_24 (Activation)       (None, 80, 120, 128)  0           batch_normalization_24[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_25 (Conv2D)               (None, 80, 120, 128)  147584      activation_24[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_25 (BatchNor (None, 80, 120, 128)  512         conv2d_25[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_25 (Activation)       (None, 80, 120, 128)  0           batch_normalization_25[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "up_sampling2d_4 (UpSampling2D)   (None, 160, 240, 128) 0           activation_25[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_4 (Concatenate)      (None, 160, 240, 192) 0           activation_8[0][0]               \n",
      "                                                                   up_sampling2d_4[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_26 (Conv2D)               (None, 160, 240, 64)  110656      concatenate_4[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_26 (BatchNor (None, 160, 240, 64)  256         conv2d_26[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_26 (Activation)       (None, 160, 240, 64)  0           batch_normalization_26[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_27 (Conv2D)               (None, 160, 240, 64)  36928       activation_26[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_27 (BatchNor (None, 160, 240, 64)  256         conv2d_27[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_27 (Activation)       (None, 160, 240, 64)  0           batch_normalization_27[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_28 (Conv2D)               (None, 160, 240, 64)  36928       activation_27[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_28 (BatchNor (None, 160, 240, 64)  256         conv2d_28[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_28 (Activation)       (None, 160, 240, 64)  0           batch_normalization_28[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "up_sampling2d_5 (UpSampling2D)   (None, 320, 480, 64)  0           activation_28[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_5 (Concatenate)      (None, 320, 480, 96)  0           activation_6[0][0]               \n",
      "                                                                   up_sampling2d_5[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_29 (Conv2D)               (None, 320, 480, 32)  27680       concatenate_5[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_29 (BatchNor (None, 320, 480, 32)  128         conv2d_29[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_29 (Activation)       (None, 320, 480, 32)  0           batch_normalization_29[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_30 (Conv2D)               (None, 320, 480, 32)  9248        activation_29[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_30 (BatchNor (None, 320, 480, 32)  128         conv2d_30[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_30 (Activation)       (None, 320, 480, 32)  0           batch_normalization_30[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_31 (Conv2D)               (None, 320, 480, 32)  9248        activation_30[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_31 (BatchNor (None, 320, 480, 32)  128         conv2d_31[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_31 (Activation)       (None, 320, 480, 32)  0           batch_normalization_31[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "up_sampling2d_6 (UpSampling2D)   (None, 640, 960, 32)  0           activation_31[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_6 (Concatenate)      (None, 640, 960, 48)  0           activation_4[0][0]               \n",
      "                                                                   up_sampling2d_6[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_32 (Conv2D)               (None, 640, 960, 16)  6928        concatenate_6[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_32 (BatchNor (None, 640, 960, 16)  64          conv2d_32[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_32 (Activation)       (None, 640, 960, 16)  0           batch_normalization_32[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_33 (Conv2D)               (None, 640, 960, 16)  2320        activation_32[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_33 (BatchNor (None, 640, 960, 16)  64          conv2d_33[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_33 (Activation)       (None, 640, 960, 16)  0           batch_normalization_33[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_34 (Conv2D)               (None, 640, 960, 16)  2320        activation_33[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_34 (BatchNor (None, 640, 960, 16)  64          conv2d_34[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_34 (Activation)       (None, 640, 960, 16)  0           batch_normalization_34[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "up_sampling2d_7 (UpSampling2D)   (None, 1280, 1920, 16 0           activation_34[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_7 (Concatenate)      (None, 1280, 1920, 24 0           activation_2[0][0]               \n",
      "                                                                   up_sampling2d_7[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_35 (Conv2D)               (None, 1280, 1920, 8) 1736        concatenate_7[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_35 (BatchNor (None, 1280, 1920, 8) 32          conv2d_35[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_35 (Activation)       (None, 1280, 1920, 8) 0           batch_normalization_35[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_36 (Conv2D)               (None, 1280, 1920, 8) 584         activation_35[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_36 (BatchNor (None, 1280, 1920, 8) 32          conv2d_36[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_36 (Activation)       (None, 1280, 1920, 8) 0           batch_normalization_36[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_37 (Conv2D)               (None, 1280, 1920, 8) 584         activation_36[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_37 (BatchNor (None, 1280, 1920, 8) 32          conv2d_37[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_37 (Activation)       (None, 1280, 1920, 8) 0           batch_normalization_37[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_38 (Conv2D)               (None, 1280, 1920, 1) 9           activation_37[0][0]              \n",
      "====================================================================================================\n",
      "Total params: 34,637,409\n",
      "Trainable params: 34,623,153\n",
      "Non-trainable params: 14,256\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_fullres.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2017-09-07T07:33:27.015Z"
    },
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "5437s - loss: 0.6051 - dice_coeff: 0.6865 - val_loss: 0.5555 - val_dice_coeff: 0.6788\n",
      "Epoch 2/50\n",
      "5413s - loss: 0.5022 - dice_coeff: 0.6988 - val_loss: 0.3965 - val_dice_coeff: 0.7570\n",
      "Epoch 3/50\n",
      "5411s - loss: 0.1670 - dice_coeff: 0.8868 - val_loss: 0.1190 - val_dice_coeff: 0.9239\n",
      "Epoch 5/50\n",
      "5414s - loss: 0.0846 - dice_coeff: 0.9406 - val_loss: 0.0596 - val_dice_coeff: 0.9635\n",
      "Epoch 6/50\n",
      "5414s - loss: 0.0291 - dice_coeff: 0.9803 - val_loss: 0.0271 - val_dice_coeff: 0.9856\n",
      "Epoch 7/50\n",
      "5415s - loss: 0.0101 - dice_coeff: 0.9936 - val_loss: 0.0176 - val_dice_coeff: 0.9929\n",
      "Epoch 8/50\n",
      "5416s - loss: 0.0048 - dice_coeff: 0.9971 - val_loss: 0.0148 - val_dice_coeff: 0.9950\n",
      "Epoch 9/50\n",
      "5415s - loss: 0.0036 - dice_coeff: 0.9978 - val_loss: 0.0150 - val_dice_coeff: 0.9954\n",
      "Epoch 10/50\n",
      "5416s - loss: 0.0030 - dice_coeff: 0.9986 - val_loss: 0.0139 - val_dice_coeff: 0.9958\n",
      "Epoch 11/50\n",
      "5418s - loss: 0.0026 - dice_coeff: 0.9986 - val_loss: 0.0145 - val_dice_coeff: 0.9958\n",
      "Epoch 12/50\n",
      "5417s - loss: 0.0023 - dice_coeff: 0.9988 - val_loss: 0.0146 - val_dice_coeff: 0.9958\n",
      "Epoch 13/50\n",
      "5417s - loss: 0.0025 - dice_coeff: 0.9986 - val_loss: 0.0148 - val_dice_coeff: 0.9958\n",
      "Epoch 14/50\n",
      "5417s - loss: 0.0027 - dice_coeff: 0.9986 - val_loss: 0.0147 - val_dice_coeff: 0.9958\n",
      "Epoch 15/50\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-22-db2ff83ed2f4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     19\u001b[0m                             \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m                             \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalid_generator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_batch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m                             validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size)))\n\u001b[0m",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     85\u001b[0m                 warnings.warn('Update your `' + object_name +\n\u001b[1;32m     86\u001b[0m                               '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 87\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m   2040\u001b[0m                     outs = self.train_on_batch(x, y,\n\u001b[1;32m   2041\u001b[0m                                                \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2042\u001b[0;31m                                                class_weight=class_weight)\n\u001b[0m\u001b[1;32m   2043\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2044\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[1;32m   1760\u001b[0m             \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1761\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1762\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1763\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1764\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2271\u001b[0m         updated = session.run(self.outputs + [self.updates_op],\n\u001b[1;32m   2272\u001b[0m                               \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2273\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2274\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2275\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    893\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    894\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 895\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    896\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    897\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1122\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1123\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1124\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1125\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1126\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1319\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1320\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1321\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1322\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1323\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1325\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1326\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1327\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1328\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1329\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1304\u001b[0m           return tf_session.TF_Run(session, options,\n\u001b[1;32m   1305\u001b[0m                                    \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1306\u001b[0;31m                                    status, run_metadata)\n\u001b[0m\u001b[1;32m   1307\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1308\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "callbacks = [EarlyStopping(monitor='val_loss',\n",
    "                           patience=8,\n",
    "                           verbose=1,\n",
    "                           min_delta=1e-4),\n",
    "             ReduceLROnPlateau(monitor='val_loss',\n",
    "                               factor=0.1,\n",
    "                               patience=4,\n",
    "                               verbose=1,\n",
    "                               epsilon=1e-4),\n",
    "             ModelCheckpoint(monitor='val_loss',\n",
    "                             filepath='weights/best_weights_fullresv3.hdf5',\n",
    "                             save_best_only=True,\n",
    "                             save_weights_only=True)]\n",
    "\n",
    "model_fullres.fit_generator(generator=train_generator(train_batch_size),\n",
    "                            steps_per_epoch=np.ceil(float(len(ids_train_split)) / float(train_batch_size)),\n",
    "                            epochs=max_epochs,\n",
    "                            verbose=2,\n",
    "                            callbacks=callbacks,\n",
    "                            validation_data=valid_generator(val_batch_size),\n",
    "                            validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-08T04:59:41.874111Z",
     "start_time": "2017-09-08T04:59:41.687937Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_fullres.load_weights('./weights/best_weights_fullresv3.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-08T04:59:42.552399Z",
     "start_time": "2017-09-08T04:59:42.502751Z"
    }
   },
   "outputs": [],
   "source": [
    "model_fullres.compile(optimizer=RMSprop(lr=0.00001),\n",
    "                  loss=bce_dice_loss,\n",
    "                  metrics=[dice_coeff])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-08T06:54:43.221837Z",
     "start_time": "2017-09-08T04:59:43.460981Z"
    },
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "5415s - loss: 0.0028 - dice_coeff: 0.9985 - val_loss: 0.0153 - val_dice_coeff: 0.9956\n",
      "Epoch 2/50\n"
     ]
    },
    {
     "ename": "ResourceExhaustedError",
     "evalue": "OOM when allocating tensor with shape[3,24,1280,1920]\n\t [[Node: training_1/RMSprop/gradients/conv2d_35/convolution_grad/Conv2DBackpropInput = Conv2DBackpropInput[T=DT_FLOAT, _class=[\"loc:@conv2d_35/convolution\"], data_format=\"NHWC\", padding=\"SAME\", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device=\"/job:localhost/replica:0/task:0/gpu:0\"](training_1/RMSprop/gradients/conv2d_35/convolution_grad/Shape, conv2d_35/kernel/read, training_1/RMSprop/gradients/AddN_18)]]\n\nCaused by op 'training_1/RMSprop/gradients/conv2d_35/convolution_grad/Conv2DBackpropInput', defined at:\n  File \"/home/sainath/anaconda3/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/home/sainath/anaconda3/lib/python3.6/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/kernelapp.py\", line 477, in start\n    ioloop.IOLoop.instance().start()\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/zmq/eventloop/ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py\", line 888, in start\n    handler_func(fd_obj, events)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 235, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/zmqshell.py\", line 533, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2827, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-28-de876fd6dcfd>\", line 21, in <module>\n    validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size)))\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py\", line 87, in wrapper\n    return func(*args, **kwargs)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\", line 1926, in fit_generator\n    self._make_train_function()\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\", line 960, in _make_train_function\n    loss=self.total_loss)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py\", line 87, in wrapper\n    return func(*args, **kwargs)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/optimizers.py\", line 225, in get_updates\n    grads = self.get_gradients(loss, params)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/optimizers.py\", line 73, in get_gradients\n    grads = K.gradients(loss, params)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\", line 2310, in gradients\n    return tf.gradients(loss, variables, colocate_gradients_with_ops=True)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py\", line 542, in gradients\n    grad_scope, op, func_call, lambda: grad_fn(op, *out_grads))\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py\", line 348, in _MaybeCompile\n    return grad_fn()  # Exit early\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py\", line 542, in <lambda>\n    grad_scope, op, func_call, lambda: grad_fn(op, *out_grads))\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_grad.py\", line 446, in _Conv2DGrad\n    op.get_attr(\"data_format\")),\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gen_nn_ops.py\", line 486, in conv2d_backprop_input\n    data_format=data_format, name=name)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py\", line 767, in apply_op\n    op_def=op_def)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 2630, in create_op\n    original_op=self._default_original_op, op_def=op_def)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 1204, in __init__\n    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access\n\n...which was originally created as op 'conv2d_35/convolution', defined at:\n  File \"/home/sainath/anaconda3/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n[elided 16 identical lines from previous traceback]\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-8-56ff7dc954b0>\", line 1, in <module>\n    model_fullres =  get_unet(input_shape=(1280,1920,4), initial_filters=8, central_filters=1024)\n  File \"<ipython-input-7-204678a39f7c>\", line 23, in get_unet\n    x = unet_up_one_block(out_list[-1], down_interim_list.pop(), num_filters)\n  File \"<ipython-input-6-592812654380>\", line 4, in unet_up_one_block\n    x = Conv2D(num_filters, (3,3), padding='same')(x)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/topology.py\", line 602, in __call__\n    output = self.call(inputs, **kwargs)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/layers/convolutional.py\", line 164, in call\n    dilation_rate=self.dilation_rate)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\", line 3164, in conv2d\n    data_format='NHWC')\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py\", line 672, in convolution\n    op=op)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py\", line 338, in with_space_to_batch\n    return op(input, num_spatial_dims, padding)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py\", line 664, in op\n    name=name)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py\", line 131, in _non_atrous_convolution\n    name=name)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gen_nn_ops.py\", line 397, in conv2d\n    data_format=data_format, name=name)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py\", line 767, in apply_op\n    op_def=op_def)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 2630, in create_op\n    original_op=self._default_original_op, op_def=op_def)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 1204, in __init__\n    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access\n\nResourceExhaustedError (see above for traceback): OOM when allocating tensor with shape[3,24,1280,1920]\n\t [[Node: training_1/RMSprop/gradients/conv2d_35/convolution_grad/Conv2DBackpropInput = Conv2DBackpropInput[T=DT_FLOAT, _class=[\"loc:@conv2d_35/convolution\"], data_format=\"NHWC\", padding=\"SAME\", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device=\"/job:localhost/replica:0/task:0/gpu:0\"](training_1/RMSprop/gradients/conv2d_35/convolution_grad/Shape, conv2d_35/kernel/read, training_1/RMSprop/gradients/AddN_18)]]\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1326\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1327\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1328\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1305\u001b[0m                                    \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1306\u001b[0;31m                                    status, run_metadata)\n\u001b[0m\u001b[1;32m   1307\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/contextlib.py\u001b[0m in \u001b[0;36m__exit__\u001b[0;34m(self, type, value, traceback)\u001b[0m\n\u001b[1;32m     88\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 89\u001b[0;31m                 \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     90\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py\u001b[0m in \u001b[0;36mraise_exception_on_not_ok_status\u001b[0;34m()\u001b[0m\n\u001b[1;32m    465\u001b[0m           \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpywrap_tensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_Message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 466\u001b[0;31m           pywrap_tensorflow.TF_GetCode(status))\n\u001b[0m\u001b[1;32m    467\u001b[0m   \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mResourceExhaustedError\u001b[0m: OOM when allocating tensor with shape[3,24,1280,1920]\n\t [[Node: training_1/RMSprop/gradients/conv2d_35/convolution_grad/Conv2DBackpropInput = Conv2DBackpropInput[T=DT_FLOAT, _class=[\"loc:@conv2d_35/convolution\"], data_format=\"NHWC\", padding=\"SAME\", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device=\"/job:localhost/replica:0/task:0/gpu:0\"](training_1/RMSprop/gradients/conv2d_35/convolution_grad/Shape, conv2d_35/kernel/read, training_1/RMSprop/gradients/AddN_18)]]",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-28-de876fd6dcfd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     19\u001b[0m                             \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m                             \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalid_generator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_batch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m                             validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size)))\n\u001b[0m",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     85\u001b[0m                 warnings.warn('Update your `' + object_name +\n\u001b[1;32m     86\u001b[0m                               '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 87\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m   2040\u001b[0m                     outs = self.train_on_batch(x, y,\n\u001b[1;32m   2041\u001b[0m                                                \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2042\u001b[0;31m                                                class_weight=class_weight)\n\u001b[0m\u001b[1;32m   2043\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2044\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[1;32m   1760\u001b[0m             \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1761\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1762\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1763\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1764\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2271\u001b[0m         updated = session.run(self.outputs + [self.updates_op],\n\u001b[1;32m   2272\u001b[0m                               \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2273\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2274\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2275\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    893\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    894\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 895\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    896\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    897\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1122\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1123\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1124\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1125\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1126\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1319\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1320\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1321\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1322\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1323\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1338\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1339\u001b[0m           \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1340\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1341\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1342\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mResourceExhaustedError\u001b[0m: OOM when allocating tensor with shape[3,24,1280,1920]\n\t [[Node: training_1/RMSprop/gradients/conv2d_35/convolution_grad/Conv2DBackpropInput = Conv2DBackpropInput[T=DT_FLOAT, _class=[\"loc:@conv2d_35/convolution\"], data_format=\"NHWC\", padding=\"SAME\", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device=\"/job:localhost/replica:0/task:0/gpu:0\"](training_1/RMSprop/gradients/conv2d_35/convolution_grad/Shape, conv2d_35/kernel/read, training_1/RMSprop/gradients/AddN_18)]]\n\nCaused by op 'training_1/RMSprop/gradients/conv2d_35/convolution_grad/Conv2DBackpropInput', defined at:\n  File \"/home/sainath/anaconda3/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/home/sainath/anaconda3/lib/python3.6/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/kernelapp.py\", line 477, in start\n    ioloop.IOLoop.instance().start()\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/zmq/eventloop/ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py\", line 888, in start\n    handler_func(fd_obj, events)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 235, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/zmqshell.py\", line 533, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2827, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-28-de876fd6dcfd>\", line 21, in <module>\n    validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size)))\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py\", line 87, in wrapper\n    return func(*args, **kwargs)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\", line 1926, in fit_generator\n    self._make_train_function()\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\", line 960, in _make_train_function\n    loss=self.total_loss)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py\", line 87, in wrapper\n    return func(*args, **kwargs)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/optimizers.py\", line 225, in get_updates\n    grads = self.get_gradients(loss, params)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/optimizers.py\", line 73, in get_gradients\n    grads = K.gradients(loss, params)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\", line 2310, in gradients\n    return tf.gradients(loss, variables, colocate_gradients_with_ops=True)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py\", line 542, in gradients\n    grad_scope, op, func_call, lambda: grad_fn(op, *out_grads))\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py\", line 348, in _MaybeCompile\n    return grad_fn()  # Exit early\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py\", line 542, in <lambda>\n    grad_scope, op, func_call, lambda: grad_fn(op, *out_grads))\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_grad.py\", line 446, in _Conv2DGrad\n    op.get_attr(\"data_format\")),\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gen_nn_ops.py\", line 486, in conv2d_backprop_input\n    data_format=data_format, name=name)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py\", line 767, in apply_op\n    op_def=op_def)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 2630, in create_op\n    original_op=self._default_original_op, op_def=op_def)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 1204, in __init__\n    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access\n\n...which was originally created as op 'conv2d_35/convolution', defined at:\n  File \"/home/sainath/anaconda3/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n[elided 16 identical lines from previous traceback]\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-8-56ff7dc954b0>\", line 1, in <module>\n    model_fullres =  get_unet(input_shape=(1280,1920,4), initial_filters=8, central_filters=1024)\n  File \"<ipython-input-7-204678a39f7c>\", line 23, in get_unet\n    x = unet_up_one_block(out_list[-1], down_interim_list.pop(), num_filters)\n  File \"<ipython-input-6-592812654380>\", line 4, in unet_up_one_block\n    x = Conv2D(num_filters, (3,3), padding='same')(x)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/topology.py\", line 602, in __call__\n    output = self.call(inputs, **kwargs)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/layers/convolutional.py\", line 164, in call\n    dilation_rate=self.dilation_rate)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\", line 3164, in conv2d\n    data_format='NHWC')\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py\", line 672, in convolution\n    op=op)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py\", line 338, in with_space_to_batch\n    return op(input, num_spatial_dims, padding)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py\", line 664, in op\n    name=name)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py\", line 131, in _non_atrous_convolution\n    name=name)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gen_nn_ops.py\", line 397, in conv2d\n    data_format=data_format, name=name)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py\", line 767, in apply_op\n    op_def=op_def)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 2630, in create_op\n    original_op=self._default_original_op, op_def=op_def)\n  File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 1204, in __init__\n    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access\n\nResourceExhaustedError (see above for traceback): OOM when allocating tensor with shape[3,24,1280,1920]\n\t [[Node: training_1/RMSprop/gradients/conv2d_35/convolution_grad/Conv2DBackpropInput = Conv2DBackpropInput[T=DT_FLOAT, _class=[\"loc:@conv2d_35/convolution\"], data_format=\"NHWC\", padding=\"SAME\", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device=\"/job:localhost/replica:0/task:0/gpu:0\"](training_1/RMSprop/gradients/conv2d_35/convolution_grad/Shape, conv2d_35/kernel/read, training_1/RMSprop/gradients/AddN_18)]]\n"
     ]
    }
   ],
   "source": [
    "callbacks = [EarlyStopping(monitor='val_loss',\n",
    "                           patience=4,\n",
    "                           verbose=1,\n",
    "                           min_delta=1e-4),\n",
    "             ReduceLROnPlateau(monitor='val_loss',\n",
    "                               factor=0.1,\n",
    "                               patience=3,\n",
    "                               verbose=1,\n",
    "                               epsilon=1e-4),\n",
    "             ModelCheckpoint(monitor='val_loss',\n",
    "                             filepath='weights/best_weights_fullresv3.hdf5',\n",
    "                             save_best_only=True,\n",
    "                             save_weights_only=True)]\n",
    "\n",
    "model_fullres.fit_generator(generator=train_generator(train_batch_size),\n",
    "                            steps_per_epoch=np.ceil(float(len(ids_train_split)) / float(train_batch_size)),\n",
    "                            epochs=max_epochs,\n",
    "                            verbose=2,\n",
    "                            callbacks=callbacks,\n",
    "                            validation_data=valid_generator(val_batch_size),\n",
    "                            validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  },
  "toc": {
   "nav_menu": {
    "height": "48px",
    "width": "252px"
   },
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": true,
   "toc_position": {
    "height": "857px",
    "left": "0px",
    "right": "1706px",
    "top": "106px",
    "width": "212px"
   },
   "toc_section_display": "block",
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}