barebone-yolo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": "true"
   },
   "source": [
    " # Table of Contents\n",
    "<div class=\"toc\" style=\"margin-top: 1em;\"><ul class=\"toc-item\" id=\"toc-level0\"><li><span><a href=\"http://localhost:8888/notebooks/barebone-yolo.ipynb#YOLO\" data-toc-modified-id=\"YOLO-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>YOLO</a></span><ul class=\"toc-item\"><li><span><a href=\"http://localhost:8888/notebooks/barebone-yolo.ipynb#Import-packages\" data-toc-modified-id=\"Import-packages-1.1\"><span class=\"toc-item-num\">1.1&nbsp;&nbsp;</span>Import packages</a></span></li><li><span><a href=\"http://localhost:8888/notebooks/barebone-yolo.ipynb#Define-and-initialize-global-variables\" data-toc-modified-id=\"Define-and-initialize-global-variables-1.2\"><span class=\"toc-item-num\">1.2&nbsp;&nbsp;</span>Define and initialize global variables</a></span></li><li><span><a href=\"http://localhost:8888/notebooks/barebone-yolo.ipynb#Construct-the-Network\" data-toc-modified-id=\"Construct-the-Network-1.3\"><span class=\"toc-item-num\">1.3&nbsp;&nbsp;</span>Construct the Network</a></span></li><li><span><a href=\"http://localhost:8888/notebooks/barebone-yolo.ipynb#Load-Pretrained-weights\" data-toc-modified-id=\"Load-Pretrained-weights-1.4\"><span class=\"toc-item-num\">1.4&nbsp;&nbsp;</span>Load Pretrained weights</a></span></li><li><span><a href=\"http://localhost:8888/notebooks/barebone-yolo.ipynb#Randomize-weights-of-the-last-layer\" data-toc-modified-id=\"Randomize-weights-of-the-last-layer-1.5\"><span class=\"toc-item-num\">1.5&nbsp;&nbsp;</span>Randomize weights of the last layer</a></span></li><li><span><a href=\"http://localhost:8888/notebooks/barebone-yolo.ipynb#Training\" data-toc-modified-id=\"Training-1.6\"><span class=\"toc-item-num\">1.6&nbsp;&nbsp;</span>Training</a></span><ul class=\"toc-item\"><li><span><a href=\"http://localhost:8888/notebooks/barebone-yolo.ipynb#Loss-Function\" data-toc-modified-id=\"Loss-Function-1.6.1\"><span class=\"toc-item-num\">1.6.1&nbsp;&nbsp;</span>Loss Function</a></span></li><li><span><a href=\"http://localhost:8888/notebooks/barebone-yolo.ipynb#Parse-the-annotations-to-construct-train-generator-and-validation-generator\" data-toc-modified-id=\"Parse-the-annotations-to-construct-train-generator-and-validation-generator-1.6.2\"><span class=\"toc-item-num\">1.6.2&nbsp;&nbsp;</span>Parse the annotations to construct train generator and validation generator</a></span></li></ul></li><li><span><a href=\"http://localhost:8888/notebooks/barebone-yolo.ipynb#Setup-a-few-callbacks-and-start-the-training\" data-toc-modified-id=\"Setup-a-few-callbacks-and-start-the-training-1.7\"><span class=\"toc-item-num\">1.7&nbsp;&nbsp;</span>Setup a few callbacks and start the training</a></span></li></ul></li></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# YOLO"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras import models\n",
    "from keras import layers\n",
    "from keras import callbacks\n",
    "from keras import optimizers\n",
    "from keras.utils.vis_utils import plot_model\n",
    "import keras.backend as K\n",
    "import tensorflow as tf\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib\n",
    "matplotlib.style.use('seaborn')\n",
    "import numpy as np\n",
    "import os\n",
    "import cv2\n",
    "import imgaug as ia\n",
    "from imgaug import augmenters as iaa\n",
    "from preprocessing import parse_annotation, BatchGenerator\n",
    "from utils import WeightReader, decode_netout, draw_boxes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define and initialize global variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "LABELS = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush']\n",
    "\n",
    "IMAGE_H, IMAGE_W = 416, 416\n",
    "GRID_H,  GRID_W  = 13 , 13\n",
    "BOX              = 5\n",
    "CLASS            = len(LABELS)\n",
    "CLASS_WEIGHTS    = np.ones(CLASS, dtype='float32')\n",
    "OBJ_THRESHOLD    = 0.3#0.5\n",
    "NMS_THRESHOLD    = 0.3#0.45\n",
    "ANCHORS          = [0.57273, 0.677385, 1.87446, 2.06253, 3.33843, 5.47434, 7.88282, 3.52778, 9.77052, 9.16828]\n",
    "\n",
    "NO_OBJECT_SCALE  = 1.0\n",
    "OBJECT_SCALE     = 5.0\n",
    "COORD_SCALE      = 1.0\n",
    "CLASS_SCALE      = 1.0\n",
    "\n",
    "BATCH_SIZE       = 16\n",
    "WARM_UP_BATCHES  = 0\n",
    "TRUE_BOX_BUFFER  = 50\n",
    "\n",
    "\n",
    "ALPHA = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pre_trained_weights='weights/yolo.weights'\n",
    "train_image_folder = '/home/abanihi/Documents/deep-data/coco/images/train2014/'\n",
    "train_annot_folder = '/home/abanihi/Documents/deep-data/coco/train2014ann/'\n",
    "val_image_folder = '/home/abanihi/Documents/deep-data/coco/images/val2014/'\n",
    "val_annot_folder = '/home/abanihi/Documents/deep-data/coco/val2014ann/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Construct the Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# the function to implement the orgnization layer (thanks to github.com/allanzelener/YAD2K)\n",
    "def space_to_depth_x2(x):\n",
    "    return tf.space_to_depth(x, block_size=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "input_image = layers.Input(shape=(IMAGE_H, IMAGE_W, 3))\n",
    "true_boxes  = layers.Input(shape=(1, 1, 1, TRUE_BOX_BUFFER , 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def yolo():\n",
    "    \n",
    " \n",
    "    # Layer 1\n",
    "    x = layers.Conv2D(32, (3, 3), strides=(1, 1), \n",
    "                        padding='same', name='conv_1', use_bias=False)(input_image)\n",
    "    x = layers.BatchNormalization(name='norm_1')(x)\n",
    "    x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)\n",
    "    x = layers.MaxPool2D(pool_size=(2,2))(x)\n",
    "    \n",
    "    # Layer 2\n",
    "    x = layers.Conv2D(64, (3, 3), strides=(1, 1), padding='same', name='conv_2', use_bias=False)(x)\n",
    "    x = layers.BatchNormalization(name='norm_2')(x)\n",
    "    x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)\n",
    "    x = layers.MaxPooling2D(pool_size=(2, 2))(x)\n",
    "    \n",
    "    \n",
    "    # Layer 3\n",
    "    x = layers.Conv2D(128, (3, 3), strides=(1, 1), padding='same', name='conv_3', use_bias=False)(x)\n",
    "    x = layers.BatchNormalization(name='norm_3')(x)\n",
    "    x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)\n",
    "    \n",
    "    # Layer 4 \n",
    "    x = layers.Conv2D(64, (1, 1), strides=(1, 1), padding='same', name='conv_4', use_bias=False)(x)\n",
    "    x = layers.BatchNormalization(name='norm_4')(x)\n",
    "    x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)\n",
    "    \n",
    "    # Layer 5\n",
    "    x = layers.Conv2D(128, (3, 3), strides=(1, 1), padding='same', name='conv_5', use_bias=False)(x)\n",
    "    x = layers.BatchNormalization(name='norm_5')(x)\n",
    "    x= layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)\n",
    "    x = layers.MaxPooling2D(pool_size=(2, 2))(x)\n",
    "    \n",
    "    # Layer 6\n",
    "    x = layers.Conv2D(256, (3, 3), strides=(1, 1), padding='same', name='conv_6', use_bias=False)(x)\n",
    "    x = layers.BatchNormalization(name='norm_6')(x)\n",
    "    x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)\n",
    "    \n",
    "    \n",
    "    # Layer 7\n",
    "    x = layers.Conv2D(128, (1, 1), strides=(1, 1), padding='same', name='conv_7', use_bias=False)(x)\n",
    "    x= layers.BatchNormalization(name='norm_7')(x)\n",
    "    x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)\n",
    "    \n",
    "    # Layer 8\n",
    "    x = layers.Conv2D(256, (3, 3), strides=(1, 1), padding='same', name='conv_8', use_bias=False)(x)\n",
    "    x = layers.BatchNormalization(name='norm_8')(x)\n",
    "    x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)\n",
    "    x = layers.MaxPooling2D(pool_size=(2, 2))(x)\n",
    "    \n",
    "    # Layer 9\n",
    "    x = layers.Conv2D(512, (3, 3), strides=(1, 1), padding='same', name='conv_9', use_bias=False)(x)\n",
    "    x = layers.BatchNormalization(name='norm_9')(x)\n",
    "    x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)\n",
    "    \n",
    "    # Layer 10\n",
    "    x = layers.Conv2D(256, (1, 1), strides=(1, 1), padding='same', name='conv_10', use_bias=False)(x)\n",
    "    x = layers.BatchNormalization(name='norm_10')(x)\n",
    "    x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)\n",
    "    \n",
    "    # Layer 11\n",
    "    x = layers.Conv2D(512, (3, 3), strides=(1, 1), padding='same', name='conv_11', use_bias=False)(x)\n",
    "    x = layers.BatchNormalization(name='norm_11')(x)\n",
    "    x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)\n",
    "    \n",
    "    \n",
    "    # Layer 12\n",
    "    x = layers.Conv2D(256, (1, 1), strides=(1, 1), padding='same', name='conv_12', use_bias=False)(x)\n",
    "    x = layers.BatchNormalization(name='norm_12')(x)\n",
    "    x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)\n",
    "    \n",
    "    # Layer 13\n",
    "    x = layers.Conv2D(512, (3, 3), strides=(1, 1), padding='same', name='conv_13', use_bias=False)(x)\n",
    "    x = layers.BatchNormalization(name='norm_13')(x)\n",
    "    x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)\n",
    "    \n",
    "    \n",
    "    skip_connection = x\n",
    "    \n",
    "    x = layers.MaxPool2D(pool_size=(2, 2))(x)\n",
    "    \n",
    "    # Layer 14\n",
    "    x = layers.Conv2D(1024, (3, 3), strides=(1, 1), padding='same', name='conv_14', use_bias=False)(x)\n",
    "    x = layers.BatchNormalization(name='norm_14')(x)\n",
    "    x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)\n",
    "    \n",
    "    # Layer 15\n",
    "    x = layers.Conv2D(512, (1, 1), strides=(1, 1), padding='same', name='conv_15', use_bias=False)(x)\n",
    "    x = layers.BatchNormalization(name='norm_15')(x)\n",
    "    x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)\n",
    "    \n",
    "    # Layer 16\n",
    "    x = layers.Conv2D(1024, (3, 3), strides=(1, 1), padding='same', name='conv_16', use_bias=False)(x)\n",
    "    x = layers.BatchNormalization(name='norm_16')(x)\n",
    "    x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)\n",
    "    \n",
    "    # Layer 17\n",
    "    x = layers.Conv2D(512, (1, 1), strides=(1, 1), padding='same', name='conv_17', use_bias=False)(x)\n",
    "    x = layers.BatchNormalization(name='norm_17')(x)\n",
    "    x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)\n",
    "    \n",
    "    # Layer 18\n",
    "    x = layers.Conv2D(1024, (3, 3), strides=(1, 1), padding='same', name='conv_18', use_bias=False)(x)\n",
    "    x = layers.BatchNormalization(name='norm_18')(x)\n",
    "    x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)\n",
    "    \n",
    "    # Layer 19\n",
    "    x = layers.Conv2D(1024, (3, 3), strides=(1, 1), padding='same', name='conv_19', use_bias=False)(x)\n",
    "    x = layers.BatchNormalization(name='norm_19')(x)\n",
    "    x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)\n",
    "    \n",
    "    # Layer 20\n",
    "    x = layers.Conv2D(1024, (3, 3), strides=(1, 1), padding='same', name='conv_20', use_bias=False)(x)\n",
    "    x = layers.BatchNormalization(name='norm_20')(x)\n",
    "    x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)\n",
    "    \n",
    "    \n",
    "    # Layer 21\n",
    "    skip_connection = layers.Conv2D(64, (1, 1), strides=(1, 1), \n",
    "                                padding='same', name='conv_21', use_bias=False)(skip_connection)\n",
    "    skip_connection = layers.BatchNormalization(name='norm_21')(skip_connection)\n",
    "    skip_connection = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(skip_connection)\n",
    "    skip_connection = layers.Lambda(space_to_depth_x2)(skip_connection)\n",
    "    \n",
    "    x = layers.concatenate([skip_connection, x])\n",
    "    \n",
    "    # Layer 22\n",
    "    x = layers.Conv2D(1024, (3, 3), strides=(1, 1), padding='same', name='conv_22',\n",
    "                     use_bias=False)(x)\n",
    "    x = layers.BatchNormalization(name='norm_22')(x)\n",
    "    x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)\n",
    "    \n",
    "    # Layer 23\n",
    "    x = layers.Conv2D((4 + 1 + CLASS) * 5, (1,1), strides=(1,1), padding='same', name='conv_23')(x)\n",
    "    output = layers.Reshape((GRID_H, GRID_W, BOX, 4 + 1 + CLASS))(x)\n",
    "    \n",
    "    # small hack to allow true_boxes to be registered when Keras build the model \n",
    "    # for more information: https://github.com/fchollet/keras/issues/2790\n",
    "    output = layers.Lambda(lambda args: args[0])([output, true_boxes])\n",
    "    \n",
    "    model = models.Model([input_image, true_boxes], output)\n",
    "                                    \n",
    "    \n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_1 (InputLayer)             (None, 416, 416, 3)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv_1 (Conv2D)                  (None, 416, 416, 32)  864         input_1[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "norm_1 (BatchNormalization)      (None, 416, 416, 32)  128         conv_1[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_1 (LeakyReLU)        (None, 416, 416, 32)  0           norm_1[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)   (None, 208, 208, 32)  0           leaky_re_lu_1[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "conv_2 (Conv2D)                  (None, 208, 208, 64)  18432       max_pooling2d_1[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "norm_2 (BatchNormalization)      (None, 208, 208, 64)  256         conv_2[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_2 (LeakyReLU)        (None, 208, 208, 64)  0           norm_2[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2D)   (None, 104, 104, 64)  0           leaky_re_lu_2[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "conv_3 (Conv2D)                  (None, 104, 104, 128) 73728       max_pooling2d_2[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "norm_3 (BatchNormalization)      (None, 104, 104, 128) 512         conv_3[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_3 (LeakyReLU)        (None, 104, 104, 128) 0           norm_3[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "conv_4 (Conv2D)                  (None, 104, 104, 64)  8192        leaky_re_lu_3[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "norm_4 (BatchNormalization)      (None, 104, 104, 64)  256         conv_4[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_4 (LeakyReLU)        (None, 104, 104, 64)  0           norm_4[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "conv_5 (Conv2D)                  (None, 104, 104, 128) 73728       leaky_re_lu_4[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "norm_5 (BatchNormalization)      (None, 104, 104, 128) 512         conv_5[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_5 (LeakyReLU)        (None, 104, 104, 128) 0           norm_5[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2D)   (None, 52, 52, 128)   0           leaky_re_lu_5[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "conv_6 (Conv2D)                  (None, 52, 52, 256)   294912      max_pooling2d_3[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "norm_6 (BatchNormalization)      (None, 52, 52, 256)   1024        conv_6[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_6 (LeakyReLU)        (None, 52, 52, 256)   0           norm_6[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "conv_7 (Conv2D)                  (None, 52, 52, 128)   32768       leaky_re_lu_6[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "norm_7 (BatchNormalization)      (None, 52, 52, 128)   512         conv_7[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_7 (LeakyReLU)        (None, 52, 52, 128)   0           norm_7[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "conv_8 (Conv2D)                  (None, 52, 52, 256)   294912      leaky_re_lu_7[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "norm_8 (BatchNormalization)      (None, 52, 52, 256)   1024        conv_8[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_8 (LeakyReLU)        (None, 52, 52, 256)   0           norm_8[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2D)   (None, 26, 26, 256)   0           leaky_re_lu_8[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "conv_9 (Conv2D)                  (None, 26, 26, 512)   1179648     max_pooling2d_4[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "norm_9 (BatchNormalization)      (None, 26, 26, 512)   2048        conv_9[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_9 (LeakyReLU)        (None, 26, 26, 512)   0           norm_9[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "conv_10 (Conv2D)                 (None, 26, 26, 256)   131072      leaky_re_lu_9[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "norm_10 (BatchNormalization)     (None, 26, 26, 256)   1024        conv_10[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_10 (LeakyReLU)       (None, 26, 26, 256)   0           norm_10[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_11 (Conv2D)                 (None, 26, 26, 512)   1179648     leaky_re_lu_10[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "norm_11 (BatchNormalization)     (None, 26, 26, 512)   2048        conv_11[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_11 (LeakyReLU)       (None, 26, 26, 512)   0           norm_11[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_12 (Conv2D)                 (None, 26, 26, 256)   131072      leaky_re_lu_11[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "norm_12 (BatchNormalization)     (None, 26, 26, 256)   1024        conv_12[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_12 (LeakyReLU)       (None, 26, 26, 256)   0           norm_12[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_13 (Conv2D)                 (None, 26, 26, 512)   1179648     leaky_re_lu_12[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "norm_13 (BatchNormalization)     (None, 26, 26, 512)   2048        conv_13[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_13 (LeakyReLU)       (None, 26, 26, 512)   0           norm_13[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_5 (MaxPooling2D)   (None, 13, 13, 512)   0           leaky_re_lu_13[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "conv_14 (Conv2D)                 (None, 13, 13, 1024)  4718592     max_pooling2d_5[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "norm_14 (BatchNormalization)     (None, 13, 13, 1024)  4096        conv_14[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_14 (LeakyReLU)       (None, 13, 13, 1024)  0           norm_14[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_15 (Conv2D)                 (None, 13, 13, 512)   524288      leaky_re_lu_14[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "norm_15 (BatchNormalization)     (None, 13, 13, 512)   2048        conv_15[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_15 (LeakyReLU)       (None, 13, 13, 512)   0           norm_15[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_16 (Conv2D)                 (None, 13, 13, 1024)  4718592     leaky_re_lu_15[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "norm_16 (BatchNormalization)     (None, 13, 13, 1024)  4096        conv_16[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_16 (LeakyReLU)       (None, 13, 13, 1024)  0           norm_16[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_17 (Conv2D)                 (None, 13, 13, 512)   524288      leaky_re_lu_16[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "norm_17 (BatchNormalization)     (None, 13, 13, 512)   2048        conv_17[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_17 (LeakyReLU)       (None, 13, 13, 512)   0           norm_17[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_18 (Conv2D)                 (None, 13, 13, 1024)  4718592     leaky_re_lu_17[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "norm_18 (BatchNormalization)     (None, 13, 13, 1024)  4096        conv_18[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_18 (LeakyReLU)       (None, 13, 13, 1024)  0           norm_18[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_19 (Conv2D)                 (None, 13, 13, 1024)  9437184     leaky_re_lu_18[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "norm_19 (BatchNormalization)     (None, 13, 13, 1024)  4096        conv_19[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_21 (Conv2D)                 (None, 26, 26, 64)    32768       leaky_re_lu_13[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_19 (LeakyReLU)       (None, 13, 13, 1024)  0           norm_19[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "norm_21 (BatchNormalization)     (None, 26, 26, 64)    256         conv_21[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_20 (Conv2D)                 (None, 13, 13, 1024)  9437184     leaky_re_lu_19[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_21 (LeakyReLU)       (None, 26, 26, 64)    0           norm_21[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "norm_20 (BatchNormalization)     (None, 13, 13, 1024)  4096        conv_20[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)                (None, 13, 13, 256)   0           leaky_re_lu_21[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_20 (LeakyReLU)       (None, 13, 13, 1024)  0           norm_20[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)      (None, 13, 13, 1280)  0           lambda_1[0][0]                   \n",
      "                                                                   leaky_re_lu_20[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "conv_22 (Conv2D)                 (None, 13, 13, 1024)  11796480    concatenate_1[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "norm_22 (BatchNormalization)     (None, 13, 13, 1024)  4096        conv_22[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_22 (LeakyReLU)       (None, 13, 13, 1024)  0           norm_22[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_23 (Conv2D)                 (None, 13, 13, 425)   435625      leaky_re_lu_22[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "reshape_1 (Reshape)              (None, 13, 13, 5, 85) 0           conv_23[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "input_2 (InputLayer)             (None, 1, 1, 1, 50, 4 0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lambda_2 (Lambda)                (None, 13, 13, 5, 85) 0           reshape_1[0][0]                  \n",
      "                                                                   input_2[0][0]                    \n",
      "====================================================================================================\n",
      "Total params: 50,983,561\n",
      "Trainable params: 50,962,889\n",
      "Non-trainable params: 20,672\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = yolo()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plot_model(model, to_file='model.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Total params: 50,983,561\n",
    "Trainable params: 50,962,889\n",
    "Non-trainable params: 20,672"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Pretrained weights\n",
    "\n",
    "Load the weights originally provided by YOLO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "weight_reader = WeightReader(pre_trained_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "weight_reader.reset()\n",
    "nb_conv = 23"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in range(1, nb_conv+1):\n",
    "    conv_layer = model.get_layer('conv_' + str(i))\n",
    "    \n",
    "    if i < nb_conv:\n",
    "        norm_layer = model.get_layer('norm_' + str(i))\n",
    "        \n",
    "        size = np.prod(norm_layer.get_weights()[0].shape)\n",
    "        \n",
    "        beta = weight_reader.read_bytes(size)\n",
    "        gamma = weight_reader.read_bytes(size)\n",
    "        mean = weight_reader.read_bytes(size)\n",
    "        var = weight_reader.read_bytes(size)\n",
    "        \n",
    "        weights = norm_layer.set_weights([gamma, beta, mean, var])\n",
    "        \n",
    "    if len(conv_layer.get_weights()) > 1:\n",
    "        bias = weight_reader.read_bytes(np.prod(conv_layer.get_weights()[1].shape))\n",
    "        kernel = weight_reader.read_bytes(np.prod(conv_layer.get_weights()[0].shape))\n",
    "        kernel = kernel.reshape(list(reversed(conv_layer.get_weights()[0].shape)))\n",
    "        kernel = kernel.transpose([2,3,1,0])\n",
    "        conv_layer.set_weights([kernel, bias])\n",
    "        \n",
    "    else:\n",
    "        kernel = weight_reader.read_bytes(np.prod(conv_layer.get_weights()[0].shape))\n",
    "        kernel = kernel.reshape(list(reversed(conv_layer.get_weights()[0].shape)))\n",
    "        kernel = kernel.transpose([2,3,1,0])\n",
    "        conv_layer.set_weights([kernel])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Randomize weights of the last layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Get last convolutional layer\n",
    "layer = model.layers[-4] \n",
    "weights = layer.get_weights()\n",
    "\n",
    "new_kernel = np.random.normal(size=weights[0].shape) / (GRID_H*GRID_W)\n",
    "new_bias   = np.random.normal(size=weights[1].shape) / (GRID_H*GRID_W)\n",
    "\n",
    "layer.set_weights([new_kernel, new_bias])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loss Function\n",
    "\n",
    "![](images/custom-loss.png)\n",
    "\n",
    "![](images/custom-loss2.png)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def custom_loss(y_true, y_pred):\n",
    "    mask_shape = tf.shape(y_true)[:4]\n",
    "    \n",
    "    cell_x = tf.to_float(tf.reshape(tf.tile(tf.range(GRID_W), [GRID_H]), (1, GRID_H, GRID_W, 1, 1)))\n",
    "    cell_y = tf.transpose(cell_x, (0,2,1,3,4))\n",
    "\n",
    "    cell_grid = tf.tile(tf.concat([cell_x,cell_y], -1), [BATCH_SIZE, 1, 1, 5, 1])\n",
    "    \n",
    "    coord_mask = tf.zeros(mask_shape)\n",
    "    conf_mask  = tf.zeros(mask_shape)\n",
    "    class_mask = tf.zeros(mask_shape)\n",
    "    \n",
    "    seen = tf.Variable(0.)\n",
    "    \n",
    "    total_AP = tf.Variable(0.)\n",
    "    \n",
    "    \"\"\"\n",
    "    Adjust prediction\n",
    "    \"\"\"\n",
    "    ### adjust x and y      \n",
    "    pred_box_xy = tf.sigmoid(y_pred[..., :2]) + cell_grid\n",
    "    \n",
    "    ### adjust w and h\n",
    "    pred_box_wh = tf.exp(y_pred[..., 2:4]) * np.reshape(ANCHORS, [1,1,1,BOX,2])\n",
    "    \n",
    "    ### adjust confidence\n",
    "    pred_box_conf = tf.sigmoid(y_pred[..., 4])\n",
    "    \n",
    "    ### adjust class probabilities\n",
    "    pred_box_class = y_pred[..., 5:]\n",
    "    \n",
    "    \"\"\"\n",
    "    Adjust ground truth\n",
    "    \"\"\"\n",
    "    ### adjust x and y\n",
    "    true_box_xy = y_true[..., 0:2] # relative position to the containing cell\n",
    "    \n",
    "    ### adjust w and h\n",
    "    true_box_wh = y_true[..., 2:4] # number of cells accross, horizontally and vertically\n",
    "    \n",
    "    ### adjust confidence\n",
    "    true_wh_half = true_box_wh / 2.\n",
    "    true_mins    = true_box_xy - true_wh_half\n",
    "    true_maxes   = true_box_xy + true_wh_half\n",
    "    \n",
    "    pred_wh_half = pred_box_wh / 2.\n",
    "    pred_mins    = pred_box_xy - pred_wh_half\n",
    "    pred_maxes   = pred_box_xy + pred_wh_half       \n",
    "    \n",
    "    intersect_mins  = tf.maximum(pred_mins,  true_mins)\n",
    "    intersect_maxes = tf.minimum(pred_maxes, true_maxes)\n",
    "    intersect_wh    = tf.maximum(intersect_maxes - intersect_mins, 0.)\n",
    "    intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]\n",
    "    \n",
    "    true_areas = true_box_wh[..., 0] * true_box_wh[..., 1]\n",
    "    pred_areas = pred_box_wh[..., 0] * pred_box_wh[..., 1]\n",
    "\n",
    "    union_areas = pred_areas + true_areas - intersect_areas\n",
    "    iou_scores  = tf.truediv(intersect_areas, union_areas)\n",
    "    \n",
    "    true_box_conf = iou_scores * y_true[..., 4]\n",
    "    \n",
    "    ### adjust class probabilities\n",
    "    true_box_class = tf.to_int32(y_true[..., 5])\n",
    "    \n",
    "    \"\"\"\n",
    "    Determine the masks\n",
    "    \"\"\"\n",
    "    ### coordinate mask: simply the position of the ground truth boxes (the predictors)\n",
    "    coord_mask = tf.expand_dims(y_true[..., 4], axis=-1) * COORD_SCALE\n",
    "    \n",
    "    ### confidence mask: penelize predictors + penalize boxes with low IOU\n",
    "    # penalize the confidence of the boxes, which have IOU with some ground truth box < 0.6\n",
    "    true_xy = true_boxes[..., 0:2]\n",
    "    true_wh = true_boxes[..., 2:4]\n",
    "    \n",
    "    true_wh_half = true_wh / 2.\n",
    "    true_mins    = true_xy - true_wh_half\n",
    "    true_maxes   = true_xy + true_wh_half\n",
    "    \n",
    "    pred_xy = tf.expand_dims(pred_box_xy, 4)\n",
    "    pred_wh = tf.expand_dims(pred_box_wh, 4)\n",
    "    \n",
    "    pred_wh_half = pred_wh / 2.\n",
    "    pred_mins    = pred_xy - pred_wh_half\n",
    "    pred_maxes   = pred_xy + pred_wh_half    \n",
    "    \n",
    "    intersect_mins  = tf.maximum(pred_mins,  true_mins)\n",
    "    intersect_maxes = tf.minimum(pred_maxes, true_maxes)\n",
    "    intersect_wh    = tf.maximum(intersect_maxes - intersect_mins, 0.)\n",
    "    intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]\n",
    "    \n",
    "    true_areas = true_wh[..., 0] * true_wh[..., 1]\n",
    "    pred_areas = pred_wh[..., 0] * pred_wh[..., 1]\n",
    "\n",
    "    union_areas = pred_areas + true_areas - intersect_areas\n",
    "    iou_scores  = tf.truediv(intersect_areas, union_areas)\n",
    "\n",
    "    best_ious = tf.reduce_max(iou_scores, axis=4)\n",
    "    conf_mask = conf_mask + tf.to_float(best_ious < 0.6) * (1 - y_true[..., 4]) * NO_OBJECT_SCALE\n",
    "    \n",
    "    # penalize the confidence of the boxes, which are reponsible for corresponding ground truth box\n",
    "    conf_mask = conf_mask + y_true[..., 4] * OBJECT_SCALE\n",
    "    \n",
    "    ### class mask: simply the position of the ground truth boxes (the predictors)\n",
    "    class_mask = y_true[..., 4] * tf.gather(CLASS_WEIGHTS, true_box_class) * CLASS_SCALE       \n",
    "    \n",
    "    \"\"\"\n",
    "    Warm-up training\n",
    "    \"\"\"\n",
    "    no_boxes_mask = tf.to_float(coord_mask < COORD_SCALE/2.)\n",
    "    seen = tf.assign_add(seen, 1.)\n",
    "    \n",
    "    true_box_xy, true_box_wh, coord_mask = tf.cond(tf.less(seen, WARM_UP_BATCHES), \n",
    "                          lambda: [true_box_xy + (0.5 + cell_grid) * no_boxes_mask, \n",
    "                                   true_box_wh + tf.ones_like(true_box_wh) * np.reshape(ANCHORS, [1,1,1,BOX,2]) * no_boxes_mask, \n",
    "                                   tf.ones_like(coord_mask)],\n",
    "                          lambda: [true_box_xy, \n",
    "                                   true_box_wh,\n",
    "                                   coord_mask])\n",
    "    \n",
    "    \"\"\"\n",
    "    Finalize the loss\n",
    "    \"\"\"\n",
    "    nb_coord_box = tf.reduce_sum(tf.to_float(coord_mask > 0.0))\n",
    "    nb_conf_box  = tf.reduce_sum(tf.to_float(conf_mask  > 0.0))\n",
    "    nb_class_box = tf.reduce_sum(tf.to_float(class_mask > 0.0))\n",
    "    \n",
    "    loss_xy    = tf.reduce_sum(tf.square(true_box_xy-pred_box_xy)     * coord_mask) / (nb_coord_box + 1e-6) / 2.\n",
    "    loss_wh    = tf.reduce_sum(tf.square(true_box_wh-pred_box_wh)     * coord_mask) / (nb_coord_box + 1e-6) / 2.\n",
    "    loss_conf  = tf.reduce_sum(tf.square(true_box_conf-pred_box_conf) * conf_mask)  / (nb_conf_box  + 1e-6) / 2.\n",
    "    loss_class = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=true_box_class, logits=pred_box_class)\n",
    "    loss_class = tf.reduce_sum(loss_class * class_mask) / (nb_class_box + 1e-6)\n",
    "    \n",
    "    loss = loss_xy + loss_wh + loss_conf + loss_class\n",
    "    \n",
    "    nb_true_box = tf.reduce_sum(y_true[..., 4])\n",
    "    nb_pred_box = tf.reduce_sum(tf.to_float(true_box_conf > 0.5) * tf.to_float(pred_box_conf > OBJ_THRESHOLD))\n",
    "    \n",
    "    total_AP = tf.assign_add(total_AP, nb_pred_box/nb_true_box) \n",
    "    \n",
    "    loss = tf.Print(loss, [loss_xy, loss_wh, loss_conf, loss_class, loss, total_AP/seen], message='DEBUG', summarize=1000)\n",
    "    \n",
    "    return loss\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parse the annotations to construct train generator and validation generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "generator_config = {\n",
    "    'IMAGE_H'         : IMAGE_H, \n",
    "    'IMAGE_W'         : IMAGE_W,\n",
    "    'GRID_H'          : GRID_H,  \n",
    "    'GRID_W'          : GRID_W,\n",
    "    'BOX'             : BOX,\n",
    "    'LABELS'          : LABELS,\n",
    "    'CLASS'           : len(LABELS),\n",
    "    'ANCHORS'         : ANCHORS,\n",
    "    'BATCH_SIZE'      : BATCH_SIZE,\n",
    "    'TRUE_BOX_BUFFER' : 50,\n",
    "}\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 26.6 s, sys: 5.42 s, total: 32 s\n",
      "Wall time: 11min 35s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_imgs, seen_train_labels = parse_annotation(train_annot_folder, train_image_folder, labels=LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import cv2\n",
    "import copy\n",
    "import numpy as np\n",
    "import imgaug as ia\n",
    "from imgaug import augmenters as iaa\n",
    "import xml.etree.ElementTree as ET\n",
    "from utils import BoundBox, normalize, bbox_iou"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class BatchGenerator:\n",
    "    def __init__(self, images, \n",
    "                       config, \n",
    "                       shuffle=True, \n",
    "                       jitter=True, \n",
    "                       norm=True):\n",
    "\n",
    "        self.images = images\n",
    "        self.config = config\n",
    "\n",
    "        self.shuffle = shuffle\n",
    "        self.jitter  = jitter\n",
    "        self.norm    = norm\n",
    "        \n",
    "\n",
    "        self.anchors = [BoundBox(0, 0, config['ANCHORS'][2*i], config['ANCHORS'][2*i+1]) for i in range(int(len(config['ANCHORS'])/2))]\n",
    "\n",
    "        ### augmentors by https://github.com/aleju/imgaug\n",
    "        sometimes = lambda aug: iaa.Sometimes(0.5, aug)\n",
    "\n",
    "        # Define our sequence of augmentation steps that will be applied to every image\n",
    "        # All augmenters with per_channel=0.5 will sample one value _per image_\n",
    "        # in 50% of all cases. In all other cases they will sample new values\n",
    "        # _per channel_.\n",
    "        self.aug_pipe = iaa.Sequential(\n",
    "            [\n",
    "                # apply the following augmenters to most images\n",
    "                #iaa.Fliplr(0.5), # horizontally flip 50% of all images\n",
    "                #iaa.Flipud(0.2), # vertically flip 20% of all images\n",
    "                #sometimes(iaa.Crop(percent=(0, 0.1))), # crop images by 0-10% of their height/width\n",
    "                sometimes(iaa.Affine(\n",
    "                    #scale={\"x\": (0.8, 1.2), \"y\": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis\n",
    "                    #translate_percent={\"x\": (-0.2, 0.2), \"y\": (-0.2, 0.2)}, # translate by -20 to +20 percent (per axis)\n",
    "                    #rotate=(-5, 5), # rotate by -45 to +45 degrees\n",
    "                    #shear=(-5, 5), # shear by -16 to +16 degrees\n",
    "                    #order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)\n",
    "                    #cval=(0, 255), # if mode is constant, use a cval between 0 and 255\n",
    "                    #mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)\n",
    "                )),\n",
    "                # execute 0 to 5 of the following (less important) augmenters per image\n",
    "                # don't execute all of them, as that would often be way too strong\n",
    "                iaa.SomeOf((0, 5),\n",
    "                    [\n",
    "                        #sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation\n",
    "                        iaa.OneOf([\n",
    "                            iaa.GaussianBlur((0, 3.0)), # blur images with a sigma between 0 and 3.0\n",
    "                            iaa.AverageBlur(k=(2, 7)), # blur image using local means with kernel sizes between 2 and 7\n",
    "                            iaa.MedianBlur(k=(3, 11)), # blur image using local medians with kernel sizes between 2 and 7\n",
    "                        ]),\n",
    "                        iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images\n",
    "                        #iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images\n",
    "                        # search either for all edges or for directed edges\n",
    "                        #sometimes(iaa.OneOf([\n",
    "                        #    iaa.EdgeDetect(alpha=(0, 0.7)),\n",
    "                        #    iaa.DirectedEdgeDetect(alpha=(0, 0.7), direction=(0.0, 1.0)),\n",
    "                        #])),\n",
    "                        iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5), # add gaussian noise to images\n",
    "                        iaa.OneOf([\n",
    "                            iaa.Dropout((0.01, 0.1), per_channel=0.5), # randomly remove up to 10% of the pixels\n",
    "                            #iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2),\n",
    "                        ]),\n",
    "                        #iaa.Invert(0.05, per_channel=True), # invert color channels\n",
    "                        iaa.Add((-10, 10), per_channel=0.5), # change brightness of images (by -10 to 10 of original value)\n",
    "                        iaa.Multiply((0.5, 1.5), per_channel=0.5), # change brightness of images (50-150% of original value)\n",
    "                        iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5), # improve or worsen the contrast\n",
    "                        #iaa.Grayscale(alpha=(0.0, 1.0)),\n",
    "                        #sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths)\n",
    "                        #sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) # sometimes move parts of the image around\n",
    "                    ],\n",
    "                    random_order=True\n",
    "                )\n",
    "            ],\n",
    "            random_order=True\n",
    "        )\n",
    "\n",
    "        if shuffle: np.random.shuffle(self.images)\n",
    "\n",
    "    def get_generator(self):\n",
    "        num_img = len(self.images)\n",
    "        \n",
    "        total_count = 0\n",
    "        batch_count = 0\n",
    "        \n",
    "        x_batch = np.zeros((self.config['BATCH_SIZE'], self.config['IMAGE_H'], self.config['IMAGE_W'], 3))                         # input images\n",
    "        b_batch = np.zeros((self.config['BATCH_SIZE'], 1     , 1     , 1    ,  self.config['TRUE_BOX_BUFFER'], 4))   # list of self.config['TRUE_self.config['BOX']_BUFFER'] GT boxes\n",
    "        y_batch = np.zeros((self.config['BATCH_SIZE'], self.config['GRID_H'],  self.config['GRID_W'], self.config['BOX'], 4+1+1))                # desired network output\n",
    "        \n",
    "        while True:\n",
    "            if total_count < num_img:\n",
    "                train_instance = self.images[total_count]\n",
    "\n",
    "                # augment input image and fix object's position and size\n",
    "                img, all_objs = self.aug_image(train_instance, jitter=self.jitter)\n",
    "                \n",
    "                # construct output from object's x, y, w, h\n",
    "                true_box_index = 0\n",
    "                \n",
    "                for obj in all_objs:\n",
    "                    if obj['xmax'] > obj['xmin'] and obj['ymax'] > obj['ymin'] and obj['name'] in self.config['LABELS']:\n",
    "                        center_x = .5*(obj['xmin'] + obj['xmax'])\n",
    "                        center_x = center_x / (float(self.config['IMAGE_W']) / self.config['GRID_W'])\n",
    "                        center_y = .5*(obj['ymin'] + obj['ymax'])\n",
    "                        center_y = center_y / (float(self.config['IMAGE_H']) / self.config['GRID_H'])\n",
    "\n",
    "                        grid_x = int(np.floor(center_x))\n",
    "                        grid_y = int(np.floor(center_y))\n",
    "\n",
    "                        if grid_x < self.config['GRID_W'] and grid_y < self.config['GRID_H']:\n",
    "                            obj_indx  = self.config['LABELS'].index(obj['name'])\n",
    "                            \n",
    "                            center_w = (obj['xmax'] - obj['xmin']) / (float(self.config['IMAGE_W']) / self.config['GRID_W']) # unit: grid cell\n",
    "                            center_h = (obj['ymax'] - obj['ymin']) / (float(self.config['IMAGE_W']) / self.config['GRID_W']) # unit: grid cell\n",
    "                            \n",
    "                            box = [center_x, center_y, center_w, center_h]\n",
    "\n",
    "                            # find the anchor that best predicts this box\n",
    "                            best_anchor = -1\n",
    "                            max_iou     = -1\n",
    "                            \n",
    "                            shifted_box = BoundBox(0, \n",
    "                                                   0, \n",
    "                                                   center_w, \n",
    "                                                   center_h)\n",
    "                            \n",
    "                            for i in range(len(self.anchors)):\n",
    "                                anchor = self.anchors[i]\n",
    "                                iou    = bbox_iou(shifted_box, anchor)\n",
    "                                \n",
    "                                if max_iou < iou:\n",
    "                                    best_anchor = i\n",
    "                                    max_iou     = iou\n",
    "                                    \n",
    "                            # assign ground truth x, y, w, h, confidence and class probs to y_batch\n",
    "                            y_batch[batch_count, grid_y, grid_x, best_anchor, 0:4]        = box\n",
    "                            y_batch[batch_count, grid_y, grid_x, best_anchor, 4  ]        = 1.\n",
    "                            y_batch[batch_count, grid_y, grid_x, best_anchor, 5  ]        = obj_indx\n",
    "                            \n",
    "                            # assign the true box to b_batch\n",
    "                            b_batch[batch_count, 0, 0, 0, true_box_index] = box\n",
    "                            \n",
    "                            true_box_index += 1\n",
    "                            true_box_index = true_box_index % self.config['TRUE_BOX_BUFFER']\n",
    "                                \n",
    "                # assign input image to x_batch\n",
    "                if self.norm: \n",
    "                    x_batch[batch_count] = normalize(img)\n",
    "                else:\n",
    "                    # plot image and bounding boxes for sanity check\n",
    "                    for obj in all_objs:\n",
    "                        if obj['xmax'] > obj['xmin'] and obj['ymax'] > obj['ymin']:\n",
    "                            cv2.rectangle(img[:,:,::-1], (obj['xmin'],obj['ymin']), (obj['xmax'],obj['ymax']), (255,0,0), 3)\n",
    "                            cv2.putText(img[:,:,::-1], obj['name'], \n",
    "                                        (obj['xmin']+2, obj['ymin']+12), \n",
    "                                        0, 1.2e-3 * img.shape[0], \n",
    "                                        (0,255,0), 2)\n",
    "                            \n",
    "                    x_batch[batch_count] = img\n",
    "\n",
    "                # increase instance counter in current batch\n",
    "                batch_count += 1  \n",
    "                    \n",
    "            total_count += 1\n",
    "            if total_count >= num_img:\n",
    "                total_count = 0\n",
    "                if self.shuffle: np.random.shuffle(self.images)                    \n",
    "\n",
    "            if batch_count >= self.config['BATCH_SIZE']:\n",
    "                yield [x_batch, b_batch], y_batch\n",
    "                \n",
    "                x_batch = np.zeros((self.config['BATCH_SIZE'], self.config['IMAGE_H'], self.config['IMAGE_W'], 3))\n",
    "                y_batch = np.zeros((self.config['BATCH_SIZE'], self.config['GRID_H'],  self.config['GRID_W'],  self.config['BOX'], 5+self.config['CLASS']))       \n",
    "                \n",
    "                batch_count = 0\n",
    "\n",
    "    def aug_image(self, train_instance, jitter):\n",
    "        image_name = train_instance['filename']\n",
    "        image = cv2.imread(image_name)\n",
    "        h, w, c = image.shape\n",
    "        \n",
    "        all_objs = copy.deepcopy(train_instance['object'])\n",
    "\n",
    "        if jitter:\n",
    "            ### scale the image\n",
    "            scale = np.random.uniform() / 10. + 1.\n",
    "            image = cv2.resize(image, (0,0), fx = scale, fy = scale)\n",
    "\n",
    "            ### translate the image\n",
    "            max_offx = (scale-1.) * w\n",
    "            max_offy = (scale-1.) * h\n",
    "            offx = int(np.random.uniform() * max_offx)\n",
    "            offy = int(np.random.uniform() * max_offy)\n",
    "            \n",
    "            image = image[offy : (offy + h), offx : (offx + w)]\n",
    "\n",
    "            ### flip the image\n",
    "            flip = np.random.binomial(1, .5)\n",
    "            if flip > 0.5: image = cv2.flip(image, 1)\n",
    "                \n",
    "            image = self.aug_pipe.augment_image(image)            \n",
    "            \n",
    "        # resize the image to standard size\n",
    "        image = cv2.resize(image, (self.config['IMAGE_H'], self.config['IMAGE_W']))\n",
    "        image = image[:,:,::-1]\n",
    "\n",
    "        # fix object's position and size\n",
    "        for obj in all_objs:\n",
    "            for attr in ['xmin', 'xmax']:\n",
    "                if jitter: obj[attr] = int(obj[attr] * scale - offx)\n",
    "                    \n",
    "                obj[attr] = int(obj[attr] * float(self.config['IMAGE_W']) / w)\n",
    "                obj[attr] = max(min(obj[attr], self.config['IMAGE_W']), 0)\n",
    "                \n",
    "            for attr in ['ymin', 'ymax']:\n",
    "                if jitter: obj[attr] = int(obj[attr] * scale - offy)\n",
    "                    \n",
    "                obj[attr] = int(obj[attr] * float(self.config['IMAGE_H']) / h)\n",
    "                obj[attr] = max(min(obj[attr], self.config['IMAGE_H']), 0)\n",
    "\n",
    "            if jitter and flip > 0.5:\n",
    "                xmin = obj['xmin']\n",
    "                obj['xmin'] = self.config['IMAGE_W'] - obj['xmax']\n",
    "                obj['xmax'] = self.config['IMAGE_W'] - xmin\n",
    "                \n",
    "        return image, all_objs\n",
    "\n",
    "    def get_dateset_size(self):\n",
    "        return int(np.ceil(float(len(self.images))/self.config['BATCH_SIZE']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 8 ms, sys: 0 ns, total: 8 ms\n",
      "Wall time: 11.5 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_batch = BatchGenerator(train_imgs, generator_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 12.4 s, sys: 2.45 s, total: 14.8 s\n",
      "Wall time: 4min 50s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "val_imgs, seen_val_labels = parse_annotation(val_annot_folder, val_image_folder, labels=LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 8 ms, sys: 0 ns, total: 8 ms\n",
      "Wall time: 5.87 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "valid_batch = BatchGenerator(val_imgs, generator_config, jitter=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup a few callbacks and start the training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "early_stop = callbacks.EarlyStopping(monitor='val_loss', \n",
    "                           min_delta=0.001, \n",
    "                           patience=3, \n",
    "                           mode='min', \n",
    "                           verbose=1)\n",
    "\n",
    "checkpoint = callbacks.ModelCheckpoint('weights_coco.h5', \n",
    "                             monitor='val_loss', \n",
    "                             verbose=1, \n",
    "                             save_best_only=True, \n",
    "                             mode='min', \n",
    "                             period=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "ename": "OSError",
     "evalue": "Unable to open file (unable to open file: name = 'weights_coco.h5', errno = 2, error message = 'No such file or directory', flags = 0, o_flags = 0)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-47-5aa1dc8f3e88>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'weights_coco.h5'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/dl/lib/python3.6/site-packages/keras/engine/topology.py\u001b[0m in \u001b[0;36mload_weights\u001b[0;34m(self, filepath, by_name)\u001b[0m\n\u001b[1;32m   2564\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mh5py\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2565\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mImportError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'`load_weights` requires h5py.'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2566\u001b[0;31m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh5py\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'r'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2567\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;34m'layer_names'\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mattrs\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m'model_weights'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2568\u001b[0m             \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'model_weights'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/dl/lib/python3.6/site-packages/h5py/_hl/files.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, name, mode, driver, libver, userblock_size, swmr, **kwds)\u001b[0m\n\u001b[1;32m    267\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mphil\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    268\u001b[0m                 \u001b[0mfapl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmake_fapl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdriver\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlibver\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 269\u001b[0;31m                 \u001b[0mfid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmake_fid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muserblock_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfapl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mswmr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mswmr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    270\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    271\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mswmr_support\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/dl/lib/python3.6/site-packages/h5py/_hl/files.py\u001b[0m in \u001b[0;36mmake_fid\u001b[0;34m(name, mode, userblock_size, fapl, fcpl, swmr)\u001b[0m\n\u001b[1;32m     97\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mswmr\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mswmr_support\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m             \u001b[0mflags\u001b[0m \u001b[0;34m|=\u001b[0m \u001b[0mh5f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mACC_SWMR_READ\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 99\u001b[0;31m         \u001b[0mfid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh5f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfapl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfapl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    100\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mmode\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'r+'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m         \u001b[0mfid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh5f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh5f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mACC_RDWR\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfapl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfapl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mh5py/_objects.pyx\u001b[0m in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mh5py/_objects.pyx\u001b[0m in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mh5py/h5f.pyx\u001b[0m in \u001b[0;36mh5py.h5f.open\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mOSError\u001b[0m: Unable to open file (unable to open file: name = 'weights_coco.h5', errno = 2, error message = 'No such file or directory', flags = 0, o_flags = 0)"
     ]
    }
   ],
   "source": [
    "model.load_weights('weights_coco.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "  91/5120 [..............................] - ETA: 296822s - loss: 4.8495"
     ]
    }
   ],
   "source": [
    "tb_counter  = len([log for log in os.listdir(os.path.expanduser('~/logs/')) if 'coco_' in log]) + 1\n",
    "tensorboard = callbacks.TensorBoard(log_dir=os.path.expanduser('~/logs/') + 'coco_' + '_' + str(tb_counter), \n",
    "                          histogram_freq=0, \n",
    "                          write_graph=True, \n",
    "                          write_images=False)\n",
    "\n",
    "optimizer = optimizers.Adam(lr=0.5e-4, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)\n",
    "#optimizer = SGD(lr=1e-4, decay=0.0005, momentum=0.9)\n",
    "#optimizer = RMSprop(lr=1e-4, rho=0.9, epsilon=1e-08, decay=0.0)\n",
    "\n",
    "model.compile(loss=custom_loss, optimizer=optimizer)\n",
    "\n",
    "model.fit_generator(generator        = train_batch.get_generator(), \n",
    "                    steps_per_epoch  = train_batch.get_dateset_size(), \n",
    "                    epochs           = 100, \n",
    "                    verbose          = 1,\n",
    "                    validation_data  = valid_batch.get_generator(),\n",
    "                    validation_steps = valid_batch.get_dateset_size(),\n",
    "                    callbacks        = [early_stop, checkpoint, tensorboard], \n",
    "                    max_queue_size   = 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%load_ext version_information\n",
    "%version_information keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}