SSD_training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import keras\n",
    "from keras.applications.imagenet_utils import preprocess_input\n",
    "from keras.backend.tensorflow_backend import set_session\n",
    "from keras.models import Model\n",
    "from keras.preprocessing import image\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pickle\n",
    "import os\n",
    "from random import shuffle\n",
    "#from scipy.misc import imread\n",
    "from imageio import imread\n",
    "from scipy.misc import imresize\n",
    "from skimage.transform import resize\n",
    "import tensorflow as tf\n",
    "\n",
    "from ssd import SSD300\n",
    "from ssd_training import MultiboxLoss\n",
    "from ssd_utils import BBoxUtility\n",
    "from keras.utils import multi_gpu_model\n",
    "\n",
    "%matplotlib inline\n",
    "plt.rcParams['figure.figsize'] = (8, 8)\n",
    "plt.rcParams['image.interpolation'] = 'nearest'\n",
    "\n",
    "np.set_printoptions(suppress=True)\n",
    "\n",
    "# config = tf.ConfigProto()\n",
    "# config.gpu_options.per_process_gpu_memory_fraction = 0.9\n",
    "# set_session(tf.Session(config=config))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0,1\"\n",
    "\n",
    "# some constants\n",
    "NUM_CLASSES = 2 #21 #4\n",
    "input_shape = (300, 300, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "priors = pickle.load(open('prior_boxes_ssd300.pkl', 'rb'))\n",
    "bbox_util = BBoxUtility(NUM_CLASSES, priors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gt = pickle.load(open('manhole/manhole.pkl', 'rb'))\n",
    "keys = sorted(gt.keys())\n",
    "num_train = int(round(0.8 * len(keys)))\n",
    "train_keys = keys[:num_train]\n",
    "val_keys = keys[num_train:]\n",
    "num_val = len(val_keys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Generator(object):\n",
    "    def __init__(self, gt, bbox_util,\n",
    "                 batch_size, path_prefix,\n",
    "                 train_keys, val_keys, image_size,\n",
    "                 saturation_var=0.5,\n",
    "                 brightness_var=0.5,\n",
    "                 contrast_var=0.5,\n",
    "                 lighting_std=0.5,\n",
    "                 hflip_prob=0.5,\n",
    "                 vflip_prob=0.5,\n",
    "                 do_crop=True,\n",
    "                 crop_area_range=[0.75, 1.0],\n",
    "                 aspect_ratio_range=[3./4., 4./3.]):\n",
    "        self.gt = gt\n",
    "        self.bbox_util = bbox_util\n",
    "        self.batch_size = batch_size\n",
    "        self.path_prefix = path_prefix\n",
    "        self.train_keys = train_keys\n",
    "        self.val_keys = val_keys\n",
    "        self.train_batches = len(train_keys)\n",
    "        self.val_batches = len(val_keys)\n",
    "        self.image_size = image_size\n",
    "        self.color_jitter = []\n",
    "        if saturation_var:\n",
    "            self.saturation_var = saturation_var\n",
    "            self.color_jitter.append(self.saturation)\n",
    "        if brightness_var:\n",
    "            self.brightness_var = brightness_var\n",
    "            self.color_jitter.append(self.brightness)\n",
    "        if contrast_var:\n",
    "            self.contrast_var = contrast_var\n",
    "            self.color_jitter.append(self.contrast)\n",
    "        self.lighting_std = lighting_std\n",
    "        self.hflip_prob = hflip_prob\n",
    "        self.vflip_prob = vflip_prob\n",
    "        self.do_crop = do_crop\n",
    "        self.crop_area_range = crop_area_range\n",
    "        self.aspect_ratio_range = aspect_ratio_range\n",
    "        \n",
    "    def grayscale(self, rgb):\n",
    "        return rgb.dot([0.299, 0.587, 0.114])\n",
    "\n",
    "    def saturation(self, rgb):\n",
    "        gs = self.grayscale(rgb)\n",
    "        alpha = 2 * np.random.random() * self.saturation_var \n",
    "        alpha += 1 - self.saturation_var\n",
    "        rgb = rgb * alpha + (1 - alpha) * gs[:, :, None]\n",
    "        return np.clip(rgb, 0, 255)\n",
    "\n",
    "    def brightness(self, rgb):\n",
    "        alpha = 2 * np.random.random() * self.brightness_var \n",
    "        alpha += 1 - self.saturation_var\n",
    "        rgb = rgb * alpha\n",
    "        return np.clip(rgb, 0, 255)\n",
    "\n",
    "    def contrast(self, rgb):\n",
    "        gs = self.grayscale(rgb).mean() * np.ones_like(rgb)\n",
    "        alpha = 2 * np.random.random() * self.contrast_var \n",
    "        alpha += 1 - self.contrast_var\n",
    "        rgb = rgb * alpha + (1 - alpha) * gs\n",
    "        return np.clip(rgb, 0, 255)\n",
    "\n",
    "    def lighting(self, img):\n",
    "        cov = np.cov(img.reshape(-1, 3) / 255.0, rowvar=False)\n",
    "        eigval, eigvec = np.linalg.eigh(cov)\n",
    "        noise = np.random.randn(3) * self.lighting_std\n",
    "        noise = eigvec.dot(eigval * noise) * 255\n",
    "        img += noise\n",
    "        return np.clip(img, 0, 255)\n",
    "    \n",
    "    def horizontal_flip(self, img, y):\n",
    "        if np.random.random() < self.hflip_prob:\n",
    "            img = img[:, ::-1]\n",
    "            y[:, [0, 2]] = 1 - y[:, [2, 0]]\n",
    "        return img, y\n",
    "    \n",
    "    def vertical_flip(self, img, y):\n",
    "        if np.random.random() < self.vflip_prob:\n",
    "            img = img[::-1]\n",
    "            y[:, [1, 3]] = 1 - y[:, [3, 1]]\n",
    "        return img, y\n",
    "    \n",
    "    def random_sized_crop(self, img, targets):\n",
    "        img_w = img.shape[1]\n",
    "        img_h = img.shape[0]\n",
    "        img_area = img_w * img_h\n",
    "        random_scale = np.random.random()\n",
    "        random_scale *= (self.crop_area_range[1] -\n",
    "                         self.crop_area_range[0])\n",
    "        random_scale += self.crop_area_range[0]\n",
    "        target_area = random_scale * img_area\n",
    "        random_ratio = np.random.random()\n",
    "        random_ratio *= (self.aspect_ratio_range[1] -\n",
    "                         self.aspect_ratio_range[0])\n",
    "        random_ratio += self.aspect_ratio_range[0]\n",
    "        w = np.round(np.sqrt(target_area * random_ratio))     \n",
    "        h = np.round(np.sqrt(target_area / random_ratio))\n",
    "        if np.random.random() < 0.5:\n",
    "            w, h = h, w\n",
    "        w = min(w, img_w)\n",
    "        w_rel = w / img_w\n",
    "        w = int(w)\n",
    "        h = min(h, img_h)\n",
    "        h_rel = h / img_h\n",
    "        h = int(h)\n",
    "        x = np.random.random() * (img_w - w)\n",
    "        x_rel = x / img_w\n",
    "        x = int(x)\n",
    "        y = np.random.random() * (img_h - h)\n",
    "        y_rel = y / img_h\n",
    "        y = int(y)\n",
    "        img = img[y:y+h, x:x+w]\n",
    "        new_targets = []\n",
    "        for box in targets:\n",
    "            cx = 0.5 * (box[0] + box[2])\n",
    "            cy = 0.5 * (box[1] + box[3])\n",
    "            if (x_rel < cx < x_rel + w_rel and\n",
    "                y_rel < cy < y_rel + h_rel):\n",
    "                xmin = (box[0] - x_rel) / w_rel\n",
    "                ymin = (box[1] - y_rel) / h_rel\n",
    "                xmax = (box[2] - x_rel) / w_rel\n",
    "                ymax = (box[3] - y_rel) / h_rel\n",
    "                xmin = max(0, xmin)\n",
    "                ymin = max(0, ymin)\n",
    "                xmax = min(1, xmax)\n",
    "                ymax = min(1, ymax)\n",
    "                box[:4] = [xmin, ymin, xmax, ymax]\n",
    "                new_targets.append(box)\n",
    "        new_targets = np.asarray(new_targets).reshape(-1, targets.shape[1])\n",
    "        return img, new_targets\n",
    "    \n",
    "    def generate(self, train=True):\n",
    "        while True:\n",
    "            if train:\n",
    "                shuffle(self.train_keys)\n",
    "                keys = self.train_keys\n",
    "            else:\n",
    "                shuffle(self.val_keys)\n",
    "                keys = self.val_keys\n",
    "            inputs = []\n",
    "            targets = []\n",
    "            for key in keys:            \n",
    "                img_path = self.path_prefix + key\n",
    "                img = imread(img_path).astype('float32')\n",
    "                img = np.stack((img,)*3)\n",
    "                y = self.gt[key].copy()\n",
    "                if train and self.do_crop:\n",
    "                    img, y = self.random_sized_crop(img, y)\n",
    "                img = imresize(img, self.image_size).astype('float32')\n",
    "                \n",
    "                #img = resize(img, self.image_size, mode= 'reflect').astype('float32')\n",
    "                if train:\n",
    "                    shuffle(self.color_jitter)\n",
    "                    for jitter in self.color_jitter:\n",
    "                        img = jitter(img)\n",
    "                    if self.lighting_std:\n",
    "                        img = self.lighting(img)\n",
    "                    if self.hflip_prob > 0:\n",
    "                        img, y = self.horizontal_flip(img, y)\n",
    "                    if self.vflip_prob > 0:\n",
    "                        img, y = self.vertical_flip(img, y)\n",
    "                y = self.bbox_util.assign_boxes(y)\n",
    "                inputs.append(img)                \n",
    "                targets.append(y)\n",
    "                if len(targets) == self.batch_size:\n",
    "                    tmp_inp = np.array(inputs)\n",
    "                    tmp_targets = np.array(targets)\n",
    "                    inputs = []\n",
    "                    targets = []\n",
    "                    yield preprocess_input(tmp_inp), tmp_targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path_prefix = '../yolo2keras/manhole/PNGImages/'\n",
    "batch_size = 16\n",
    "gen = Generator(gt, bbox_util, batch_size, path_prefix,\n",
    "                train_keys, val_keys,\n",
    "                (input_shape[0], input_shape[1]), do_crop=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = SSD300(input_shape, num_classes=NUM_CLASSES)\n",
    "model.load_weights('weights_SSD300.hdf5', by_name=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "freeze = ['input_1', 'conv1_1', 'conv1_2', 'pool1',\n",
    "          'conv2_1', 'conv2_2', 'pool2',\n",
    "          'conv3_1', 'conv3_2', 'conv3_3', 'pool3']#,\n",
    "#           'conv4_1', 'conv4_2', 'conv4_3', 'pool4']\n",
    "\n",
    "for L in model.layers:\n",
    "    if L.name in freeze:\n",
    "        L.trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def schedule(epoch, decay=0.9):\n",
    "    return base_lr * decay**(epoch)\n",
    "\n",
    "callbacks = [keras.callbacks.ModelCheckpoint('./checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5',\n",
    "                                             verbose=1,\n",
    "                                             save_weights_only=True),\n",
    "             keras.callbacks.LearningRateScheduler(schedule)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_lr = 3e-4\n",
    "optim = keras.optimizers.Adam(lr=base_lr)\n",
    "# optim = keras.optimizers.RMSprop(lr=base_lr)\n",
    "# optim = keras.optimizers.SGD(lr=base_lr, momentum=0.9, decay=decay, nesterov=True)\n",
    "\n",
    "#model = multi_gpu_model(model, gpus= 2)\n",
    "\n",
    "model.compile(optimizer=optim,\n",
    "              loss=MultiboxLoss(NUM_CLASSES, neg_pos_ratio=2.0).compute_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py:96: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "22/51 [===========>..................] - ETA: 5:26 - loss: 5.2813"
     ]
    }
   ],
   "source": [
    "nb_epoch = 30 #number of epoch\n",
    "history = model.fit_generator(gen.generate(True), \n",
    "                              #gen.train_batches, #samples per epoch\n",
    "                              steps_per_epoch = gen.train_batches//batch_size,\n",
    "                              epochs = nb_epoch,\n",
    "                              verbose=1,\n",
    "                              callbacks=callbacks,\n",
    "                              validation_data=gen.generate(False),\n",
    "                              #nb_val_samples=gen.val_batches,\n",
    "                              validation_steps = gen.val_batches//batch_size,\n",
    "                              #nb_worker=1\n",
    "                              workers=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "inputs = []\n",
    "images = []\n",
    "sorted_val_keys=sorted(val_keys)\n",
    "for i in range(30) :\n",
    "    img_path = path_prefix + sorted_val_keys[i]\n",
    "    img = image.load_img(img_path, target_size=(300, 300))\n",
    "    img = image.img_to_array(img)\n",
    "    inputs.append(img.copy())\n",
    "    images.append(imread(img_path))\n",
    "inputs = preprocess_input(np.array(inputs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "images[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "preds = model.predict(inputs, batch_size=1, verbose=1)\n",
    "results = bbox_util.detection_out(preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i, img in enumerate(images):\n",
    "    # Parse the outputs.\n",
    "    det_label = results[i][:, 0]\n",
    "    det_conf = results[i][:, 1]\n",
    "    det_xmin = results[i][:, 2]\n",
    "    det_ymin = results[i][:, 3]\n",
    "    det_xmax = results[i][:, 4]\n",
    "    det_ymax = results[i][:, 5]\n",
    "\n",
    "    # Get detections with confidence higher than 0.6.\n",
    "    top_indices = [i for i, conf in enumerate(det_conf) if conf >= 0.6]\n",
    "\n",
    "    top_conf = det_conf[top_indices]\n",
    "    top_label_indices = det_label[top_indices].tolist()\n",
    "    top_xmin = det_xmin[top_indices]\n",
    "    top_ymin = det_ymin[top_indices]\n",
    "    top_xmax = det_xmax[top_indices]\n",
    "    top_ymax = det_ymax[top_indices]\n",
    "\n",
    "    colors = plt.cm.hsv(np.linspace(0, 1, 4)).tolist()\n",
    "\n",
    "    plt.imshow(img / 255.)\n",
    "    plt.gray()\n",
    "    \n",
    "\n",
    "    for i in range(top_conf.shape[0]):\n",
    "        xmin = int(round(top_xmin[i] * img.shape[1]))\n",
    "        ymin = int(round(top_ymin[i] * img.shape[0]))\n",
    "        xmax = int(round(top_xmax[i] * img.shape[1]))\n",
    "        ymax = int(round(top_ymax[i] * img.shape[0]))\n",
    "        score = top_conf[i]\n",
    "        label = int(top_label_indices[i])\n",
    "#         label_name = voc_classes[label - 1]\n",
    "        display_txt = '{:0.2f}, {}'.format(score, label)\n",
    "        coords = (xmin, ymin), xmax-xmin+1, ymax-ymin+1\n",
    "        color = colors[label]\n",
    "        currentAxis.add_patch(plt.Rectangle(*coords, fill=False, edgecolor=color, linewidth=2))\n",
    "        currentAxis.text(xmin, ymin, display_txt, bbox={'facecolor':color, 'alpha':0.5})\n",
    "    \n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "%run ./train.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}