content/simpsons-detector/Train Model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simpsons Detector\n",
    "## Creating dataset\n",
    "\n",
    "The following Python notebook refers the [simpsons detector post](http://zachmoshe.com).\n",
    "\n",
    "This notebook shows how to train the model.\n",
    "\n",
    "Some classes are from my `general` and `simpsons` modules. They are available in [my github repo](https://github.com/zachmoshe/datalab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib\n",
    "import seaborn as sns\n",
    "import keras\n",
    "import keras.applications\n",
    "import keras.preprocessing.image\n",
    "from keras.layers import *\n",
    "from keras.regularizers import *\n",
    "from keras.optimizers import *\n",
    "from keras.models import *\n",
    "import tensorflow as tf\n",
    "import collections\n",
    "import PIL\n",
    "import glob \n",
    "\n",
    "import scipy\n",
    "import sklearn\n",
    "import sklearn.metrics\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "import math\n",
    "\n",
    "import sys\n",
    "import logging \n",
    "logging.basicConfig(stream=sys.stdout, level=logging.INFO)\n",
    "logging.getLogger(\"py.warnings\").setLevel(logging.ERROR)\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "# for auto-reloading external modules\n",
    "# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython\n",
    "%load_ext autoreload\n",
    "%autoreload \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import general\n",
    "from general.plotting import gridplot\n",
    "general.dl.make_keras_picklable()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import simpsons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('1.0.0', '1.2.1', '1.12.1')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.__version__, keras.__version__, np.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/cpu:0']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.python.client import device_lib\n",
    "local_device_protos = device_lib.list_local_devices()\n",
    "[x.name for x in local_device_protos]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Some configs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "BASE_PATH = \"/Users/zach/proj/datalab-data/simpsons/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "config = dict(\n",
    "    training_img_shape=(202,360),     # 1/4 of HD image\n",
    "    network_input_shape=(202,360),    \n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import gzip\n",
    "import pickle\n",
    "\n",
    "with gzip.GzipFile(BASE_PATH+\"dataset.pickle.zip\", \"r\") as f:\n",
    "    data = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train images by char:  [ 109.  118.   76.   49.]\n",
      "train_dev images by char:  [ 36.  40.  25.  17.]\n",
      "dev  images by char:  [  74.  102.   65.   63.]\n",
      "test images by char:  [  73.  101.   65.   66.]\n",
      "train\n",
      "  - X: (663, 300, 300, 3)\n",
      "  - y: (663, 4)\n",
      "train_dev\n",
      "  - X: (222, 300, 300, 3)\n",
      "  - y: (222, 4)\n",
      "dev\n",
      "  - X: (351, 202, 360, 3)\n",
      "  - y: (351, 4)\n",
      "test\n",
      "  - X: (351, 202, 360, 3)\n",
      "  - y: (351, 4)\n"
     ]
    }
   ],
   "source": [
    "print(\"train images by char: \", data.train.y.sum(axis=0))\n",
    "print(\"train_dev images by char: \", data.train_dev.y.sum(axis=0))\n",
    "print(\"dev  images by char: \", data.dev.y.sum(axis=0))\n",
    "print(\"test images by char: \", data.test.y.sum(axis=0))\n",
    "data.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build network\n",
    "\n",
    "`create_model` builds and compiles the actual neural network model based on many hyper-parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_model(opt='adam', fc_detectors=[], train_all=False, batchnorm=True, dropout=0.5, base_output_layer='block5_pool', \n",
    "                 num_detectors=16, num_outputs=4, reg_penalty=0.0):\n",
    "    if type(base_output_layer) is int:\n",
    "        base_output_layer = \"block{}_pool\".format(base_output_layer)\n",
    "\n",
    "    unknown_img_size_inp = Input(shape=(None,None,3), name='input')\n",
    "    vgg = keras.applications.vgg16.VGG16(include_top=False)\n",
    "    base_model = Model(input=vgg.input, output=vgg.get_layer(base_output_layer).output)\n",
    "    \n",
    "    base_model_output = x = base_model(unknown_img_size_inp)\n",
    "    \n",
    "    if batchnorm:\n",
    "        x = BatchNormalization(name='bn')(x)\n",
    "    x = Dropout(dropout, name=\"dropout\")(x)\n",
    "\n",
    "    for ind, fc_det in enumerate(fc_detectors):\n",
    "        x = Convolution2D(fc_det, 1, 1, activation='relu', name=\"fc_detector_{}\".format(ind), W_regularizer=keras.regularizers.l2(reg_penalty))(x)\n",
    "    \n",
    "    x = Convolution2D(num_detectors,1,1, activation='relu', name='detectors_spatial', W_regularizer=keras.regularizers.l2(reg_penalty))(x)\n",
    "    x = GlobalMaxPooling2D(name='detectors')(x)\n",
    "    \n",
    "    x = Dense(num_outputs, name='output', activation='sigmoid')(x)\n",
    "       \n",
    "    model = Model(input=unknown_img_size_inp, output=x) \n",
    "    \n",
    "    # Compile model    \n",
    "    if not train_all:\n",
    "        for l in base_model.layers:\n",
    "            l.trainable = False\n",
    "    \n",
    "    model.compile(optimizer=opt, loss='mse', metrics=['mae', 'accuracy'])\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train\n",
    "\n",
    "chars_chooser is a generic class that allows me to transform also the `y` vector. I've used it before, when trying the model only on a singel character. Now it's practically disabled.\n",
    "\n",
    "`BatchTrainingsetGenerator` is the generator that reads training set files (that we generated before) and feeds training images to the network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# choose characters\n",
    "chars_chooser = simpsons.preprocessing.CharactersChooser([0,1,2,3])  # bart,homer,lisa,marge\n",
    "train_X, train_y         = chars_chooser.transform(data.train.X, data.train.y)\n",
    "train_dev_X, train_dev_y = chars_chooser.transform(data.train_dev.X, data.train_dev.y)\n",
    "dev_X, dev_y             = chars_chooser.transform(data.dev.X, data.dev.y)\n",
    "\n",
    "# Preprocess pipeline\n",
    "simpsons_preprocess = Pipeline([\n",
    "        ('vgg_preprocessor', simpsons.preprocessing.VGGPreprocessing()),\n",
    "        ('resize', general.utils.ImageResizeTransformer(config['network_input_shape'])),\n",
    "    ])\n",
    "\n",
    "class BatchTrainingsetGenerator:\n",
    "    def __init__(self, folder):\n",
    "        self.folder = folder\n",
    "        \n",
    "    def generate(self, X, y, batch_size=None):\n",
    "        while True:\n",
    "            for f in np.random.permutation(glob.glob(self.folder+\"/*\")):\n",
    "                X, y = pickle.load(open(f, \"rb\"))\n",
    "                bs = batch_size if batch_size is not None else X.shape[0]\n",
    "                for i in range(0, X.shape[0], bs):\n",
    "                    yield X[i:i+bs], y[i:i+bs]\n",
    "            \n",
    "frame_generator = BatchTrainingsetGenerator(BASE_PATH+\"trainingset\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build params\n",
    "\n",
    "After some tuning, I've disabled the batchnorm layer and used 128 detectors without any other fully-connected layers before."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Run model only and visualize learning curve (loss/epoch)\n",
    "build_params = [\n",
    "    {\n",
    "        'num_outputs': [4],\n",
    "        'batchnorm': [False],\n",
    "        'num_detectors': [128],\n",
    "        'fc_detectors': [[]],\n",
    "        'reg_penalty': [ 0.001 ],\n",
    "    }\n",
    "]\n",
    "generator_params = [\n",
    "    {\n",
    "        'batch_size': [128],\n",
    "    }\n",
    "]\n",
    "\n",
    "gch = general.dl.DLGridSearch(create_model, build_params=build_params,\n",
    "                generator_fn=frame_generator.generate, generator_params=generator_params)\n",
    "\n",
    "gch.fit(data.train.X, data.train.y, nb_epoch=2500, samples_per_epoch=1024, \n",
    "       callbacks=[\n",
    "           keras.callbacks.ModelCheckpoint(BASE_PATH+\"model_checkpoints/model.{epoch:d}.hdf5\", monitor='val_loss', \n",
    "                                          verbose=0, save_best_only=False, save_weights_only=False, \n",
    "                                          mode='auto', period=100)\n",
    "       ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "general.plotting.KerasHistoryPlotter().plot(gch.results, ylim=(0.,0.2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "simpsons.scoring.SimpsonsReporter(\n",
    "        train_dev_X, train_dev_y, dev_X, dev_y, \n",
    "        dev_preprocess=simpsons_preprocess, generator_fn=frame_generator.generate,\n",
    "        train_dev_num_images=500\n",
    ").report(gch.results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dumping the model and GridSearch results to disk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "id = 0\n",
    "\n",
    "model = gch.results[id].history.model\n",
    "params = gch.results[id].params\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving model for  {'batchnorm': False, 'fc_detectors': [], 'reg_penalty': 0.001, 'num_outputs': 4, 'num_detectors': 128, 'batch_size': 128}\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import gzip\n",
    "\n",
    "# with gzip.GzipFile(\"/data/simpsons/gch.pickle.gz\", \"rb\") as f:\n",
    "#     gch = pickle.load(f)\n",
    "    \n",
    "with gzip.GzipFile(\"/data/simpsons/gch.pickle.gz\", \"wb\") as f:\n",
    "    pickle.dump(gch, f)\n",
    "\n",
    "print(\"saving model for \", params)\n",
    "model.save(\"/data/simpsons/model.hdf5\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load a pre-trained model (if needed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#model = keras.models.load_model(BASE_PATH + \"model_checkpoints/model.999.hdf5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Optimize classifier's thresholds \n",
    "\n",
    "runs the selected model on the dev set and sets thresholds according to a specific precision, specific recall, maximize accuracy or maximize fscore."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dev_X = data.dev.X\n",
    "dev_y = data.dev.y\n",
    "dev_preds = model.predict(simpsons_preprocess.transform(dev_X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cls = general.utils.ClassifierThresholdOptimizer(model, None, dev_y, preds=dev_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "cls.fit(maximize_fscore=True)\n",
    "preds_fscore = cls.predict(None, preds=dev_preds)\n",
    "\n",
    "cls.fit(maximize_accuracy=True)\n",
    "preds_acc = cls.predict(None, preds=dev_preds)\n",
    "\n",
    "cls.fit(recall=0.75)\n",
    "preds_rec_075 = cls.predict(None, preds=dev_preds)\n",
    "\n",
    "cls.fit(precision=0.75)\n",
    "preds_pre_075 = cls.predict(None, preds=dev_preds)\n",
    "\n",
    "num_imgs = 25\n",
    "general.plotting.gridplot_sidebyside(\n",
    "    [dev_X[:num_imgs], preds_pre_075[:num_imgs]], \n",
    "    [dev_X[:num_imgs], preds_rec_075[:num_imgs]], \n",
    "    [dev_X[:num_imgs], preds_fscore[:num_imgs]], \n",
    "    [dev_X[:num_imgs], preds_acc[:num_imgs]]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# diff between fscore and acc\n",
    "diff_ind = (preds_fscore != preds_acc).any(axis=1)\n",
    "general.plotting.gridplot_sidebyside(\n",
    "    [dev_X[diff_ind], preds_fscore[diff_ind]],\n",
    "    [dev_X[diff_ind], preds_acc[diff_ind]],\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "FScore was chosen as the best one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cls.fit(maximize_fscore=True)\n",
    "cls.thresholds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performance on test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_preds = model.predict(simpsons_preprocess.transform(data.test.X))\n",
    "test_preds = (test_preds > cls.thresholds).astype(int)\n",
    "test_true = data.test.y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Char #0 - bart\n",
      "ACCURACY - 0.863\n",
      "[[263  15]\n",
      " [ 33  40]]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.89      0.95      0.92       278\n",
      "        1.0       0.73      0.55      0.62        73\n",
      "\n",
      "avg / total       0.85      0.86      0.86       351\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Char #1 - homer\n",
      "ACCURACY - 0.744\n",
      "[[188  62]\n",
      " [ 28  73]]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.87      0.75      0.81       250\n",
      "        1.0       0.54      0.72      0.62       101\n",
      "\n",
      "avg / total       0.78      0.74      0.75       351\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Char #2 - lisa\n",
      "ACCURACY - 0.889\n",
      "[[272  14]\n",
      " [ 25  40]]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.92      0.95      0.93       286\n",
      "        1.0       0.74      0.62      0.67        65\n",
      "\n",
      "avg / total       0.88      0.89      0.88       351\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Char #3 - marge\n",
      "ACCURACY - 0.917\n",
      "[[277   8]\n",
      " [ 21  45]]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.93      0.97      0.95       285\n",
      "        1.0       0.85      0.68      0.76        66\n",
      "\n",
      "avg / total       0.91      0.92      0.91       351\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "simpsons.scoring.final_report(test_true, test_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Store final model and thresholds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model_and_thresholds = { \"model\": model, \"preprocess_pipeline\": simpsons_preprocess, \"thresholds\": cls.thresholds }\n",
    "pickle.dump(model_and_thresholds, open(BASE_PATH+\"model_and_thresholds.pickle\", \"wb\"))"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}