StreakImageRetraining.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Streak Interpretations for Image Prediction (Transfer Learning)\n",
    "\n",
    "(Adapted from LIME Keras tutorial\n",
    "https://github.com/marcotcr/lime/blob/master/doc/notebooks/Tutorial%20-%20Image%20Classification%20Keras.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import keras\n",
    "from keras.applications import inception_v3 as inc_net\n",
    "from keras.preprocessing import image\n",
    "from keras.applications.imagenet_utils import decode_predictions\n",
    "from skimage.io import imread\n",
    "from lime.explanation import id_generator\n",
    "import numpy as np\n",
    "from time import time\n",
    "from skimage import io\n",
    "import load_networks\n",
    "import lime\n",
    "import lime_image_streak\n",
    "from skimage.segmentation import mark_boundaries\n",
    "from tf_predict import *\n",
    "# from tensorflow.examples import label_image\n",
    "# import label_image\n",
    "print('using keras:', keras.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inception Transfer Learning\n",
    "\n",
    "Retrain the last layer of the InceptionV3 pretrained model, and interpret the predictions of new, preprocessed images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#keras preprocessing function\n",
    "def transform_img_fn(path_list):\n",
    "    out = []\n",
    "    for img_path in path_list:\n",
    "        img = image.load_img(img_path, target_size=(299, 299))\n",
    "        x = image.img_to_array(img)\n",
    "        x = np.expand_dims(x, axis=0)\n",
    "        x = inc_net.preprocess_input(x)\n",
    "        out.append(x)\n",
    "    return np.vstack(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#tensorflow preprocessing functions\n",
    "def import_tf_imgs(path_list):\n",
    "    out = []\n",
    "    for img_path in path_list:\n",
    "        # tmp  = label_image.read_tensor_from_image_file(img_path, input_height=299, input_width=299)\n",
    "        tmp = io.imread(img_path)\n",
    "        out.append(tmp)\n",
    "    # return np.vstack(out)\n",
    "    return out\n",
    "\n",
    "def import_tf_img(img_path):\n",
    "    # return label_image.read_tensor_from_image_file(img_path, input_height=299, input_width=299)\n",
    "    return np.expand_dims(io.imread(img_path),axis=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#setup transfer learning code\n",
    "\n",
    "from keras import applications\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras import optimizers\n",
    "from keras.models import Sequential, Model \n",
    "from keras.layers import Activation, Dropout, Flatten, Dense, GlobalAveragePooling2D, Conv2D, MaxPooling2D\n",
    "from keras import backend as k \n",
    "from keras.callbacks import ModelCheckpoint, LearningRateScheduler, TensorBoard, EarlyStopping\n",
    "from keras.utils.np_utils import to_categorical\n",
    "# from tf.examples.image_retraining import retrain\n",
    "import retrain\n",
    "from shutil import copy2\n",
    "\n",
    "# #preprocessing, separate full dataset into training, validation, and testing\n",
    "# data_dir = '~/flower_photos'\n",
    "# dest_dir = '~/flower_photos_retrain'\n",
    "# image_lists = retrain.create_image_lists(data_dir,testing_percentage=10,validation_percentage=10)\n",
    "# print sum([len(image_lists[label]['training']) for label in image_lists.keys()])\n",
    "# print sum([len(image_lists[label]['validation']) for label in image_lists.keys()])\n",
    "# print image_lists['tulips'].keys()\n",
    "# for label in image_lists.keys():\n",
    "#     for set_name in ['training','validation','testing']:\n",
    "#         for file_name in image_lists[label][set_name]:\n",
    "#             copy2(os.path.join(data_dir,label,file_name),os.path.join(dest_dir,set_name,label,file_name))\n",
    "\n",
    "img_width, img_height = 299, 299\n",
    "train_data_dir = \"~/flower_photos_retrain/training\"\n",
    "validation_data_dir = \"~/flower_photos_retrain/validation\"\n",
    "nb_train_samples = 3056\n",
    "nb_validation_samples = 451\n",
    "batch_size = 16\n",
    "epochs = 50\n",
    "bottleneck_size = 2048\n",
    "model = inc_net.InceptionV3(weights = \"imagenet\", include_top=False, input_shape = (img_width, img_height, 3), pooling=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#save 2nd to last layer of inceptionv3 as numpy features\n",
    "#then train a fully connected layer and save\n",
    "#then load this fc layer and append to inceptionV3\n",
    "\n",
    "datagen = ImageDataGenerator(rescale=1. / 255)\n",
    "\n",
    "generator = datagen.flow_from_directory(\n",
    "        train_data_dir,\n",
    "        target_size=(img_width, img_height),\n",
    "        batch_size=1,\n",
    "        class_mode=None,  # this means our generator will only yield batches of data, no labels\n",
    "        shuffle=False)  # our data will be in order, so all first 1000 images will be cats, then 1000 dogs\n",
    "bottleneck_features_train = model.predict_generator(generator, nb_train_samples, verbose=1)\n",
    "# save the output as a Numpy array\n",
    "np.save(open('bottleneck_features_train.npy', 'w'), bottleneck_features_train)\n",
    "\n",
    "generator2 = datagen.flow_from_directory(\n",
    "        validation_data_dir,\n",
    "        target_size=(img_width, img_height),\n",
    "        batch_size=1,\n",
    "        class_mode=None,\n",
    "        shuffle=False)\n",
    "bottleneck_features_validation = model.predict_generator(generator2, nb_validation_samples, verbose=1)\n",
    "# save the output as a Numpy array\n",
    "np.save(open('bottleneck_features_validation.npy', 'w'), bottleneck_features_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#retrain\n",
    "from keras.utils.np_utils import to_categorical\n",
    "train_data = np.load(open('bottleneck_features_train.npy'))\n",
    "validation_data = np.load(open('bottleneck_features_validation.npy'))\n",
    "classes = ['daisy','dandelion','roses','sunflowers','tulips']\n",
    "train_sizes = [len([name for name in os.listdir(os.path.join(train_data_dir,c)) \n",
    "                        ]) for c in classes]\n",
    "validation_sizes = [len([name for name in os.listdir(os.path.join(validation_data_dir,c)) \n",
    "                        ]) for c in classes]\n",
    "# print train_data.shape,validation_data.shape\n",
    "train_labels = []\n",
    "validation_labels = []\n",
    "for l,label in enumerate(classes):\n",
    "    train_labels.extend([l]*train_sizes[l])\n",
    "    validation_labels.extend([l]*validation_sizes[l])\n",
    "train_labels = to_categorical(train_labels)\n",
    "validation_labels = to_categorical(validation_labels)\n",
    "    \n",
    "#retrain the top layer\n",
    "top_model = Sequential()\n",
    "top_model.add(Flatten(input_shape=model.output_shape[1:]))\n",
    "top_model.add(Dense(len(classes), activation='sigmoid'))\n",
    "top_model.compile(optimizer=optimizers.SGD(lr=1e-4, momentum=0.9),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "top_model.fit(train_data, train_labels,\n",
    "          epochs=50,\n",
    "          batch_size=batch_size,\n",
    "          validation_data=(validation_data, validation_labels))\n",
    "\n",
    "#save the weights of the top layer, 2.6MB\n",
    "top_model.save_weights('retrained/bottleneck_fc_model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Look at predictions for a few images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "images_list = [os.path.join('daisy','705422469_ffa28c566d.jpg'),\n",
    "        'flowers_etsy.jpg', #contains both sunflowers and daisies, good for 2 top_labels\n",
    "        os.path.join('daisy','3445110406_0c1616d2e3_n.jpg'),\n",
    "        os.path.join('daisy','176375506_201859bb92_m.jpg'),\n",
    "        os.path.join('sunflowers','2979297519_17a08b37f6_m.jpg'), \n",
    "        os.path.join('daisy','301964511_fab84ea1c1.jpg'),\n",
    "        os.path.join('sunflowers','19504937128_a4ae90fcbd_m.jpg')\n",
    "        ]\n",
    "classes = ['daisy','dandelion','roses','sunflowers','tulips']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#import keras transfer learning model\n",
    "inet_model = load_networks.load_keras_inception_transfer()\n",
    "images = transform_img_fn(images_list)\n",
    "\n",
    "#print predictions\n",
    "preds = inet_model.predict(images)\n",
    "# print preds\n",
    "for ii,pr in enumerate(preds):\n",
    "    print images_list[ii]\n",
    "    sortedClasses = np.argsort(preds[ii])[-5:][::-1] #the indices that lime uses in explanation\n",
    "    for s in sortedClasses:\n",
    "        # print (classes[s],preds[ii][s])\n",
    "        print '(%s, %.5f)' % (classes[s],preds[ii][s])\n",
    "        \n",
    "predict = inet_model.predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# #keras (original) inception model\n",
    "# inet_model = load_networks.load_keras_inception_imagenet()\n",
    "# images = transform_img_fn(images_list)\n",
    "\n",
    "# preds = inet_model.predict(images)\n",
    "# for ii,pr in enumerate(preds):\n",
    "#     print images_list[ii]\n",
    "#     for x in decode_predictions(preds)[ii]:\n",
    "#         print '(%s, %.5f)' %  (x[1],x[2])\n",
    "# predict = inet_model.predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# #tensorflow transfer learning\n",
    "# g = load_networks.load_tf_transfer()\n",
    "# images = import_tf_imgs(images_list)\n",
    "\n",
    "# preds = tf_predict(images,0,5,g)\n",
    "# print preds\n",
    "# for ii,pr in enumerate(preds):\n",
    "#     print images_list[ii]\n",
    "#     sortedClasses = np.argsort(preds[ii])[-5:][::-1] #the indices that lime uses in explanation\n",
    "#     for s in sortedClasses:\n",
    "#         # print (classes[s],preds[ii][s])\n",
    "#         print '(%s, %.5f)' % (classes[s],preds[ii][s])\n",
    "\n",
    "# predict = lambda x: tf_predict(x,0,5,g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#explanation plotting function\n",
    "def explanation_plotting(explanation,label_idx,fs_string):\n",
    "    label = explanation.top_labels[label_idx]\n",
    "    temp, mask = explanation.get_image_and_mask(label, positive_only=True, num_features=5, hide_rest=True)\n",
    "    plt.imshow(mark_boundaries(temp / 2 + 0.5, mask))\n",
    "    print \"%s, %s\" % (classes[label],fs_string)\n",
    "    plt.show()\n",
    "\n",
    "    #then plot image with positive segments marked in green and negative segments marked in red\n",
    "    temp, mask = explanation.get_image_and_mask(label, positive_only=False, num_features=5, hide_rest=False)\n",
    "    plt.imshow(mark_boundaries(temp / 2 + 0.5, mask))\n",
    "    plt.show()\n",
    "\n",
    "    #plot original image, explantation agains black background with segments\n",
    "    fig = plt.figure()\n",
    "    ax1 = fig.add_subplot(121)\n",
    "    ax1.imshow(explanation.image/2 + 0.5)\n",
    "    ax1.set_xticklabels(\"\")\n",
    "    ax1.set_yticklabels(\"\")\n",
    "    ax2 = fig.add_subplot(122)\n",
    "    finalMask = explanation.segments\n",
    "    finalTemp = lime_image_streak.SegmentedImage.add_image_segments(-1*np.ones_like(explanation.image),\n",
    "                                explanation.image,explanation.segments,[x[0] for x in explanation.local_exp[label]][:5])\n",
    "    ax2.imshow(mark_boundaries(finalTemp/2 + 0.5,finalMask))\n",
    "    ax2.set_xticklabels(\"\")\n",
    "    ax2.set_yticklabels(\"\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Get an explanation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#new explainer class that can support streak feature_selection\n",
    "query_image = images[0]\n",
    "#Method described in Section 6.2 of the paper\n",
    "explainer = lime_image_streak.LimeImageExplainer(feature_selection='greedy_likelihood')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "explanation = explainer.explain_instance(query_image, classifier_fn=predict, \n",
    "#                                          top_labels=2, num_features=5, \n",
    "                                         top_labels=1, num_features=5, \n",
    "                                         qs_kernel_size=6, hide_color=0, \n",
    "                                         num_samples=1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Explanations for the top class**\n",
    "\n",
    "(for explanation of the 2nd top class, change top_labels to 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "explanation_plotting(explanation,0,'greedy_likelihood')\n",
    "# explanation_plotting(explanation,1,'greedy_likelihood')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#now try another selection method\n",
    "#LIME baseline method\n",
    "explainerFS = lime_image_streak.LimeImageExplainer(feature_selection='forward_selection')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "explanationFS = explainerFS.explain_instance(query_image, classifier_fn=predict, \n",
    "                                             top_labels=2, num_features=5, \n",
    "                                             qs_kernel_size=6, hide_color=0, \n",
    "                                             num_samples=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "explanation_plotting(explanationFS,0,'forward_selection')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "explanation_plotting(explanationFS,1,'forward_selection')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#now try another selection method\n",
    "#Method described in Section A.8 of the paper\n",
    "explainerSG = lime_image_streak.LimeImageExplainer(feature_selection='streaming_greedy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "explanationSG = explainerSG.explain_instance(query_image, classifier_fn=predict, \n",
    "                                             top_labels=2, num_features=5, \n",
    "                                             qs_kernel_size=6, hide_color=0, \n",
    "                                             num_samples=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "explanation_plotting(explanationSG,0,'streaming_greedy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "explanation_plotting(explanationSG,1,'streaming_greedy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}