examples/anchor_image_fashion_mnist.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Anchor explanations for fashion MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "tf.logging.set_verbosity(tf.logging.ERROR)  # suppress deprecation messages\n",
    "from tensorflow.keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D, Input\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from alibi.explainers import AnchorImage\n",
    "tf.logging.set_verbosity(tf.logging.ERROR)  # suppress deprecation messages"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load and prepare fashion MNIST data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (60000, 28, 28) y_train shape: (60000,)\n"
     ]
    }
   ],
   "source": [
    "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()\n",
    "print('x_train shape:', x_train.shape, 'y_train shape:', y_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAUDklEQVR4nO3da2yc1ZkH8P8z4/ElzjiJc3FCcAmXUJLCEqhJgFSUkkJDtNqQUioQYkFCG7QL3bbLBxDtquyXFUILCC277RrIElaFqlVBUBRRgrlkgZLGhJTcNgQSk5tjOzGxHcdjz+XZDx5aE3ye18w7M+/A+f8ky/Y8PjPHM/77nZnznnNEVUFEX36xqDtAROXBsBN5gmEn8gTDTuQJhp3IE1XlvLFqqdFa1JfzJom8ksIgRnRYxquFCruILAfwMIA4gMdU9T7r52tRjyWyLMxNEpFho7Y5awU/jReROID/AHA1gIUAbhCRhYVeHxGVVpjX7IsBfKCqe1R1BMCvAKwsTreIqNjChH0ugP1jvj+Qv+xTRGS1iLSLSHsawyFujojCCBP28d4E+My5t6raqqotqtqSQE2ImyOiMMKE/QCA5jHfnwrgULjuEFGphAn7JgDzReR0EakGcD2A54vTLSIqtoKH3lQ1IyJ3APg9Rofe1qjq9qL1jIiKKtQ4u6quA7CuSH0hohLi6bJEnmDYiTzBsBN5gmEn8gTDTuQJhp3IEww7kScYdiJPMOxEnmDYiTzBsBN5gmEn8gTDTuSJsi4lTRGQcVcV/ouQG3vGpzea9Y+/c7az1vDU26FuO+h3k6qEs6bpkXC3HVbQ42Ip8DHjkZ3IEww7kScYdiJPMOxEnmDYiTzBsBN5gmEn8gTH2b/kJB4365rJmPXYInuvzp23TbbbD7lricHFZtuqoZxZT7zUbtZDjaUHjeEH3K8Q+zgapm9SZcTWeDh5ZCfyBMNO5AmGncgTDDuRJxh2Ik8w7ESeYNiJPMFx9i85c0wWwePs+78z1azfeMn/mvU3e85w1j6qmW221TqzjKpvX2LWz/7Pg85apmOffeUBc8aD7rcg8WnT3MVs1myb7e93F41uhwq7iHQAGACQBZBR1ZYw10dEpVOMI/u3VPVIEa6HiEqIr9mJPBE27ArgJRF5R0RWj/cDIrJaRNpFpD2N4ZA3R0SFCvs0fqmqHhKRWQDWi8j/qeqGsT+gqq0AWgGgQRrDrW5IRAULdWRX1UP5z90AngVgT2MiosgUHHYRqReR5CdfA7gKwLZidYyIiivM0/gmAM/K6LzfKgBPqeqLRekVFU0ulQrVfuSC42b9e1PsOeW1sbSz9nrMnq9+8JVms579K7tvHz2YdNZy715qtp2+zR7rbni306wfuWyuWe/5uvsVbVPAcvrTXv7QWZNed6QLDruq7gFwfqHtiai8OPRG5AmGncgTDDuRJxh2Ik8w7ESeEA25Ze/n0SCNukSWle32vGEtexzw+B7//sVm/eqfvmbWF9QeMusDuVpnbUTDncD5yK5vmvXBPVOctdhIwJbJAeVsk70UtKbt4+i0ze7fvW5ll9lWHp3prL3X9jCO9+4ft/c8shN5gmEn8gTDTuQJhp3IEww7kScYdiJPMOxEnuA4eyUI2B44lIDH99x37P/3351mT2ENEjfWNh7UarPtsWx9qNvuybinuKYDxvgf221PgT1ujOEDQCxjP6ZXfutdZ+3axk1m2/vPPM9Z26ht6NdejrMT+YxhJ/IEw07kCYadyBMMO5EnGHYiTzDsRJ7gls2VoIznOpxs9/FZZv1ow2Szfjhjb+k8Pe5e7jkZGzLbzkvY+4X2ZN3j6AAQT7iXqh7RuNn2X772O7OeWpAw6wmxl6K+1FgH4Lodf2u2rcces+7CIzuRJxh2Ik8w7ESeYNiJPMGwE3mCYSfyBMNO5AmOs3tuZo297XGtuLdcBoBqyZj1Q+lpztruoa+abd/vt88BWN603aynjbF0a549EDxOfkriY7OeUnsc3rpXlzbZ4+hbzKpb4JFdRNaISLeIbBtzWaOIrBeR3fnP7keUiCrCRJ7GPwFg+UmX3Q2gTVXnA2jLf09EFSww7Kq6AUDvSRevBLA2//VaANcUuV9EVGSFvkHXpKqdAJD/7HxxJSKrRaRdRNrTGC7w5ogorJK/G6+qraraoqotCdSU+uaIyKHQsHeJyBwAyH/uLl6XiKgUCg378wBuzn99M4DnitMdIiqVwHF2EXkawOUAZojIAQA/A3AfgF+LyK0A9gG4rpSd/NILWDde4vbca824x7rj0+xR0W9O3WrWe7INZv1YdpJZnxo/4awNZNx7twNA75B93efUdJr1zSfmOWszq+1xcqvfANAxMsOsz685bNbv73Lvn9Bce/L74Z+WWXaZs6Yb/+CsBYZdVW9wlLjbA9EXCE+XJfIEw07kCYadyBMMO5EnGHYiT3CKayUIWEpaquyHyRp623/rArPtFZPsJZPfSs016zOrBsy6Nc10Tk2f2TbZlDLrQcN+jVXu6bsD2Tqz7aSYfWp30O99YbW9DPaPX77QWUuee9Rs25AwjtHGKC6P7ESeYNiJPMGwE3mCYSfyBMNO5AmGncgTDDuRJzjOXgEkUW3Wcyl7vNkyY+uIWT+StZc8nhqzp3pWByy5bG2NfGnjXrNtT8BY+Oah0816Mu7eEnpmzB4nb07YY91bU81mfd3gWWb91r9+2Vl7uvVKs231i285a6Lux4tHdiJPMOxEnmDYiTzBsBN5gmEn8gTDTuQJhp3IE1+scXZjyWWpsseLJR7wfy1m13MpY35zzh5rDqJpeyw8jIf/6xGzvj8z1awfTtv1oCWXs8YE67eHpphta2P2dtEzq/rNen/OHqe3DOTsZa6tefpAcN/vmr7bWXum79tm20LxyE7kCYadyBMMO5EnGHYiTzDsRJ5g2Ik8wbATeaKixtnDrI8eNFat9rBnpIZWLjbr+6+xx/FvvOCPztrhTNJs+66xrTEATDHmhANAfcD66il1n/9waMTeTjporNpaFx4AZhnj8Fm1j3MH03bfggSdf3AgY6xp/zf2XPupTxbUpeAju4isEZFuEdk25rJ7ReSgiGzJf6wo7OaJqFwm8jT+CQDLx7n8IVVdlP9YV9xuEVGxBYZdVTcA6C1DX4iohMK8QXeHiLyXf5rvfIEjIqtFpF1E2tOwX98RUekUGvafAzgTwCIAnQAecP2gqraqaouqtiRQU+DNEVFYBYVdVbtUNauqOQCPArDfTiaiyBUUdhGZM+bbVQC2uX6WiCpD4Di7iDwN4HIAM0TkAICfAbhcRBYBUAAdAG4rRmescfSwqubMNuvp05vMeu8C917gJ2Ybm2IDWLRip1m/pem/zXpPtsGsJ8TYnz093Wx7waQOs/5K30KzfqRqslm3xukvrXfP6QaAYzl7//VTqj4263d98D1nrWmSPZb92Gn2AFNac2Z9V9p+ydqXc8+H/8eFr5ptn8VMs+4SGHZVvWGcix8v6NaIKDI8XZbIEww7kScYdiJPMOxEnmDYiTxRUVNch6++yKzP+skeZ21RwwGz7cK6N8x6KmcvRW1Nt9wxNNdseyJnb8m8e8QeFuzL2ENQcXEPA3WP2FNcH9hrL1vctvgXZv2nh8abI/UXsTp11o5m7WG7ayfbS0UD9mN221c2OGtnVHebbV8YnGPWDwVMgW1K9Jn1eYkeZ+27yffNtoUOvfHITuQJhp3IEww7kScYdiJPMOxEnmDYiTzBsBN5orzj7GIvF73kXzeZzZcltztrJ9SeUhg0jh40bmqZUmUvGzyctu/m7rQ9hTXI2TWHnbVVDVvMthseWWLWv5H6gVn/8Ap7em7bkHsqZ0/G/r2v33uFWd+8r9msXzxvr7N2XvKg2Tbo3IZkPGXWrWnHADCYc/+9vp2yzz8oFI/sRJ5g2Ik8wbATeYJhJ/IEw07kCYadyBMMO5EnRNU937jY6mY365k3/ZOz3nr7v5vtn+q92FlrrrW3ozut+ohZnx63t/+1JGP2mOtXE/aY6wuDp5r1146dY9a/nuxw1hJib/d8+aQPzPotP77TrGdq7WW0++e5jyeZevtvr+H8o2b9B2e9Ytarjd/9WNYeRw+634K2ZA5irUGQjNnbZD+wYpWz9oeOJ9A31Dnug8IjO5EnGHYiTzDsRJ5g2Ik8wbATeYJhJ/IEw07kibLOZ4+lgUld7vHFF/oXme3PqHOvtX0kba+P/vvj55n1U+vs7X+trYfPMuaTA8CW1FSz/mLP18z6KXX2+uld6SnO2tF0vdn2hDGvGgAef+hBs/5Al73u/KrGzc7a+dX2OPqxnH0s2hGw3v5ArtZZS6m9vkFfwDh80vh7AIC02tGKG1s+T43ZY/j957m34c52uW838MguIs0i8qqI7BSR7SLyw/zljSKyXkR25z8XvvoDEZXcRJ7GZwDcqaoLAFwM4HYRWQjgbgBtqjofQFv+eyKqUIFhV9VOVd2c/3oAwE4AcwGsBLA2/2NrAVxTqk4SUXif6w06EZkH4AIAGwE0qWonMPoPAcAsR5vVItIuIu2Z4cFwvSWigk047CIyGcBvAfxIVYN23PszVW1V1RZVbamqsd8sIqLSmVDYRSSB0aD/UlWfyV/cJSJz8vU5AOxtMYkoUoFDbyIiAB4HsFNVx47DPA/gZgD35T8/F3Rd8ZEckvuHnfWc2tMlXzninurZVDtgtl2U3G/Wd52wh3G2Dp3irG2u+orZti7u3u4ZAKZU21Nk66vc9xkAzEi4f/fTa+z/wdY0UADYlLJ/t7+f+ZpZ35dxD9L8bvBss+2OE+77HACmBSzhvbXf3f5Ext5GezhrRyOVsYdyp9TYj+lFjR85a7tgbxfdc74xbfhNd7uJjLMvBXATgK0i8ski5PdgNOS/FpFbAewDcN0ErouIIhIYdlV9A4DrkLusuN0holLh6bJEnmDYiTzBsBN5gmEn8gTDTuSJ8m7ZfHwIsdffdZZ/89JSs/k/r/yNs/Z6wHLLLxy2x0X7R+ypnjMnuU/1bTDGuQGgMWGfJhy05XNtwPa/H2fcZyYOx+ypnFnnQMuow8Pu6bMA8GZuvllP59xbNg8bNSD4/ITekRlm/ZS6PmdtIOOe/goAHQONZv1In72tcmqSHa03smc6a8tnu7cmB4C6bvdjFjP+VHhkJ/IEw07kCYadyBMMO5EnGHYiTzDsRJ5g2Ik8UdYtmxukUZdI4RPl+m50b9l8xj/sMtsunrrXrG/ut+dt7zPGXdMBSx4nYu5lgwFgUmLErNcGjDdXx91z0mOwH99cwDh7fdzuW9Bc+4Yq97zuZNye8x0ztjWeiLjxu/+xb16o604G/N4Ztf8mLpnyobO2Zu+lZtspK9zbbG/UNvRrL7dsJvIZw07kCYadyBMMO5EnGHYiTzDsRJ5g2Ik8Uf5x9vhV7h/I2WuYhzF47RKzvuSeTXY96R4XPae6y2ybgD1eXBswnlwfs8fCU8ZjGPTf/I2hZrOeDbiGVz5eYNbTxnhz14kGs23COH9gIqx9CIYyAVs2D9nz3eMxOzep1+y59tN3uM+dqFln/y1aOM5ORAw7kS8YdiJPMOxEnmDYiTzBsBN5gmEn8kTgOLuINAN4EsBsADkArar6sIjcC+DvAPTkf/QeVV1nXVfY+eyVSi6y16Qfml1n1muO2nOjB06z2zd86F6XPjZsrzmf+9NOs05fLNY4+0Q2icgAuFNVN4tIEsA7IrI+X3tIVf+tWB0lotKZyP7snQA6818PiMhOAHNL3TEiKq7P9ZpdROYBuADAxvxFd4jIeyKyRkSmOdqsFpF2EWlPw366SkSlM+Gwi8hkAL8F8CNV7QfwcwBnAliE0SP/A+O1U9VWVW1R1ZYE7P3UiKh0JhR2EUlgNOi/VNVnAEBVu1Q1q6o5AI8CWFy6bhJRWIFhFxEB8DiAnar64JjL54z5sVUAthW/e0RULBN5N34pgJsAbBWRLfnL7gFwg4gsAqAAOgDcVpIefgHopq1m3Z4sGazhrcLbhluMmb5MJvJu/BvAuIuLm2PqRFRZeAYdkScYdiJPMOxEnmDYiTzBsBN5gmEn8gTDTuQJhp3IEww7kScYdiJPMOxEnmDYiTzBsBN5gmEn8kRZt2wWkR4AH425aAaAI2XrwOdTqX2r1H4B7Fuhitm301R15niFsob9Mzcu0q6qLZF1wFCpfavUfgHsW6HK1Tc+jSfyBMNO5Imow94a8e1bKrVvldovgH0rVFn6FulrdiIqn6iP7ERUJgw7kSciCbuILBeRXSLygYjcHUUfXESkQ0S2isgWEWmPuC9rRKRbRLaNuaxRRNaLyO7853H32Iuob/eKyMH8fbdFRFZE1LdmEXlVRHaKyHYR+WH+8kjvO6NfZbnfyv6aXUTiAN4HcCWAAwA2AbhBVXeUtSMOItIBoEVVIz8BQ0QuA3AcwJOqem7+svsB9Krqffl/lNNU9a4K6du9AI5HvY13freiOWO3GQdwDYBbEOF9Z/Tr+yjD/RbFkX0xgA9UdY+qjgD4FYCVEfSj4qnqBgC9J128EsDa/NdrMfrHUnaOvlUEVe1U1c35rwcAfLLNeKT3ndGvsogi7HMB7B/z/QFU1n7vCuAlEXlHRFZH3ZlxNKlqJzD6xwNgVsT9OVngNt7ldNI24xVz3xWy/XlYUYR9vK2kKmn8b6mqXgjgagC355+u0sRMaBvvchlnm/GKUOj252FFEfYDAJrHfH8qgEMR9GNcqnoo/7kbwLOovK2ouz7ZQTf/uTvi/vxZJW3jPd4246iA+y7K7c+jCPsmAPNF5HQRqQZwPYDnI+jHZ4hIff6NE4hIPYCrUHlbUT8P4Ob81zcDeC7CvnxKpWzj7dpmHBHfd5Fvf66qZf8AsAKj78h/COAnUfTB0a8zAPwp/7E96r4BeBqjT+vSGH1GdCuA6QDaAOzOf26soL79D4CtAN7DaLDmRNS3b2D0peF7ALbkP1ZEfd8Z/SrL/cbTZYk8wTPoiDzBsBN5gmEn8gTDTuQJhp3IEww7kScYdiJP/D866iIlQ3gtyAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "idx = 0\n",
    "plt.imshow(x_train[idx]);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Scale, reshape and categorize data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (60000, 28, 28, 1) x_test shape: (10000, 28, 28, 1)\n",
      "y_train shape: (60000, 10) y_test shape: (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "x_train = x_train.astype('float32') / 255\n",
    "x_test = x_test.astype('float32') / 255\n",
    "x_train = np.reshape(x_train, x_train.shape + (1,))\n",
    "x_test = np.reshape(x_test, x_test.shape + (1,))\n",
    "print('x_train shape:', x_train.shape, 'x_test shape:', x_test.shape)\n",
    "y_train = to_categorical(y_train)\n",
    "y_test = to_categorical(y_test)\n",
    "print('y_train shape:', y_train.shape, 'y_test shape:', y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define CNN model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model():\n",
    "    x_in = Input(shape=(28, 28, 1))\n",
    "    x = Conv2D(filters=64, kernel_size=2, padding='same', activation='relu')(x_in)\n",
    "    x = MaxPooling2D(pool_size=2)(x)\n",
    "    x = Dropout(0.3)(x)\n",
    "    \n",
    "    x = Conv2D(filters=32, kernel_size=2, padding='same', activation='relu')(x)\n",
    "    x = MaxPooling2D(pool_size=2)(x)\n",
    "    x = Dropout(0.3)(x)\n",
    "    \n",
    "    x = Flatten()(x)\n",
    "    x = Dense(256, activation='relu')(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    x_out = Dense(10, activation='softmax')(x)\n",
    "    \n",
    "    cnn = Model(inputs=x_in, outputs=x_out)\n",
    "    cnn.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    \n",
    "    return cnn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         [(None, 28, 28, 1)]       0         \n",
      "_________________________________________________________________\n",
      "conv2d (Conv2D)              (None, 28, 28, 64)        320       \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 14, 14, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 14, 14, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 14, 14, 32)        8224      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 7, 7, 32)          0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 7, 7, 32)          0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 1568)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 256)               401664    \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 10)                2570      \n",
      "=================================================================\n",
      "Total params: 412,778\n",
      "Trainable params: 412,778\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "cnn = model()\n",
    "cnn.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "60000/60000 [==============================] - 38s 630us/sample - loss: 0.5847 - acc: 0.7862\n",
      "Epoch 2/3\n",
      "60000/60000 [==============================] - 39s 643us/sample - loss: 0.4134 - acc: 0.8507\n",
      "Epoch 3/3\n",
      "60000/60000 [==============================] - 39s 644us/sample - loss: 0.3660 - acc: 0.8671\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7ffb62bdafd0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnn.fit(x_train, y_train, batch_size=64, epochs=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy:  0.8839\n"
     ]
    }
   ],
   "source": [
    "# Evaluate the model on test set\n",
    "score = cnn.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test accuracy: ', score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define superpixels\n",
    "\n",
    "Function to generate rectangular superpixels for a given image. Alternatively, use one of the built in methods. It is important to have meaningful superpixels in order to generate a useful explanation. Please check scikit-image's [segmentation methods](http://scikit-image.org/docs/dev/api/skimage.segmentation.html) (*felzenszwalb*, *slic* and *quickshift* built in the explainer) for more information on the built in methods."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def superpixel(image, size=(4, 7)):\n",
    "    segments = np.zeros([image.shape[0], image.shape[1]])\n",
    "    row_idx, col_idx = np.where(segments == 0)\n",
    "    for i, j in zip(row_idx, col_idx):\n",
    "        segments[i, j] = int((image.shape[1]/size[1]) * (i//size[0]) + j//size[1])\n",
    "    return segments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAALdUlEQVR4nO3dXYhc9RnH8d8va5KNm7RNtEljDNXa0KKFxrKkBaVYrBK9id6IubApCPFCQcGLir2ol6FUpRetEGswLVYpqJiL0JoGIUhFXCXNi7bGSsSEmK3Elxhtkt08vdgTWeNOdpzzNu7z/cAyM+fM5jw72e/OK/wdEQIw881qewAAzSB2IAliB5IgdiAJYgeSOKfJg83x3BjUUJOHbIRnzeC/mQMz9GebNdD2BLX45OQHOjH+safaVyp226sl/VbSgKQ/RMSGs11/UEP6oa8uc8i+NGv+grZHqM2s+TPvj7MkxYKZ+XO9sH9zx309/9m2PSDpd5Kuk3SppLW2L+313wNQrzKP0VZJeiMi3oyIE5KekLSmmrEAVK1M7MskvT3p8oFi22fYXm97xPbISR0vcTgAZdT+6ktEbIyI4YgYnq25dR8OQAdlYj8oafmkyxcW2wD0oTKxvyRphe2Lbc+RdLOkLdWMBaBqPb/1FhFjtu+Q9DdNvPW2KSL2VjYZgEqVep89IrZK2lrRLABqNEM/HgXgTMQOJEHsQBLEDiRB7EASxA4kQexAEsQOJEHsQBLEDiRB7EASxA4kQexAEsQOJEHsQBLEDiRB7EASxA4kQexAEsQOJEHsQBKNLtmscwfl717W6CGbMDZ/Ttsj1GZsqNlfkaacHJqZ93Pjo53/v2bmTwzgc4gdSILYgSSIHUiC2IEkiB1IgtiBJIgdSKLUJyZs75d0VNK4pLGIGK5iKADVq+LjUT+JiHcr+HcA1IiH8UASZWMPSc/aftn2+qmuYHu97RHbIyfHPi55OAC9Kvsw/sqIOGh7saRttv8VETsmXyEiNkraKElfGbogSh4PQI9K3bNHxMHidFTS05JWVTEUgOr1HLvtIdsLTp+XdK2kPVUNBqBaZR7GL5H0tO3T/86fI+KvlUwFoHI9xx4Rb0r6foWzAKgRb70BSRA7kASxA0kQO5AEsQNJEDuQBLEDSRA7kASxA0kQO5AEsQNJEDuQBLEDSTS6Hu/44IDev2xBk4dsxMlz3fYItRmbNzN/trFz256gHuPPd97HPTuQBLEDSRA7kASxA0kQO5AEsQNJEDuQBLEDSRA7kASxA0kQO5AEsQNJEDuQBLEDSRA7kASxA0lMG7vtTbZHbe+ZtG2R7W229xWnC+sdE0BZ3dyzPypp9Rnb7pG0PSJWSNpeXAbQx6aNPSJ2SDpyxuY1kjYX5zdLuqHiuQBUrNfn7Esi4lBx/h1JSzpd0fZ62yO2R8aOH+vxcADKKv0CXUSEpDjL/o0RMRwRw+fMHSp7OAA96jX2w7aXSlJxOlrdSADq0GvsWyStK86vk/RMNeMAqEs3b709LukFSd+xfcD2rZI2SLrG9j5JPy0uA+hj0y4SERFrO+y6uuJZANSIT9ABSRA7kASxA0kQO5AEsQNJNLxks/T+ipn392V8XscPEH7pjc8bb3uEWsQM/blOze38uzjzygMwJWIHkiB2IAliB5IgdiAJYgeSIHYgCWIHkiB2IAliB5IgdiAJYgeSIHYgCWIHkiB2IAliB5IgdiAJYgeSIHYgCWIHkiB2IAliB5IgdiAJYgeS6GZ99k22R23vmbTtPtsHbe8svq6vd0wAZXVzz/6opNVTbH8wIlYWX1urHQtA1aaNPSJ2SDrSwCwAalTmOfsdtncVD/MXdrqS7fW2R2yPjB87VuJwAMroNfaHJF0iaaWkQ5Lu73TFiNgYEcMRMTwwNNTj4QCU1VPsEXE4IsYj4pSkhyWtqnYsAFXrKXbbSyddvFHSnk7XBdAfpl2f3fbjkq6SdL7tA5J+Jekq2yslhaT9km7r5mAx95TGvv1Jz8P2q8F5J9oeoTbzB4+3PUItvjY4834PJem9uSc77ps29ohYO8XmR8oMBKB5fIIOSILYgSSIHUiC2IEkiB1IgtiBJIgdSILYgSSIHUiC2IEkiB1IgtiBJIgdSILYgSSIHUiC2IEkiB1IgtiBJIgdSILYgSSIHUiC2IEkiB1IgtiBJIgdSILYgSSIHUiC2IEkiB1IYtpVXKs0NPeEVl20v8lDNmLx4NG2R6jN4tkz82e7YM57bY9Qi9dnH+u4b9p7dtvLbT9n+1Xbe23fWWxfZHub7X3F6cIKZwZQsW4exo9JujsiLpX0I0m3275U0j2StkfECknbi8sA+tS0sUfEoYh4pTh/VNJrkpZJWiNpc3G1zZJuqGtIAOV9oRfobF8k6XJJL0paEhGHil3vSFrS4XvW2x6xPXL8vf+VGBVAGV3Hbnu+pCcl3RURH07eFxEhKab6vojYGBHDETE8d+FgqWEB9K6r2G3P1kToj0XEU8Xmw7aXFvuXShqtZ0QAVejm1XhLekTSaxHxwKRdWyStK86vk/RM9eMBqEo377NfIekWSbtt7yy23Stpg6S/2L5V0luSbqpnRABVmDb2iHhekjvsvrracQDUhY/LAkkQO5AEsQNJEDuQBLEDSRA7kASxA0kQO5AEsQNJEDuQBLEDSRA7kASxA0kQO5AEsQNJEDuQBLEDSRA7kASxA0kQO5AEsQNJNLpk83nnfKSfLflHk4dsxAXnfND2CLX5xsB42yPUYvHAUNsj1OL3A2Md93HPDiRB7EASxA4kQexAEsQOJEHsQBLEDiTRzfrsy20/Z/tV23tt31lsv8/2Qds7i6/r6x8XQK+6+VDNmKS7I+IV2wskvWx7W7HvwYj4TX3jAahKN+uzH5J0qDh/1PZrkpbVPRiAan2h5+y2L5J0uaQXi0132N5le5PthR2+Z73tEdsjHxyZmR+9BL4Muo7d9nxJT0q6KyI+lPSQpEskrdTEPf/9U31fRGyMiOGIGP7qooEKRgbQi65itz1bE6E/FhFPSVJEHI6I8Yg4JelhSavqGxNAWd28Gm9Jj0h6LSIemLR96aSr3ShpT/XjAahKN6/GXyHpFkm7be8stt0raa3tlZJC0n5Jt9UyIYBKdPNq/POSPMWurdWPA6AufIIOSILYgSSIHUiC2IEkiB1IgtiBJIgdSILYgSSIHUiC2IEkiB1IgtiBJIgdSILYgSQcEc0dzP6vpLcmbTpf0ruNDfDF9Ots/TqXxGy9qnK2b0bE16fa0Wjsnzu4PRIRw60NcBb9Olu/ziUxW6+amo2H8UASxA4k0XbsG1s+/tn062z9OpfEbL1qZLZWn7MDaE7b9+wAGkLsQBKtxG57te1/237D9j1tzNCJ7f22dxfLUI+0PMsm26O290zatsj2Ntv7itMp19hraba+WMb7LMuMt3rbtb38eePP2W0PSHpd0jWSDkh6SdLaiHi10UE6sL1f0nBEtP4BDNs/lvSRpD9GxPeKbb+WdCQiNhR/KBdGxC/6ZLb7JH3U9jLexWpFSycvMy7pBkk/V4u33VnmukkN3G5t3LOvkvRGRLwZESckPSFpTQtz9L2I2CHpyBmb10jaXJzfrIlflsZ1mK0vRMShiHilOH9U0ullxlu97c4yVyPaiH2ZpLcnXT6g/lrvPSQ9a/tl2+vbHmYKSyLiUHH+HUlL2hxmCtMu492kM5YZ75vbrpflz8viBbrPuzIifiDpOkm3Fw9X+1JMPAfrp/dOu1rGuylTLDP+qTZvu16XPy+rjdgPSlo+6fKFxba+EBEHi9NRSU+r/5aiPnx6Bd3idLTleT7VT8t4T7XMuPrgtmtz+fM2Yn9J0grbF9ueI+lmSVtamONzbA8VL5zI9pCka9V/S1FvkbSuOL9O0jMtzvIZ/bKMd6dlxtXybdf68ucR0fiXpOs18Yr8fyT9so0ZOsz1LUn/LL72tj2bpMc18bDupCZe27hV0nmStkvaJ+nvkhb10Wx/krRb0i5NhLW0pdmu1MRD9F2SdhZf17d9251lrkZuNz4uCyTBC3RAEsQOJEHsQBLEDiRB7EASxA4kQexAEv8HaVx2v2KULksAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "segments = superpixel(x_train[idx])\n",
    "plt.imshow(segments);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define prediction function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "predict_fn = lambda x: cnn.predict(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize anchor image explainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_shape = x_train[idx].shape\n",
    "explainer = AnchorImage(predict_fn, image_shape, segmentation_fn=superpixel)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Explain a prediction\n",
    "\n",
    "The explanation returns a mask with the superpixels that constitute the anchor.\n",
    "\n",
    "Image to be explained:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAQQklEQVR4nO3dW4xd9XXH8d+amTMXxjb24EtdY7ANBuFWwrRTkzaoIiJJCS8mUovgIaUSkiMVpCAhtYg+BPWJNk2jPlSRnAbFrVJQqgSBKtRALRoaJUKYS4yBhotlGpuxjRlfxte5rT7MBg0we+3h3NP1/UijObPX7H2Wz5yf9znnv/f+m7sLwP9/PZ1uAEB7EHYgCcIOJEHYgSQIO5BEXzvvrN8GfFDD7bxLIJXzOqNJv2AL1RoKu5ndLOkfJPVK+id3fyj6/UEN63q7qZG7BBB4zneX1up+GW9mvZL+UdKXJG2RdIeZbal3ewBaq5H37NskveXu+919UtKjkrY3py0AzdZI2NdJ+tW8nw8Wyz7CzHaY2R4z2zOlCw3cHYBGtPzTeHff6e6j7j5a00Cr7w5AiUbCfkjS+nk/X1osA9CFGgn785I2m9lGM+uXdLukJ5rTFoBmq3vozd2nzeweST/W3NDbw+7+atM6A9BUDY2zu/uTkp5sUi8AWojDZYEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJNDRls5kdkDQhaUbStLuPNqMpAM3XUNgLn3P3Y03YDoAW4mU8kESjYXdJT5nZC2a2Y6FfMLMdZrbHzPZM6UKDdwegXo2+jL/B3Q+Z2WpJT5vZ/7j7s/N/wd13StopSctsxBu8PwB1amjP7u6Hiu9HJT0maVszmgLQfHWH3cyGzWzpB7clfVHSvmY1BqC5GnkZv0bSY2b2wXb+1d3/oyldAWi6usPu7vslXdvEXgC0EENvQBKEHUiCsANJEHYgCcIOJNGME2GAjrC++OnrMzNBsbGDOXsuuiisz549G9btut8qrflLr9bVUxX27EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBOPs2c2dohzUK/YHs8FYtqTezZtKa0dvXBOuu/rfXgvrMydOhvVWqhpHr7L/tmWltY0vNbTpUuzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtkRqxhHr3L48+Vj6cdHp8J1z6wtP+dbki7765/V1VMz9F2+Pqwf2h7XaxPN7GZx2LMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMsydnfbWw7lOTYX3q878b1k9eXX599tp78X1fuOJ8XH9qQ1g/fGJpae2iwfjfdfzgxWG9tuJCWL946bGwfvLdePutULlnN7OHzeyome2bt2zEzJ42szeL7yta2yaARi3mZfz3JN38sWX3S9rt7psl7S5+BtDFKsPu7s9KGv/Y4u2SdhW3d0m6tcl9AWiyet+zr3H3seL2YUmlB0Cb2Q5JOyRpUPH8WABap+FP493dJZV+CuPuO9191N1Haxpo9O4A1KnesB8xs7WSVHw/2ryWALRCvWF/QtKdxe07JT3enHYAtErle3Yze0TSjZJWmtlBSV+X9JCkH5jZXZLekXRbK5tEA3p6w3LVOHrv8ng8+I0/jrdvwXD0zEA8R/rQkngs2yxev6envF617pVXj4X1/e+uDOvHTw6HdfU1Nj98PSrD7u53lJRuanIvAFqIw2WBJAg7kARhB5Ig7EAShB1IglNcFyua2tgrhlEqhr/ksxX1ePvWV/5n9OnpeNsV3r5vS1gfqDicqvd8+eN29rK4t4sG4ktNH3wvPtmyp7f8cZ2djfdz42eHwvrsZPw3HVgaDxvW+sv/7VXDnfVOVc2eHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSyDPOHo2TS9Vj5VX1SIPTHkfj6FJjY+lH//wPwvrk6nise/ne+HLQs0Hrfcvi02vHj8enifrx/rh+Sfn2a33x36TW29jfLDq9VpKWDJWPw09duyne9k9eqq+nutYC8GuHsANJEHYgCcIOJEHYgSQIO5AEYQeSyDPO3sg4uRSek269FZdrno7Hqqt6a2Qcfey+eBx94sp424OHKqZVHonv34PDGwaH4nH202NL4o0vicfCo8sEnD4Xz040NBD3psrDNip+IfDOzYNhfeNP6tsue3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSOLXa5y96vrrkaprs1vF/3vBOene4PnqVXqv3BjWD9y+trQ2M1RxXvXb8VNgumLm4applydHyh+b/sn4vq1irLpvqOL4hcDMTPz3Pj8ZH1+gmbi3C2crzvOfLV//8m0H4/uuU+We3cweNrOjZrZv3rIHzeyQmb1cfN3Sku4ANM1iXsZ/T9LNCyz/lrtvLb6ebG5bAJqtMuzu/qyk8Tb0AqCFGvmA7h4z21u8zC+ddMvMdpjZHjPbM6V4/isArVNv2L8t6QpJWyWNSfpm2S+6+053H3X30Zrikw8AtE5dYXf3I+4+4+6zkr4jaVtz2wLQbHWF3czmj/V8WdK+st8F0B0qx9nN7BFJN0paaWYHJX1d0o1mtlWSSzog6auLujdrcC7xVo5ne/3b7lt/aVg/d/WasD5+Tfz25txvxGPZPcGp17WJeDx48uJ429NLK861r1VcJ6C//PgGD8aaJeniS+N5yAdq8fNl/GT5QQIz0xXXIKjoTRXXhfdzFccv9Javf+x0fHDDqt+/trz4i5+VlirD7u53LLD4u1XrAeguHC4LJEHYgSQIO5AEYQeSIOxAEu09xdUbuyxy34bLSmvnrlodrju1JB5qmRyO/9+bHiqvTWwIV608zbRnKq73nYmHgTxofXJZvO2ZwbhuVaOhQ/Gpw3au/HGfmowf88n++M5PHFka1mvLyg/PrrqM9ZkTwR9cUm04Xn/V8tNh/eTZ8u1fs/JIuO7B1ZtLa7O18ucKe3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSKKrLiV9+k+uj+u/WT5m21MxHnx+ZVz34JRDSbLg0sE90xXrno7HyaeH4/XPr6k4/TbafHCKqST1noifAtEYviT1Lokf+J6e8vufqrjc8rkz8am/vafiYycGVtV/TEeVqRPxtMpHZ+MHLhrnX95/Llz33eC4DAueSuzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiCJto6zz64Y1sQffaa0Pv2n74frn37zktLa4JH4/61afHqxvCceC48u1+y9FZcdrijXKsbhZ2vxv82CofSpiktBV/VWdb575UzYfeXrj6w+Fa57zSVH441fGZeX1c6X1vqs4tiF9XH58PllYX31QPyEG5+8qLT27tmLw3WH3j1TWuuZLP+DsGcHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSTaOs7eO3FBy/9rf2n9jW2bwvVXb3mvtHb57x2vuy9JOj8dn1t95OyS0tqx4/H1y6dP9If1WsV52bMV0yJ7MFbuI1Phuls3/W9YXzUYjxdvGjoW1meCE+IfWPnLcN2/eb/8+uiS9NSRa8L6N67699LaSG98rvyMVxyfUOGsx4/7j8+Wz4Hw1vl4iu//Xr6utOZ95Y935Z7dzNab2TNm9pqZvWpmXyuWj5jZ02b2ZvF9RdW2AHTOYl7GT0u6z923SPqMpLvNbIuk+yXtdvfNknYXPwPoUpVhd/cxd3+xuD0h6XVJ6yRtl7Sr+LVdkm5tVZMAGvep3rOb2QZJ10l6TtIadx8rSoclLfhGw8x2SNohSYM95e97AbTWoj+NN7Mlkn4o6V53/8gZDO7ukhb8RMPdd7r7qLuP9vfEk+UBaJ1Fhd3MapoL+vfd/UfF4iNmtraor5VUcYoSgE4yrxhiMDPT3HvycXe/d97yb0h6390fMrP7JY24+19E21pmI3693dSEtj+pd0U8GHDqpqvC+vGr4uGvvm3lQ3tXjMTDT5cNx8OC6wbieu/CL5o+NBOcpzo1G79Te+302rD+8/0bw/qKZ+JLKq96dG9pbfZM+amazTC7u/w81c+teiNcd+9E+fCWJB0+E5/i+v6Z8lNYJWl6OprKOv6bXXV3+fD1z089rpPT7y34hFjMe/bPSvqKpFfM7OVi2QOSHpL0AzO7S9I7km5bxLYAdEhl2N39pyq/xEFrdtMAmo7DZYEkCDuQBGEHkiDsQBKEHUiicpy9mVo5zg5Aes5365SPLzh6xp4dSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSqAy7ma03s2fM7DUze9XMvlYsf9DMDpnZy8XXLa1vF0C9FjM/+7Sk+9z9RTNbKukFM3u6qH3L3f+ude0BaJbFzM8+JmmsuD1hZq9LWtfqxgA016d6z25mGyRdJ+m5YtE9ZrbXzB42sxUl6+wwsz1mtmdKFxpqFkD9Fh12M1si6YeS7nX3U5K+LekKSVs1t+f/5kLruftOdx9199GaBprQMoB6LCrsZlbTXNC/7+4/kiR3P+LuM+4+K+k7kra1rk0AjVrMp/Em6buSXnf3v5+3fO28X/uypH3Nbw9Asyzm0/jPSvqKpFfM7OVi2QOS7jCzrZJc0gFJX21JhwCaYjGfxv9U0kLzPT/Z/HYAtApH0AFJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Iwd2/fnZm9J+mdeYtWSjrWtgY+nW7trVv7kuitXs3s7XJ3X7VQoa1h/8Sdm+1x99GONRDo1t66tS+J3urVrt54GQ8kQdiBJDod9p0dvv9It/bWrX1J9FavtvTW0ffsANqn03t2AG1C2IEkOhJ2M7vZzH5pZm+Z2f2d6KGMmR0ws1eKaaj3dLiXh83sqJntm7dsxMyeNrM3i+8LzrHXod66YhrvYJrxjj52nZ7+vO3v2c2sV9Ibkr4g6aCk5yXd4e6vtbWREmZ2QNKou3f8AAwz+0NJpyX9s7v/drHsbyWNu/tDxX+UK9z9L7uktwclne70NN7FbEVr508zLulWSX+mDj52QV+3qQ2PWyf27NskveXu+919UtKjkrZ3oI+u5+7PShr/2OLtknYVt3dp7snSdiW9dQV3H3P3F4vbE5I+mGa8o49d0FdbdCLs6yT9at7PB9Vd8727pKfM7AUz29HpZhawxt3HituHJa3pZDMLqJzGu50+Ns141zx29Ux/3ig+oPukG9z9dyR9SdLdxcvVruRz78G6aex0UdN4t8sC04x/qJOPXb3TnzeqE2E/JGn9vJ8vLZZ1BXc/VHw/Kukxdd9U1Ec+mEG3+H60w/18qJum8V5omnF1wWPXyenPOxH25yVtNrONZtYv6XZJT3Sgj08ws+HigxOZ2bCkL6r7pqJ+QtKdxe07JT3ewV4+olum8S6bZlwdfuw6Pv25u7f9S9ItmvtE/m1Jf9WJHkr62iTpF8XXq53uTdIjmntZN6W5zzbuknSJpN2S3pT0n5JGuqi3f5H0iqS9mgvW2g71doPmXqLvlfRy8XVLpx+7oK+2PG4cLgskwQd0QBKEHUiCsANJEHYgCcIOJEHYgSQIO5DE/wE8/ft8ncLFKQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "i = 0\n",
    "image = x_test[i]\n",
    "plt.imshow(image[:,:,0]);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model prediction:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnn.predict(image.reshape(1, 28, 28, 1)).argmax()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The predicted category correctly corresponds to the class `ankle boot`:\n",
    "\n",
    "| Label | Description |\n",
    "| --- | --- |\n",
    "| 0 | T-shirt/top |\n",
    "| 1 | Trouser |\n",
    "| 2 | Pullover |\n",
    "| 3 | Dress |\n",
    "| 4 | Coat |\n",
    "| 5 | Sandal |\n",
    "| 6 | Shirt |\n",
    "| 7 | Sneaker |\n",
    "| 8 | Bag |\n",
    "| 9 | Ankle boot |"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generate explanation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "explanation = explainer.explain(image, threshold=.95, p_sample=.8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Show anchor:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAK3UlEQVR4nO3df6jd9X3H8edLjbFNO4izC8G62XbZDxksHZdsUBkdrp31n9h/SgPrUhDSPyq00D8m7o/5pxtry/4YhXSGZlurDFrRP2QzDYIUhvPqMo26VifpTIjJnH/UFhajvvfH/Vpu9d5zr+fHPWd9Px9wOOd8v+fc75uDz5yf8klVIenn3yXzHkDS1jB2qQljl5owdqkJY5eauGwrD3Z5ttcV7NjKQ0qt/C8/4dW6kLX2TRR7khuBvwYuBf62qu4cdfsr2MHv5oZJDilphEfq+Lr7xn4Zn+RS4G+ATwDXAQeSXDfu35M0W5O8Z98HPFdVz1fVq8A9wP7pjCVp2iaJ/WrghVXXTw/bfkaSQ0mWkyxf5MIEh5M0iZl/Gl9Vh6tqqaqWtrF91oeTtI5JYj8DXLPq+vuHbZIW0CSxPwrsSfKBJJcDnwbun85YkqZt7K/equq1JLcC/8zKV29HquqpqU0maaom+p69qh4AHpjSLJJmyJ/LSk0Yu9SEsUtNGLvUhLFLTRi71ISxS00Yu9SEsUtNGLvUhLFLTRi71ISxS00Yu9SEsUtNGLvUhLFLTRi71ISxS00Yu9SEsUtNGLvUhLFLTRi71ISxS00Yu9SEsUtNGLvUhLFLTRi71MRESzYnOQW8ArwOvFZVS9MYStL0TRT74A+q6qUp/B1JM+TLeKmJSWMv4MEkjyU5tNYNkhxKspxk+SIXJjycpHFN+jL++qo6k+SXgGNJ/qOqHl59g6o6DBwG+IVcWRMeT9KYJnpmr6ozw/l54F5g3zSGkjR9Y8eeZEeS9755Gfg4cHJag0marklexu8C7k3y5t/5VlX901SmkjR1Y8deVc8Dvz3FWSTNkF+9SU0Yu9SEsUtNGLvUhLFLTRi71ISxS00Yu9SEsUtNGLvUhLFLTRi71ISxS00Yu9SEsUtNGLvUhLFLTRi71ISxS00Yu9SEsUtNGLvUhLFLTRi71ISxS00Yu9SEsUtNGLvUhLFLTRi71MSGsSc5kuR8kpOrtl2Z5FiSZ4fznbMdU9KkNvPM/g3gxrdsuw04XlV7gOPDdUkLbMPYq+ph4OW3bN4PHB0uHwVunvJckqbssjHvt6uqzg6XXwR2rXfDJIeAQwBX8O4xDydpUhN/QFdVBdSI/YeraqmqlraxfdLDSRrTuLGfS7IbYDg/P72RJM3CuLHfDxwcLh8E7pvOOJJmZTNfvd0N/Avw60lOJ7kFuBP4WJJngT8crktaYBt+QFdVB9bZdcOUZ5E0Q/6CTmrC2KUmjF1qwtilJoxdasLYpSaMXWrC2KUmjF1qwtilJoxdasLYpSaMXWrC2KUmjF1qwtilJoxdasLYpSaMXWrC2KUmjF1qwtilJoxdasLYpSaMXWrC2KUmjF1qwtilJoxdasLYpSY2sz77kSTnk5xcte2OJGeSnBhON812TEmT2swz+zeAG9fY/tWq2jucHpjuWJKmbcPYq+ph4OUtmEXSDE3ynv3WJE8ML/N3rnejJIeSLCdZvsiFCQ4naRLjxv414EPAXuAs8OX1blhVh6tqqaqWtrF9zMNJmtRYsVfVuap6vareAL4O7JvuWJKmbazYk+xedfWTwMn1bitpMVy20Q2S3A18FLgqyWngz4GPJtkLFHAK+NwMZ5Q0BRvGXlUH1th81wxmkTRD/oJOasLYpSaMXWrC2KUmjF1qwtilJoxdasLYpSaMXWrC2KUmjF1qwtilJoxdasLYpSaMXWrC2KUmjF1qwtilJoxdasLYpSaMXWrC2KUmjF1qwtilJoxdasLYpSaMXWrC2KUmjF1qYsNVXNXbtf/6rpH7P/iul0buf73Wfz65/arvj7zvX/zPnpH7Hzz3myP3/8OvfWvdfZ/95etH3vfn0YbP7EmuSfJQkqeTPJXkC8P2K5McS/LscL5z9uNKGtdmXsa/Bnypqq4Dfg/4fJLrgNuA41W1Bzg+XJe0oDaMvarOVtXjw+VXgGeAq4H9wNHhZkeBm2c1pKTJvaP37EmuBT4MPALsqqqzw64XgV3r3OcQcAjgCt497pySJrTpT+OTvAf4NvDFqvrR6n1VVUCtdb+qOlxVS1W1tI3tEw0raXybij3JNlZC/2ZVfWfYfC7J7mH/buD8bEaUNA0bvoxPEuAu4Jmq+sqqXfcDB4E7h/P7ZjKh5uq7P/iNkft3PnTFyP3vu+eJdff90U9Gf623kcv4r5H7/+T4gXX3XcILEx37/6PNvGf/CPAZ4MkkJ4Ztt7MS+T8muQX4IfCp2YwoaRo2jL2qvgdknd03THccSbPiz2WlJoxdasLYpSaMXWrC2KUm/F9cNdKv/vG/TXT/N6Y0xzguuaHfd+mj+MwuNWHsUhPGLjVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41sWHsSa5J8lCSp5M8leQLw/Y7kpxJcmI43TT7cSWNazOLRLwGfKmqHk/yXuCxJMeGfV+tqr+a3XiSpmUz67OfBc4Ol19J8gxw9awHkzRd7+g9e5JrgQ8Djwybbk3yRJIjSXauc59DSZaTLF/kwkTDShrfpmNP8h7g28AXq+pHwNeADwF7WXnm//Ja96uqw1W1VFVL29g+hZEljWNTsSfZxkro36yq7wBU1bmqer2q3gC+Duyb3ZiSJrWZT+MD3AU8U1VfWbV996qbfRI4Of3xJE3LZj6N/wjwGeDJJCeGbbcDB5LsBQo4BXxuJhNKmorNfBr/PSBr7Hpg+uNImhV/QSc1YexSE8YuNWHsUhPGLjVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41YexSE6mqrTtY8t/AD1dtugp4acsGeGcWdbZFnQucbVzTnO1Xqup9a+3Y0tjfdvBkuaqW5jbACIs626LOBc42rq2azZfxUhPGLjUx79gPz/n4oyzqbIs6FzjbuLZktrm+Z5e0deb9zC5pixi71MRcYk9yY5LvJ3kuyW3zmGE9SU4leXJYhnp5zrMcSXI+yclV265McizJs8P5mmvszWm2hVjGe8Qy43N97Oa9/PmWv2dPcinwA+BjwGngUeBAVT29pYOsI8kpYKmq5v4DjCS/D/wY+Luq+q1h218CL1fVncM/lDur6k8XZLY7gB/PexnvYbWi3auXGQduBj7LHB+7EXN9ii143ObxzL4PeK6qnq+qV4F7gP1zmGPhVdXDwMtv2bwfODpcPsrKfyxbbp3ZFkJVna2qx4fLrwBvLjM+18duxFxbYh6xXw28sOr6aRZrvfcCHkzyWJJD8x5mDbuq6uxw+UVg1zyHWcOGy3hvpbcsM74wj904y59Pyg/o3u76qvod4BPA54eXqwupVt6DLdJ3p5taxnurrLHM+E/N87Ebd/nzSc0j9jPANauuv3/YthCq6sxwfh64l8VbivrcmyvoDufn5zzPTy3SMt5rLTPOAjx281z+fB6xPwrsSfKBJJcDnwbun8Mcb5Nkx/DBCUl2AB9n8Zaivh84OFw+CNw3x1l+xqIs473eMuPM+bGb+/LnVbXlJ+AmVj6R/0/gz+YxwzpzfRD49+H01LxnA+5m5WXdRVY+27gF+EXgOPAs8F3gygWa7e+BJ4EnWAlr95xmu56Vl+hPACeG003zfuxGzLUlj5s/l5Wa8AM6qQljl5owdqkJY5eaMHapCWOXmjB2qYn/A9ogU9n7bD0zAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(explanation['anchor'][:,:,0]);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the example, it looks like the arch of the boot alone is sufficient to predict an ankle boot."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}