examples/anchor_image_fashion_mnist.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Anchor explanations for fashion MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D, Input\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from alibi.explainers import AnchorImage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load and prepare fashion MNIST data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (60000, 28, 28) y_train shape: (60000,)\n"
     ]
    }
   ],
   "source": [
    "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()\n",
    "print('x_train shape:', x_train.shape, 'y_train shape:', y_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAUFElEQVR4nO3da2yc1ZkH8P8z4/ElzjiJk+CE4BIuoZDCEqhJuIlSKDREVQOli4gQCxLaoF3otl0+gGhXZb+sEFpAaNntroEsYVWoWhUERREFzCULlDQmpOS2ITeHxDi2ExPbcTz2XJ794Bdqgs/zmnnnRs7/J1kezzNn5njGf78zc+acI6oKIjr+xcrdASIqDYadyBMMO5EnGHYiTzDsRJ6oKuWNVUuN1qK+lDdJ5JUUhjCqIzJRLVLYRWQpgEcAxAE8rqr3W5evRT2WyJVRbpKIDOu0zVnL+2m8iMQB/DuAawAsBLBCRBbme31EVFxRXrMvBrBTVXer6iiAXwNYXphuEVGhRQn7PAD7xv28Pzjvc0RkpYi0i0h7GiMRbo6Ioij6u/Gq2qqqLarakkBNsW+OiByihL0TQPO4n08KziOiChQl7OsBLBCRU0SkGsCNAF4oTLeIqNDyHnpT1YyI3AngDxgbelulqlsK1jMiKqhI4+yqugbAmgL1hYiKiB+XJfIEw07kCYadyBMMO5EnGHYiTzDsRJ5g2Ik8wbATeYJhJ/IEw07kCYadyBMMO5EnGHYiT5R0KWkqA5lwVeG/iLixZ3xmo1n/5LtnOGsNT78b6bbDfjepSjhrmh6NdttRhT0uljwfMx7ZiTzBsBN5gmEn8gTDTuQJhp3IEww7kScYdiJPcJz9OCfxuFnXTMasxxbZe3Vuu32q3X7YXUsMLTbbVg3nzHri5XazHmksPWwMP+R+hdjH0Sh9kyojtsbDySM7kScYdiJPMOxEnmDYiTzBsBN5gmEn8gTDTuQJjrMf58wxWYSPs+/77nSzftNF/2vW3+491VnbWzPHbKt1ZhlV37nIrJ/xH53OWqbjI/vKQ+aMh91vYeIzZriL2azZNjsw4C4a3Y4UdhHpADAIIAsgo6otUa6PiIqnEEf2b6vqwQJcDxEVEV+zE3kiatgVwMsi8p6IrJzoAiKyUkTaRaQ9jZGIN0dE+Yr6NP5SVe0UkRMAvCIi/6eqa8dfQFVbAbQCQIM0RlvdkIjyFunIrqqdwfceAM8BsKcxEVHZ5B12EakXkeSnpwFcDWBzoTpGRIUV5Wl8E4DnZGzebxWAp1X1pYL0igoml0pFaj963hGz/sNp9pzy2ljaWXszZs9X73yt2axn/8ru296Hks5a7v2LzbYzN9tj3Q3vd5n1g5fNM+u933S/om0KWU5/xqu7nDXpc0c677Cr6m4A5+bbnohKi0NvRJ5g2Ik8wbATeYJhJ/IEw07kCdGIW/Z+GQ3SqEvkypLdnjesZY9DHt8jN1xo1q/5+Rtm/azaj836YK7WWRvVaB/gfHT7t8z60O5pzlpsNGTL5JBytsleClrT9nF0xgb37163vNtsK4/NdtY+aHsER/r2Tdh7HtmJPMGwE3mCYSfyBMNO5AmGncgTDDuRJxh2Ik9wnL0ShGwPHEnI43v2e/b/+x/MsKewhokbaxsPabXZ9nC2PtJt92bcU1zTIWP8j++wp8AeMcbwASCWsR/Tq779vrN2feN6s+0Dp53jrK3TNgxoH8fZiXzGsBN5gmEn8gTDTuQJhp3IEww7kScYdiJPcMvmSlDCzzoca8eRE8z6oYapZv1Axt7SeWbcvdxzMjZstp2fsPcL7c26x9EBIJ5wL1U9qnGz7T9/4/dmPXVWwqwnxF6K+mJjHYC/3vo3Ztt67DbrLjyyE3mCYSfyBMNO5AmGncgTDDuRJxh2Ik8w7ESe4Di752bX2Nse14p7y2UAqJaMWf84PcNZ2zH8dbPthwP2ZwCWNm0x62ljLN2aZw+Ej5OfmPjErKfUHoe37tVLmuxx9I1m1S30yC4iq0SkR0Q2jzuvUUReEZEdwXf3I0pEFWEyT+OfBLD0mPPuAdCmqgsAtAU/E1EFCw27qq4F0HfM2csBrA5OrwZwbYH7RUQFlu9r9iZV7QpOHwDQ5LqgiKwEsBIAajElz5sjoqgivxuvYytWOt/tUNVWVW1R1ZYEaqLeHBHlKd+wd4vIXAAIvvcUrktEVAz5hv0FALcEp28B8HxhukNExRL6ml1EngFwOYBZIrIfwC8A3A/gNyJyG4C9AG4oZiePeyHrxkvcnnutGfdYd3yGPSr6rembzHpvtsGsH87a78NMjx911gYz7r3bAaBv2L7uM2u6zPqGo/OdtdnV9ji51W8A6BidZdYX1Bww6w90u/dPaK499v3wz8tceZmzpuv+6KyFhl1VVzhK3O2B6CuEH5cl8gTDTuQJhp3IEww7kScYdiJPcIprJQhZSlqq7IfJGnrbd9tZZtsrpthLJr+TmmfWZ1cNmnVrmuncmn6zbbIpZdbDhv0aq9zTdwezdWbbKbERsx72e59fbS+D/dNXz3fWkmcfMts2JIxjtDGKyyM7kScYdiJPMOxEnmDYiTzBsBN5gmEn8gTDTuQJjrNXAElUm/Vcyh5vtszaNGrWD2btJY+nx+ypntUhSy5bWyNf3LjHbNsbMha+YfgUs56Mu7eEnh2zx8mbE/ZY96ZUs1lfM3S6Wb/te686a8+0XmW2rX7pHWdN1P148chO5AmGncgTDDuRJxh2Ik8w7ESeYNiJPMGwE3niqzXObiy5LFX2eLHEQ/6vxex6LmXMb87ZY81hNG2PhUfxyH89atb3Zaab9QNpux625HLWmGD97vA0s21tzN4uenbVgFkfyNnj9JbBnL3MtTVPHwjv+90zdzhrz/Z/x2ybLx7ZiTzBsBN5gmEn8gTDTuQJhp3IEww7kScYdiJPVNQ4e5T10cPGqtUe9iyr4eWLzfq+a+1x/JvO+5OzdiCTNNu+b2xrDADTjDnhAFAfsr56St2ff/h41N5OOmys2loXHgBOMMbhs2of5zrTdt/ChH3+YH/GWNP++/Zc++lP5dWl8CO7iKwSkR4R2TzuvPtEpFNENgZfy/K7eSIqlck8jX8SwNIJzn9YVRcFX2sK2y0iKrTQsKvqWgB9JegLERVRlDfo7hSRD4Kn+c4XOCKyUkTaRaQ9Dfv1HREVT75h/yWA0wAsAtAF4EHXBVW1VVVbVLUlgZo8b46Iosor7KrarapZVc0BeAyA/XYyEZVdXmEXkbnjfrwOwGbXZYmoMoSOs4vIMwAuBzBLRPYD+AWAy0VkEQAF0AHg9kJ0xhpHj6pq7hyznj6lyaz3neXeC/zoHGNTbACLlm0z67c2/bdZ7802mPWEGPuzp2eabc+b0mHWX+tfaNYPVk0169Y4/cX17jndAHA4Z++/fmLVJ2b97p0/dNaapthj2Y+fbA8wpTVn1ren7Zes/Tn3fPh/WPi62fY5zDbrLqFhV9UVE5z9RF63RkRlw4/LEnmCYSfyBMNO5AmGncgTDDuRJypqiuvINReY9RN+tttZW9Sw32y7sO4ts57K2UtRW9Mttw7PM9sezdlbMu8YtYcF+zP2EFRc3MNAPaP2FNcH99jLFrct/k+z/vOPJ5oj9RexOnXWDmXtYbvrp9pLRQP2Y3b719Y6a6dW95htXxyaa9Y/DpkC25ToN+vzE73O2g+SH5pt8x1645GdyBMMO5EnGHYiTzDsRJ5g2Ik8wbATeYJhJ/JEacfZxV4uesm/rDebX5nc4qwdVXtKYdg4eti4qWValb1s8Ejavpt70vYU1jBn1Bxw1q5r2Gi2XfvoErN+aepHZn3XFfb03LZh91TO3oz9e9+45wqzvuGjZrN+4fw9zto5yU6zbdhnG5LxlFm3ph0DwFDO/ff6bsr+/EG+eGQn8gTDTuQJhp3IEww7kScYdiJPMOxEnmDYiTwhqu75xoVWN6dZT7v5H5311jv+zWz/dN+Fzlpzrb0d3cnVB836zLi9/a8lGbPHXL+esMdcXxw6yay/cfhMs/7NZIezlhB7u+fLp+w067f+9C6znqm1l9EemO8+nmTq7b+9hnMPmfUfnf6aWa82fvfDWXscPex+C9uSOYy1BkEyZm+T/eCy65y1P3Y8if7hrgkfFB7ZiTzBsBN5gmEn8gTDTuQJhp3IEww7kScYdiJPlHQ+eywNTOl2jy++OLDIbH9qnXut7YNpe330Pxw5x6yfVGdv/2ttPXy6MZ8cADamppv1l3q/YdZPrLPXT+9OT3PWDqXrzbZHjXnVAPDEww+Z9Qe77XXnr2vc4KydW22Pox/O2ceirSHr7Q/map21lNrrG/SHjMMnjb8HAEirHa24seXz9Jg9hj9wjnsb7my3+3ZDj+wi0iwir4vIVhHZIiI/Ds5vFJFXRGRH8D3/1R+IqOgm8zQ+A+AuVV0I4EIAd4jIQgD3AGhT1QUA2oKfiahChYZdVbtUdUNwehDANgDzACwHsDq42GoA1xark0QU3Zd6g05E5gM4D8A6AE2q2hWUDgBocrRZKSLtItKeGRmK0FUiimLSYReRqQB+B+Anqvq5d4x0bDbNhLMaVLVVVVtUtaWqxn6ziIiKZ1JhF5EExoL+K1V9Nji7W0TmBvW5AOxtMYmorEKH3kREADwBYJuqjh+HeQHALQDuD74/H3Zd8dEckvtGnPWc2tMlXzvonurZVDtotl2U3GfWtx+1h3E2DZ/orG2o+prZti7u3u4ZAKZV21Nk66vc9xkAzEq4f/dTauz/wdY0UABYn7J/t7+b/YZZ/yjjHqT5/dAZZtutR933OQDMCFnCe9OAu/3RjL2N9kjWjkYqYw/lTquxH9MLGvc6a9thbxfde64xbfhtd7vJjLNfAuBmAJtE5NNFyO/FWMh/IyK3AdgL4IZJXBcRlUlo2FX1LQCuQ+6Vhe0OERULPy5L5AmGncgTDDuRJxh2Ik8w7ESeKO2WzUeGEXvzfWf5ty9fYjb/p+W/ddbeDFlu+cUD9rjowKg91XP2FPdHfRuMcW4AaEzYHxMO2/K5NmT7308y7k8mjsTsqZxZ50DLmAMj7umzAPB2boFZT+fcWzaPGDUg/PMJfaOzzPqJdf3O2mDGPf0VADoGG836wX57W+XUFDtab2VPc9aWznFvTQ4AdT3uxyxm/KnwyE7kCYadyBMMO5EnGHYiTzDsRJ5g2Ik8wbATeaKkWzY3SKMukfwnyvXf5N6y+dS/3262XTx9j1nfMGDP2/7IGHdNhyx5nIi5lw0GgCmJUbNeGzLeXB13z0mPTbyA0GdyIePs9XG7b2Fz7Ruq3PO6k3F7znfM2NZ4MuLG7/6n/vmRrjsZ8ntn1P6buGjaLmdt1Z6LzbbTlrm32V6nbRjQPm7ZTOQzhp3IEww7kScYdiJPMOxEnmDYiTzBsBN5ovTj7PGr3RfI2WuYRzF0/RKzvuTe9XY96R4XPbO622ybgD1eXBsynlwfs8fCU8ZjGPbf/K3hZrOeDbmG1z45y6ynjfHm7qMNZtuE8fmBybD2IRjOhGzZPGzPd4/H7Nyk3rDn2s/c6v7sRM0a+2/RwnF2ImLYiXzBsBN5gmEn8gTDTuQJhp3IEww7kSdCx9lFpBnAUwCaACiAVlV9RETuA/C3AHqDi96rqmus64o6n71SyQX2mvTDc+rMes0he2704Ml2+4Zd7nXpYyP2mvO5P28z6/TVYo2zT2aTiAyAu1R1g4gkAbwnIq8EtYdV9V8L1VEiKp7J7M/eBaArOD0oItsAzCt2x4iosL7Ua3YRmQ/gPADrgrPuFJEPRGSViMxwtFkpIu0i0p6G/XSViIpn0mEXkakAfgfgJ6o6AOCXAE4DsAhjR/4HJ2qnqq2q2qKqLQnY+6kRUfFMKuwiksBY0H+lqs8CgKp2q2pWVXMAHgOwuHjdJKKoQsMuIgLgCQDbVPWhcefPHXex6wBsLnz3iKhQJvNu/CUAbgawSUQ2BufdC2CFiCzC2HBcB4Dbi9LDrwBdv8ms25MlwzW8k3/baIsx0/FkMu/GvwVMuLi4OaZORJWFn6Aj8gTDTuQJhp3IEww7kScYdiJPMOxEnmDYiTzBsBN5gmEn8gTDTuQJhp3IEww7kScYdiJPMOxEnijpls0i0gtg77izZgE4WLIOfDmV2rdK7RfAvuWrkH07WVVnT1Qoadi/cOMi7araUrYOGCq1b5XaL4B9y1ep+san8USeYNiJPFHusLeW+fYtldq3Su0XwL7lqyR9K+trdiIqnXIf2YmoRBh2Ik+UJewislREtovIThG5pxx9cBGRDhHZJCIbRaS9zH1ZJSI9IrJ53HmNIvKKiOwIvk+4x16Z+nafiHQG991GEVlWpr41i8jrIrJVRLaIyI+D88t63xn9Ksn9VvLX7CISB/AhgKsA7AewHsAKVd1a0o44iEgHgBZVLfsHMETkMgBHADylqmcH5z0AoE9V7w/+Uc5Q1bsrpG/3AThS7m28g92K5o7fZhzAtQBuRRnvO6NfN6AE91s5juyLAexU1d2qOgrg1wCWl6EfFU9V1wLoO+bs5QBWB6dXY+yPpeQcfasIqtqlqhuC04MAPt1mvKz3ndGvkihH2OcB2Dfu5/2orP3eFcDLIvKeiKwsd2cm0KSqXcHpAwCaytmZCYRu411Kx2wzXjH3XT7bn0fFN+i+6FJVPR/ANQDuCJ6uViQdew1WSWOnk9rGu1Qm2Gb8M+W87/Ld/jyqcoS9E0DzuJ9PCs6rCKraGXzvAfAcKm8r6u5Pd9ANvveUuT+fqaRtvCfaZhwVcN+Vc/vzcoR9PYAFInKKiFQDuBHAC2XoxxeISH3wxglEpB7A1ai8rahfAHBLcPoWAM+XsS+fUynbeLu2GUeZ77uyb3+uqiX/ArAMY+/I7wLws3L0wdGvUwH8OfjaUu6+AXgGY0/r0hh7b+M2ADMBtAHYAeBVAI0V1Lf/AbAJwAcYC9bcMvXtUow9Rf8AwMbga1m57zujXyW53/hxWSJP8A06Ik8w7ESeYNiJPMGwE3mCYSfyBMNO5AmGncgT/w8K8iUImXY9pQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "idx = 0\n",
    "plt.imshow(x_train[idx]);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Scale, reshape and categorize data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (60000, 28, 28, 1) x_test shape: (10000, 28, 28, 1)\n",
      "y_train shape: (60000, 10) y_test shape: (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "x_train = x_train.astype('float32') / 255\n",
    "x_test = x_test.astype('float32') / 255\n",
    "x_train = np.reshape(x_train, x_train.shape + (1,))\n",
    "x_test = np.reshape(x_test, x_test.shape + (1,))\n",
    "print('x_train shape:', x_train.shape, 'x_test shape:', x_test.shape)\n",
    "y_train = to_categorical(y_train)\n",
    "y_test = to_categorical(y_test)\n",
    "print('y_train shape:', y_train.shape, 'y_test shape:', y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define CNN model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model():\n",
    "    x_in = Input(shape=(28, 28, 1))\n",
    "    x = Conv2D(filters=64, kernel_size=2, padding='same', activation='relu')(x_in)\n",
    "    x = MaxPooling2D(pool_size=2)(x)\n",
    "    x = Dropout(0.3)(x)\n",
    "    \n",
    "    x = Conv2D(filters=32, kernel_size=2, padding='same', activation='relu')(x)\n",
    "    x = MaxPooling2D(pool_size=2)(x)\n",
    "    x = Dropout(0.3)(x)\n",
    "    \n",
    "    x = Flatten()(x)\n",
    "    x = Dense(256, activation='relu')(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    x_out = Dense(10, activation='softmax')(x)\n",
    "    \n",
    "    cnn = Model(inputs=x_in, outputs=x_out)\n",
    "    cnn.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    \n",
    "    return cnn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         [(None, 28, 28, 1)]       0         \n",
      "_________________________________________________________________\n",
      "conv2d (Conv2D)              (None, 28, 28, 64)        320       \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 14, 14, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 14, 14, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 14, 14, 32)        8224      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 7, 7, 32)          0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 7, 7, 32)          0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 1568)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 256)               401664    \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 10)                2570      \n",
      "=================================================================\n",
      "Total params: 412,778\n",
      "Trainable params: 412,778\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "cnn = model()\n",
    "cnn.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples\n",
      "Epoch 1/3\n",
      "60000/60000 [==============================] - 29s 481us/sample - loss: 0.5932 - acc: 0.7819\n",
      "Epoch 2/3\n",
      "60000/60000 [==============================] - 33s 542us/sample - loss: 0.4066 - acc: 0.8506\n",
      "Epoch 3/3\n",
      "60000/60000 [==============================] - 32s 525us/sample - loss: 0.3624 - acc: 0.8681\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fae6dd5cb70>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnn.fit(x_train, y_train, batch_size=64, epochs=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy:  0.8867\n"
     ]
    }
   ],
   "source": [
    "# Evaluate the model on test set\n",
    "score = cnn.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test accuracy: ', score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define superpixels\n",
    "\n",
    "Function to generate rectangular superpixels for a given image. Alternatively, use one of the built in methods. It is important to have meaningful superpixels in order to generate a useful explanation. Please check scikit-image's [segmentation methods](http://scikit-image.org/docs/dev/api/skimage.segmentation.html) (*felzenszwalb*, *slic* and *quickshift* built in the explainer) for more information on the built in methods."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def superpixel(image, size=(4, 7)):\n",
    "    segments = np.zeros([image.shape[0], image.shape[1]])\n",
    "    row_idx, col_idx = np.where(segments == 0)\n",
    "    for i, j in zip(row_idx, col_idx):\n",
    "        segments[i, j] = int((image.shape[1]/size[1]) * (i//size[0]) + j//size[1])\n",
    "    return segments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAALdUlEQVR4nO3dXYhc9RnH8d8va5KNm7RNtEljDNXa0KKFxrKkBaVYrBK9id6IubApCPFCQcGLir2ol6FUpRetEGswLVYpqJiL0JoGIUhFXCXNi7bGSsSEmK3Elxhtkt08vdgTWeNOdpzzNu7z/cAyM+fM5jw72e/OK/wdEQIw881qewAAzSB2IAliB5IgdiAJYgeSOKfJg83x3BjUUJOHbIRnzeC/mQMz9GebNdD2BLX45OQHOjH+safaVyp226sl/VbSgKQ/RMSGs11/UEP6oa8uc8i+NGv+grZHqM2s+TPvj7MkxYKZ+XO9sH9zx309/9m2PSDpd5Kuk3SppLW2L+313wNQrzKP0VZJeiMi3oyIE5KekLSmmrEAVK1M7MskvT3p8oFi22fYXm97xPbISR0vcTgAZdT+6ktEbIyI4YgYnq25dR8OQAdlYj8oafmkyxcW2wD0oTKxvyRphe2Lbc+RdLOkLdWMBaBqPb/1FhFjtu+Q9DdNvPW2KSL2VjYZgEqVep89IrZK2lrRLABqNEM/HgXgTMQOJEHsQBLEDiRB7EASxA4kQexAEsQOJEHsQBLEDiRB7EASxA4kQexAEsQOJEHsQBLEDiRB7EASxA4kQexAEsQOJEHsQBKNLtmscwfl717W6CGbMDZ/Ttsj1GZsqNlfkaacHJqZ93Pjo53/v2bmTwzgc4gdSILYgSSIHUiC2IEkiB1IgtiBJIgdSKLUJyZs75d0VNK4pLGIGK5iKADVq+LjUT+JiHcr+HcA1IiH8UASZWMPSc/aftn2+qmuYHu97RHbIyfHPi55OAC9Kvsw/sqIOGh7saRttv8VETsmXyEiNkraKElfGbogSh4PQI9K3bNHxMHidFTS05JWVTEUgOr1HLvtIdsLTp+XdK2kPVUNBqBaZR7GL5H0tO3T/86fI+KvlUwFoHI9xx4Rb0r6foWzAKgRb70BSRA7kASxA0kQO5AEsQNJEDuQBLEDSRA7kASxA0kQO5AEsQNJEDuQBLEDSTS6Hu/44IDev2xBk4dsxMlz3fYItRmbNzN/trFz256gHuPPd97HPTuQBLEDSRA7kASxA0kQO5AEsQNJEDuQBLEDSRA7kASxA0kQO5AEsQNJEDuQBLEDSRA7kASxA0lMG7vtTbZHbe+ZtG2R7W229xWnC+sdE0BZ3dyzPypp9Rnb7pG0PSJWSNpeXAbQx6aNPSJ2SDpyxuY1kjYX5zdLuqHiuQBUrNfn7Esi4lBx/h1JSzpd0fZ62yO2R8aOH+vxcADKKv0CXUSEpDjL/o0RMRwRw+fMHSp7OAA96jX2w7aXSlJxOlrdSADq0GvsWyStK86vk/RMNeMAqEs3b709LukFSd+xfcD2rZI2SLrG9j5JPy0uA+hj0y4SERFrO+y6uuJZANSIT9ABSRA7kASxA0kQO5AEsQNJNLxks/T+ipn392V8XscPEH7pjc8bb3uEWsQM/blOze38uzjzygMwJWIHkiB2IAliB5IgdiAJYgeSIHYgCWIHkiB2IAliB5IgdiAJYgeSIHYgCWIHkiB2IAliB5IgdiAJYgeSIHYgCWIHkiB2IAliB5IgdiAJYgeS6GZ99k22R23vmbTtPtsHbe8svq6vd0wAZXVzz/6opNVTbH8wIlYWX1urHQtA1aaNPSJ2SDrSwCwAalTmOfsdtncVD/MXdrqS7fW2R2yPjB87VuJwAMroNfaHJF0iaaWkQ5Lu73TFiNgYEcMRMTwwNNTj4QCU1VPsEXE4IsYj4pSkhyWtqnYsAFXrKXbbSyddvFHSnk7XBdAfpl2f3fbjkq6SdL7tA5J+Jekq2yslhaT9km7r5mAx95TGvv1Jz8P2q8F5J9oeoTbzB4+3PUItvjY4834PJem9uSc77ps29ohYO8XmR8oMBKB5fIIOSILYgSSIHUiC2IEkiB1IgtiBJIgdSILYgSSIHUiC2IEkiB1IgtiBJIgdSILYgSSIHUiC2IEkiB1IgtiBJIgdSILYgSSIHUiC2IEkiB1IgtiBJIgdSILYgSSIHUiC2IEkiB1IYtpVXKs0NPeEVl20v8lDNmLx4NG2R6jN4tkz82e7YM57bY9Qi9dnH+u4b9p7dtvLbT9n+1Xbe23fWWxfZHub7X3F6cIKZwZQsW4exo9JujsiLpX0I0m3275U0j2StkfECknbi8sA+tS0sUfEoYh4pTh/VNJrkpZJWiNpc3G1zZJuqGtIAOV9oRfobF8k6XJJL0paEhGHil3vSFrS4XvW2x6xPXL8vf+VGBVAGV3Hbnu+pCcl3RURH07eFxEhKab6vojYGBHDETE8d+FgqWEB9K6r2G3P1kToj0XEU8Xmw7aXFvuXShqtZ0QAVejm1XhLekTSaxHxwKRdWyStK86vk/RM9eMBqEo377NfIekWSbtt7yy23Stpg6S/2L5V0luSbqpnRABVmDb2iHhekjvsvrracQDUhY/LAkkQO5AEsQNJEDuQBLEDSRA7kASxA0kQO5AEsQNJEDuQBLEDSRA7kASxA0kQO5AEsQNJEDuQBLEDSRA7kASxA0kQO5AEsQNJNLpk83nnfKSfLflHk4dsxAXnfND2CLX5xsB42yPUYvHAUNsj1OL3A2Md93HPDiRB7EASxA4kQexAEsQOJEHsQBLEDiTRzfrsy20/Z/tV23tt31lsv8/2Qds7i6/r6x8XQK+6+VDNmKS7I+IV2wskvWx7W7HvwYj4TX3jAahKN+uzH5J0qDh/1PZrkpbVPRiAan2h5+y2L5J0uaQXi0132N5le5PthR2+Z73tEdsjHxyZmR+9BL4Muo7d9nxJT0q6KyI+lPSQpEskrdTEPf/9U31fRGyMiOGIGP7qooEKRgbQi65itz1bE6E/FhFPSVJEHI6I8Yg4JelhSavqGxNAWd28Gm9Jj0h6LSIemLR96aSr3ShpT/XjAahKN6/GXyHpFkm7be8stt0raa3tlZJC0n5Jt9UyIYBKdPNq/POSPMWurdWPA6AufIIOSILYgSSIHUiC2IEkiB1IgtiBJIgdSILYgSSIHUiC2IEkiB1IgtiBJIgdSILYgSQcEc0dzP6vpLcmbTpf0ruNDfDF9Ots/TqXxGy9qnK2b0bE16fa0Wjsnzu4PRIRw60NcBb9Olu/ziUxW6+amo2H8UASxA4k0XbsG1s+/tn062z9OpfEbL1qZLZWn7MDaE7b9+wAGkLsQBKtxG57te1/237D9j1tzNCJ7f22dxfLUI+0PMsm26O290zatsj2Ntv7itMp19hraba+WMb7LMuMt3rbtb38eePP2W0PSHpd0jWSDkh6SdLaiHi10UE6sL1f0nBEtP4BDNs/lvSRpD9GxPeKbb+WdCQiNhR/KBdGxC/6ZLb7JH3U9jLexWpFSycvMy7pBkk/V4u33VnmukkN3G5t3LOvkvRGRLwZESckPSFpTQtz9L2I2CHpyBmb10jaXJzfrIlflsZ1mK0vRMShiHilOH9U0ullxlu97c4yVyPaiH2ZpLcnXT6g/lrvPSQ9a/tl2+vbHmYKSyLiUHH+HUlL2hxmCtMu492kM5YZ75vbrpflz8viBbrPuzIifiDpOkm3Fw9X+1JMPAfrp/dOu1rGuylTLDP+qTZvu16XPy+rjdgPSlo+6fKFxba+EBEHi9NRSU+r/5aiPnx6Bd3idLTleT7VT8t4T7XMuPrgtmtz+fM2Yn9J0grbF9ueI+lmSVtamONzbA8VL5zI9pCka9V/S1FvkbSuOL9O0jMtzvIZ/bKMd6dlxtXybdf68ucR0fiXpOs18Yr8fyT9so0ZOsz1LUn/LL72tj2bpMc18bDupCZe27hV0nmStkvaJ+nvkhb10Wx/krRb0i5NhLW0pdmu1MRD9F2SdhZf17d9251lrkZuNz4uCyTBC3RAEsQOJEHsQBLEDiRB7EASxA4kQexAEv8HaVx2v2KULksAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "segments = superpixel(x_train[idx])\n",
    "plt.imshow(segments);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define prediction function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "predict_fn = lambda x: cnn.predict(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize anchor image explainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_shape = x_train[idx].shape\n",
    "explainer = AnchorImage(predict_fn, image_shape, segmentation_fn=superpixel)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Explain a prediction\n",
    "\n",
    "The explanation returns a mask with the superpixels that constitute the anchor.\n",
    "\n",
    "Image to be explained:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAATxUlEQVR4nO3de3Bc5XkG8OddaXWxLCPLMkJgcbGxIYQQQ1UTLmWgNMShUy6dlluTQofGbgcKaTItDEnH/NMp7YQkpGkg4hKcDrWHaUKgGUMhnrQOaWMsqDE2Bt/AwbItC4wvsrzWavftHzowCui8n7xnd8/C+/xmPJL33bP76diPzu6+53yfqCqI6OMvk/YAiKg6GHYiJxh2IicYdiInGHYiJ+qr+WQN0qhNaKnmU34sSH2dWS+0NsXWMu8eKvdwjk7rlPhaoWhvO5wr71gcyOEQRvSITFRLFHYRWQjgPgB1AB5S1Xus+zehBefKpUme0qW6tnazfuCSubG1ln9fXe7hHJXCb58TW6s/cMTcVl/cUO7hfOyt1pWxtZJfxotIHYB/AfB5AGcAuF5Ezij18YiospK8Z18AYIuqblPVEQDLAVxZnmERUbklCfsJAN4a9/cd0W2/QUQWiUifiPTlYb9sI6LKqfin8araq6o9qtqTRWOln46IYiQJez+A7nF/nxXdRkQ1KEnY1wCYKyKniEgDgOsAPFWeYRFRuZXcelPVURG5FcB/Yqz19oiquuyVZFrscwe2/t1ZZv3m3/+ZWT+z+TWzfm7jf8TWdn7D7tGf1RDfoy+Htwu/jK0NFOxjTU7tsd/2+nVmvbj02NjatGW/Mrf9OErUZ1fVFQBWlGksRFRBPF2WyAmGncgJhp3ICYadyAmGncgJhp3ICanm7LLTpF0/qpe4bnpgQWxtxcJvm9vOzmbN+kDBvmZgd8E+zfhgMb5XflzdkLntMZmCWW+QCS+Nft++wCXpO0dbY2tZGTW3bc/Y17MfZ7fh0SjxneXb+y8xt/31uSnPA1Ci1boSB3TvhP9oPLITOcGwEznBsBM5wbATOcGwEznBsBM5UdWppGtZ/x3nm/U3rvhebG1VzpguGcBbh+3WWxFTzXoGdn9rmtGiGizYl98O2p03FGC33gpqHy9aMqVPRTZYtPfr9lG7JZnT+P3+3Vn/ZW57xcqrzTou3WHXaxCP7EROMOxETjDsRE4w7EROMOxETjDsRE4w7EROsM8eeWjxP5v1rfnDsbW8HmNu25TJm/WLEs7mvGFkJLY2UrSvAx0u2r3q7vp9Zn1mnX0OwNojbbG1BrGb/FafHADaA5fv1iH+8u3nc83mtt87dblZv23WtWZ9dEftrZfCIzuREww7kRMMO5ETDDuREww7kRMMO5ETDDuRE+yzR07L2tdd7zXaydlAvzjUR5+z8s/M+uxee/ufLo+/Q3/gWvqFU+yf+428/bP9ZGieWb+geWtsbV+gx39xs93Df3bYvt59sDAttja3Ybe5bWedHY3DZ3SZ9WwN9tkThV1E3gRwEEABwKiq9pRjUERUfuU4sl+iqm+X4XGIqIL4np3IiaRhVwDPisiLIrJoojuIyCIR6RORvjxKn4+MiJJJ+jL+QlXtF5FjATwnIq+p6qrxd1DVXgC9wNhabwmfj4hKlOjIrqr90dc9AJ4AEL/6IRGlquSwi0iLiLS+9z2AywCsL9fAiKi8kryM7wTwhIwt6VsP4N9U9ZmyjCoF0+sCPdti/BK+dYF53UO/U0/7ij0HeWFw0Kw3Snwv/bj6g+a2f7r9MrM+cN4Bsx6SfzX+evpb2t4yt738U79r1jffcZpd/8L9sbUXAh8fZcWeB2Dnhfb5Cyc9az9+GkoOu6puA/DpMo6FiCqIrTciJxh2IicYdiInGHYiJxh2IifcXOKaaUo2X3PeWJq43VgyeYzd1juyzJ7WuP73Ag9vOKvB/rlDrbXN933GrGcP2ks6/2Rx/L5ZPrPB3LZ5nr1f5ywLtAW/EF9qCLRLc2rXs5/abz93DeKRncgJhp3ICYadyAmGncgJhp3ICYadyAmGncgJN312mXNS4B6/MqtWn72zzl6SOeS8jjfM+hrYl1taepb8pVmfgf816/MetS+RzRwKnGNQHz/2zC/+z9509slmXfcnu/w2iUtP3GTWN1ZpHEeDR3YiJxh2IicYdiInGHYiJxh2IicYdiInGHYiJ9z02XNdUyv22K0ZezcOFe1e9GXTXjHrazK/ddRjek/nM/Z0zaOB7W9avsKsX9f6rllfeyR+zuavLL7F3PbRh75t1v9hzyVm/dejQ7G10FTRw0V7qerfaQ312Web9TTwyE7kBMNO5ATDTuQEw07kBMNO5ATDTuQEw07khJs++8Fue47ykIxoydvuLNg924sCU9r/faDn+7nj58fWpKfN3Hb7vdPN+g/sVZHxA9jzBFz9avxy0+98wv43+fPzrzXrr/91t1n/zvVrYmvrRuxzH/YV7ePg56bsMeu9H8U+u4g8IiJ7RGT9uNvaReQ5EdkcfbX/xxBR6ibzMv5RAAs/cNudAFaq6lwAK6O/E1ENC4ZdVVcB2PuBm68EsDT6fimAq8o8LiIqs1Lfs3eq6q7o+90AOuPuKCKLACwCgKbAmmdEVDmJP41XVQUQ++mVqvaqao+q9mTRmPTpiKhEpYZ9QES6ACD6an80SUSpKzXsTwG4Mfr+RgBPlmc4RFQpwffsIrIMwMUAOkRkB4AlAO4B8LiI3AxgO4BrKjnIcsjNtNcRD7HmjW8MXBs9Reyrxq3rrgFg83fPNetaH38OwJfO/29z22c6Xjfrf/PS2Wb95Ka3zfpftPXH1k6/7QFz23980F4b/vgzSz93oknscxesf28AmJoJnBxRg4JhV9XrY0qXlnksRFRBPF2WyAmGncgJhp3ICYadyAmGncgJN5e4Hu4sJto+r/HttdC0xC1i/059PW+fWbjtD79v1i2b8ofM+i9zzWb9rzp+UfJzA8CqXPwU3gsa7ctMn97yP4meu6Dx/+ZNgUuW86Vf0QwAkHo7WjoamsS7/HhkJ3KCYSdygmEncoJhJ3KCYSdygmEncoJhJ3LCTZ+92DFSscfeXzxs1v9kyx+Z9QfmPG7WnxmeYdZzmo2ttWXs3+dTMvFLKgPAtvw0sx7SmonvpT+fazG3nVFnnyOwNT/TrG/KdcXWvt7xmrmttdT0ZMgn55p1fXljoscvBY/sRE4w7EROMOxETjDsRE4w7EROMOxETjDsRE646bNPPcbuhYecVB+//dOH7KWDB5bbyxqfuCT+mm8A2Dk6bNYt2cCUyXXxi/mMCfThQwqIn8K7JfDY7Rn73IhD9fvN+l3Pxk2MDHz9BrvPnlTuOPscgoaXK/r0E+KRncgJhp3ICYadyAmGncgJhp3ICYadyAmGncgJN332WcfYPVlrjnEA6KqP74WvGTrF3Lbp3WSTkB8o2ssDW/3qjNHnroaisfRxU2Ap69BM/23GtfIAcOwao3iD/djW+QEAsKdgX2uvmXT3+0SCR3YReURE9ojI+nG33S0i/SKyNvpzeWWHSURJTeZl/KMAFk5w+7dUdX70Z0V5h0VE5RYMu6quArC3CmMhogpK8gHdrSKyLnqZPz3uTiKySET6RKQvj2TnWRNR6UoN+/0A5gCYD2AXgHvj7qiqvarao6o9WdgLGBJR5ZQUdlUdUNWCqhYBPAhgQXmHRUTlVlLYRWT8HL1XA1gfd18iqg3BPruILANwMYAOEdkBYAmAi0VkPgAF8CaAxRUcY1nMnvqOWX83MPd7R1389cn9uTZz272nJzt3aVjttz/TYPebLaF+clIZie+Wh547VP9ENn6+fAAILMFuCl3nnw2M7fBMO1ppvKENhl1VJ5oB4OEKjIWIKoinyxI5wbATOcGwEznBsBM5wbATOeHmEtfGTN6shy6ntKzZZk8VXTwl4XTMxmWigD1ddKh9FZxKOiHr+ZsC01zvLdiX9s7L1pn1KbtK3++NgbFlJNR6s+t2s7YyeGQncoJhJ3KCYSdygmEncoJhJ3KCYSdygmEncsJNn725zu6z57T0fnPDlmazPuO83SU/NhBe2tgS6qOH6kkvgbUePxs4u+GQNgQe3e6FN2wbiK09M2xfZHpOoz1VNAL7JW+v2JwKHtmJnGDYiZxg2ImcYNiJnGDYiZxg2ImcYNiJnHDTZ98baHzmtPR+sjFbMgDg2u4XzfpQ0Z4KOiv2ddtpygZ++KKxX/OBY01O7amiQ3324TOPj62tOniaue1FTX1mfX9xxKwXplR2noBS8MhO5ATDTuQEw07kBMNO5ATDTuQEw07kBMNO5ISbPvvhgt2zbUqwvm8xa297TvMbZn1nwe4XN4l9LX4lha5nD3XCLfnAfPhJf+7tV8Sfn5DbPdfcdsmx9rkR9r8YkG8L3aP6gkd2EekWkZ+LyKsiskFEbo9ubxeR50Rkc/R1euWHS0SlmszL+FEAX1XVMwB8BsAtInIGgDsBrFTVuQBWRn8nohoVDLuq7lLVl6LvDwLYCOAEAFcCWBrdbSmAqyo1SCJK7qjes4vIyQDOBrAaQKeq7opKuwF0xmyzCMAiAGjClFLHSUQJTfrTeBGZCuBHAL6sqgfG11RVgYlnFlTVXlXtUdWeLOxJ/oiociYVdhHJYizoj6nqj6ObB0SkK6p3AdhTmSESUTkEX8aLiAB4GMBGVf3muNJTAG4EcE/09cmKjLBMjhTsH7UjE5q2OF5x7rBZbwtMBR1amrgl0IIaMX5nJ12SOelU1MUEU1GHW2/2saqte19sbXDDTHPbxk/bTcUiAtN71ydZBLwyJvOe/QIAXwTwioisjW67C2Mhf1xEbgawHcA1lRkiEZVDMOyq+jziZ8S/tLzDIaJK4emyRE4w7EROMOxETjDsRE4w7EROuLnEdWjUPnuvTkrvB89oGzLrnXV2z3Vf0X5uq48ekld7GupQJzt0iWuoXjQuY80EpqEO9fA35e1llb92+tOxtb/deoO5bUghcPpCXfNH8BJXIvp4YNiJnGDYiZxg2ImcYNiJnGDYiZxg2ImccNNnPzxqX588ULCvTz6xPn77xu+02499v/079bg6+3r4XKBXbgqcPhDuk9v1TGgKbonvNzcZNSD8c8+pbzbrizddEls7+aeBMwyutcu5wDTY9dlR+wFSwCM7kRMMO5ETDDuREww7kRMMO5ETDDuREww7kRNu+uwzmuxrn3OBfvJQMRdbKzbY267JnWTWb5pmr6/x2MEZZj0rlevpJp533rhmfSTQRx8u2nMQnNVg77f+t9tia6futucgCDkSGPv8E/rN+ruJnr00PLITOcGwEznBsBM5wbATOcGwEznBsBM5wbATOTGZ9dm7AfwQQCcABdCrqveJyN0AvgRgMLrrXaq6olIDTeqFvnlmvbXb7icPFuJ72a3rBsxtl51+vF2HXaeJhfbbKXg5tqZnnW5u+0be7sN3BKYYWP3yqWZ9Hl6wH6ACJnNSzSiAr6rqSyLSCuBFEXkuqn1LVb9RueERUblMZn32XQB2Rd8fFJGNAE6o9MCIqLyO6j27iJwM4GwAq6ObbhWRdSLyiIhMj9lmkYj0iUhfHvbUT0RUOZMOu4hMBfAjAF9W1QMA7gcwB8B8jB35751oO1XtVdUeVe3Jwj7XmYgqZ1JhF5EsxoL+mKr+GABUdUBVC6paBPAggAWVGyYRJRUMu4gIgIcBbFTVb467vWvc3a4GsL78wyOicpnMp/EXAPgigFdEZG10210ArheR+Rhrx70JYHFFRlgmM/vsy1C7/niqWd9fPBxfLNpLD1Pt0Qb7v357nd1bOyZjT2NdP5Rg+u8Kmcyn8c9j4tnHa7anTkQfxjPoiJxg2ImcYNiJnGDYiZxg2ImcYNiJnHAzlXTrW/Z5+UsGP2nW3xmJ78Pr/gMljek9km0w6zoaWF5YfP7Olox97oSOGlNsr33N3PYPNtxg1mdN3WfWO1+ovXMvfP4vIXKIYSdygmEncoJhJ3KCYSdygmEncoJhJ3JCVJMtyXtUTyYyCGD7uJs6ALxdtQEcnVodW62OC+DYSlXOsZ2kqjMnKlQ17B96cpE+Ve1JbQCGWh1brY4L4NhKVa2x8WU8kRMMO5ETaYe9N+Xnt9Tq2Gp1XADHVqqqjC3V9+xEVD1pH9mJqEoYdiInUgm7iCwUkddFZIuI3JnGGOKIyJsi8oqIrBWRvpTH8oiI7BGR9eNuaxeR50Rkc/R1wjX2Uhrb3SLSH+27tSJyeUpj6xaRn4vIqyKyQURuj25Pdd8Z46rKfqv6e3YRqQOwCcBnAewAsAbA9ar6alUHEkNE3gTQo6qpn4AhIhcBGALwQ1U9M7rtnwDsVdV7ol+U01X1jhoZ290AhtJexjtarahr/DLjAK4CcBNS3HfGuK5BFfZbGkf2BQC2qOo2VR0BsBzAlSmMo+ap6ioAez9w85UAlkbfL8XYf5aqixlbTVDVXar6UvT9QQDvLTOe6r4zxlUVaYT9BABvjfv7DtTWeu8K4FkReVFEFqU9mAl0ququ6PvdADrTHMwEgst4V9MHlhmvmX1XyvLnSfEDug+7UFXPAfB5ALdEL1drko69B6ul3umklvGulgmWGX9fmvuu1OXPk0oj7P0Ausf9fVZ0W01Q1f7o6x4AT6D2lqIeeG8F3ejrnpTH875aWsZ7omXGUQP7Ls3lz9MI+xoAc0XkFBFpAHAdgKdSGMeHiEhL9MEJRKQFwGWovaWonwJwY/T9jQCeTHEsv6FWlvGOW2YcKe+71Jc/V9Wq/wFwOcY+kd8K4GtpjCFmXLMBvBz92ZD22AAsw9jLujzGPtu4GcAMACsBbAbwMwDtNTS2fwXwCoB1GAtWV0pjuxBjL9HXAVgb/bk87X1njKsq+42nyxI5wQ/oiJxg2ImcYNiJnGDYiZxg2ImcYNiJnGDYiZz4f4SixgHCORlKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "i = 1\n",
    "image = x_test[i]\n",
    "plt.imshow(image[:,:,0]);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model prediction:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnn.predict(image.reshape(1, 28, 28, 1)).argmax()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The predicted category correctly corresponds to the class `Pullover`:\n",
    "\n",
    "| Label | Description |\n",
    "| --- | --- |\n",
    "| 0 | T-shirt/top |\n",
    "| 1 | Trouser |\n",
    "| 2 | Pullover |\n",
    "| 3 | Dress |\n",
    "| 4 | Coat |\n",
    "| 5 | Sandal |\n",
    "| 6 | Shirt |\n",
    "| 7 | Sneaker |\n",
    "| 8 | Bag |\n",
    "| 9 | Ankle boot |"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generate explanation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "explanation = explainer.explain(image, threshold=.95, p_sample=.8, seed=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Show anchor:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAK8UlEQVR4nO3dX6jf9X3H8eerMcY1dsPMLQSVtetswQ1mx8ENlNHhWlQY2ptSCyUDWXpRoYVeTNzFvJSxtvRiFNIpTUdnN2jFXIStWShIYViPkmrUrnHO0mQxaXGjOliM+t7F+VpO9ZzfOf7+r+/nAw6/3+/7/Z7zffMjT39/8ZOqQtIvvncsegBJ82HsUhPGLjVh7FITxi41cdE8T3ZxdtUl7J7nKaVW/pf/4ZU6n432TRR7kpuALwI7gL+tqntHHX8Ju/n93DjJKSWN8Egd23Tf2E/jk+wA/ga4GbgGuD3JNeP+PUmzNclr9uuAZ6vquap6Bfg6cOt0xpI0bZPEfgXwo3W3Tw3bfk6SA0lWk6xe4PwEp5M0iZm/G19VB6tqpapWdrJr1qeTtIlJYj8NXLXu9pXDNklLaJLYHwWuTvKeJBcDHwMOT2csSdM29kdvVfVqkjuBf2bto7f7q+qpqU0maaom+py9qo4AR6Y0i6QZ8uuyUhPGLjVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41YexSE8YuNTHRks1JngdeAl4DXq2qlWkMJWn6Jop98EdV9ZMp/B1JM+TTeKmJSWMv4FtJHktyYKMDkhxIsppk9QLnJzydpHFN+jT+hqo6neTXgaNJvl9VD68/oKoOAgcBfjl7asLzSRrTRI/sVXV6uDwHPAhcN42hJE3f2LEn2Z3kXW9cBz4MnJjWYJKma5Kn8XuBB5O88Xf+vqr+aSpTSZq6sWOvqueA353iLJJmyI/epCaMXWrC2KUmjF1qwtilJoxdasLYpSaMXWrC2KUmjF1qwtilJoxdasLYpSaMXWrC2KUmjF1qwtilJoxdasLYpSaMXWrC2KUmjF1qwtilJoxdasLYpSaMXWrC2KUmjF1qwtilJoxdamLL2JPcn+RckhPrtu1JcjTJyeHystmOKWlS23lk/wpw05u23QUcq6qrgWPDbUlLbMvYq+ph4MU3bb4VODRcPwTcNuW5JE3ZRWP+3t6qOjNcfwHYu9mBSQ4ABwAu4Z1jnk7SpCZ+g66qCqgR+w9W1UpVrexk16SnkzSmcWM/m2QfwHB5bnojSZqFcWM/DOwfru8HHprOOJJmZTsfvT0A/Cvw/iSnktwB3At8KMlJ4I+H25KW2JZv0FXV7ZvsunHKs0iaIb9BJzVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41YexSE8YuNWHsUhPbWZ/9/iTnkpxYt+2eJKeTHB9+bpntmJImtZ1H9q8AN22w/QtVde3wc2S6Y0mati1jr6qHgRfnMIukGZrkNfudSZ4YnuZfttlBSQ4kWU2yeoHzE5xO0iTGjf1LwHuBa4EzwOc2O7CqDlbVSlWt7GTXmKeTNKmxYq+qs1X1WlW9DnwZuG66Y0matrFiT7Jv3c2PACc2O1bScrhoqwOSPAB8ELg8ySngL4EPJrkWKOB54JMznFHSFGwZe1XdvsHm+2Ywi6QZ8ht0UhPGLjVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41YexSE8YuNbFl7EmuSvLtJE8neSrJp4fte5IcTXJyuLxs9uNKGtd2HtlfBT5bVdcAfwB8Ksk1wF3Asaq6Gjg23Ja0pLaMvarOVNXjw/WXgGeAK4BbgUPDYYeA22Y1pKTJXfR2Dk7ybuADwCPA3qo6M+x6Adi7ye8cAA4AXMI7x51T0oS2/QZdkkuBbwCfqaqfrt9XVQXURr9XVQeraqWqVnaya6JhJY1vW7En2cla6F+rqm8Om88m2Tfs3wecm82IkqZhO+/GB7gPeKaqPr9u12Fg/3B9P/DQ9MeTNC3bec1+PfAJ4Mkkx4dtdwP3Av+Y5A7gh8BHZzOipGnYMvaq+g6QTXbfON1xJM2K36CTmjB2qQljl5owdqkJY5eaMHapCWOXmjB2qQljl5owdqkJY5eaMHapCWOXmjB2qQljl5owdqkJY5eaMHapCWOXmjB2qQljl5p4W8s/Sb8odvz2+0fuP3L0H0buP37+/Mj99/7nzSP3/9f1L47cPws+sktNGLvUhLFLTRi71ISxS00Yu9SEsUtNbPk5e5KrgK8Ce4ECDlbVF5PcA/wZ8OPh0Lur6sisBpWmqXZstjDxmv+48PLI/ZfvGP33H/neb43c/z6+O/oPzMB2vlTzKvDZqno8ybuAx5IcHfZ9oar+enbjSZqW7azPfgY4M1x/KckzwBWzHkzSdL2t1+xJ3g18AHhk2HRnkieS3J/ksk1+50CS1SSrFxj9FUNJs7Pt2JNcCnwD+ExV/RT4EvBe4FrWHvk/t9HvVdXBqlqpqpWd7JrCyJLGsa3Yk+xkLfSvVdU3AarqbFW9VlWvA18GrpvdmJImtWXsSQLcBzxTVZ9ft33fusM+ApyY/niSpmU778ZfD3wCeDLJ8WHb3cDtSa5l7eO454FPzmRCaQbq4tH/9PfsGP3Z2q+845dG7r/o5S0+m1uA7bwb/x1gow8l/Uxd+n/Eb9BJTRi71ISxS00Yu9SEsUtNGLvUhP8rafV0/Psjd//JUx8fuf/KS/975P693339bY80az6yS00Yu9SEsUtNGLvUhLFLTRi71ISxS02kquZ3suTHwA/Xbboc+MncBnh7lnW2ZZ0LnG1c05ztN6rq1zbaMdfY33LyZLWqVhY2wAjLOtuyzgXONq55zebTeKkJY5eaWHTsBxd8/lGWdbZlnQucbVxzmW2hr9klzc+iH9klzYmxS00sJPYkNyX5tyTPJrlrETNsJsnzSZ5McjzJ6oJnuT/JuSQn1m3bk+RokpPD5YZr7C1otnuSnB7uu+NJblnQbFcl+XaSp5M8leTTw/aF3ncj5prL/Tb31+xJdgA/AD4EnAIeBW6vqqfnOsgmkjwPrFTVwr+AkeQPgZeBr1bV7wzb/gp4saruHf5DeVlV/fmSzHYP8PKil/EeVivat36ZceA24E9Z4H03Yq6PMof7bRGP7NcBz1bVc1X1CvB14NYFzLH0quph4MU3bb4VODRcP8TaP5a522S2pVBVZ6rq8eH6S8Aby4wv9L4bMddcLCL2K4Afrbt9iuVa772AbyV5LMmBRQ+zgb1VdWa4/gKwd5HDbGDLZbzn6U3LjC/NfTfO8ueT8g26t7qhqn4PuBn41PB0dSnV2muwZfrsdFvLeM/LBsuM/8wi77txlz+f1CJiPw1cte72lcO2pVBVp4fLc8CDLN9S1GffWEF3uDy34Hl+ZpmW8d5omXGW4L5b5PLni4j9UeDqJO9JcjHwMeDwAuZ4iyS7hzdOSLIb+DDLtxT1YWD/cH0/8NACZ/k5y7KM92bLjLPg+27hy59X1dx/gFtYe0f+34G/WMQMm8z1m8D3hp+nFj0b8ABrT+susPbexh3ArwLHgJPAvwB7lmi2vwOeBJ5gLax9C5rtBtaeoj8BHB9+bln0fTdirrncb35dVmrCN+ikJoxdasLYpSaMXWrC2KUmjF1qwtilJv4PxpFZt10mFtcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(explanation.anchor[:,:,0]);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the example, it looks like the end of the sleeve alone is sufficient to predict a pullover."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}