3. CIFAR-10 Training Dataset Occlusion.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, collections\n",
    "\n",
    "if not os.path.exists('CIFAR10_data'):\n",
    "    \n",
    "    !wget https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz\n",
    "    !mkdir CIFAR10_data\n",
    "    !tar -xf cifar-10-python.tar.gz -C CIFAR10_data\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "from utils import unpickle, preprocess, batch_run, pixel_range\n",
    "from deepexplain.tensorflow import DeepExplain\n",
    "from models import CIFAR_CNN\n",
    "from trainer import Trainer\n",
    "\n",
    "datadir = 'CIFAR10_data/cifar-10-batches-py/'\n",
    "batches_train = sorted([datadir + batch for batch in os.listdir(datadir) if 'data_batch' in batch], key=lambda x: int(x[-1]))\n",
    "batch_test = datadir + 'test_batch'\n",
    "\n",
    "logdir = 'tf_logs/standard/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5/5 [00:02<00:00,  2.35it/s]\n"
     ]
    }
   ],
   "source": [
    "for i in tqdm(range(5)):\n",
    "    \n",
    "    batch = unpickle(batches_train[i])\n",
    "\n",
    "    if i == 0:\n",
    "        data = batch[b'data'].astype(np.float32)\n",
    "        cifar = np.transpose(np.reshape(data, [-1,3,32,32]), [0,2,3,1])\n",
    "        labels = batch[b'labels']\n",
    "    else:\n",
    "        data = batch[b'data'].astype(np.float32)\n",
    "        cifar = np.concatenate((cifar, np.transpose(np.reshape(data, [-1,3,32,32]), [0,2,3,1])), axis=0)\n",
    "        labels = np.concatenate((labels, batch[b'labels']), axis=0)\n",
    "\n",
    "test_batch = unpickle(batch_test)\n",
    "cifar_test = np.transpose(np.reshape(test_batch[b'data'], [-1,3,32,32]), [0,2,3,1])\n",
    "labels_test = np.array(test_batch[b'labels'])\n",
    "\n",
    "# Set random seed so I don't have to retrain every time\n",
    "np.random.seed(100)\n",
    "random_mask = np.random.randint(low=0, high=255, size=[1,10,10,3])\n",
    "cifar[:,:10,:10,:] = random_mask\n",
    "cifar_test[:,:10,:10,:] = random_mask\n",
    "\n",
    "data_train = (cifar / 127.5 - 1.0, labels)\n",
    "data_test = (cifar_test / 127.5 - 1.0, labels_test)\n",
    "\n",
    "label_map = {0:'Airplane', 1:'Automobile', 2:'Bird', 3:'Cat', 4:'Deer', 5:'Dog', 6:'Frog', 7:'Horse', 8:'Ship', 9:'Truck'}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training with Occluded Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 5   | Loss : 0.44810 | Train Accuracy : 0.84166\n",
      "Epoch : 10  | Loss : 0.13555 | Train Accuracy : 0.95238\n",
      "Epoch : 15  | Loss : 0.06766 | Train Accuracy : 0.97640\n",
      "Epoch : 20  | Loss : 0.05823 | Train Accuracy : 0.98040\n",
      "Test Accuracy : 0.72050\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "DNN = CIFAR_CNN(logdir, 'CNN')\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "trainer = Trainer(sess, DNN, data_train)\n",
    "trainer.train(n_epochs=20, p_epochs=5)\n",
    "\n",
    "test_acc = DNN.evaluate(sess, data_test)\n",
    "print('Test Accuracy : {:.5f}'.format(test_acc))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comparing Amount of Attribution in Random Patch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from tf_logs/standard/model\n",
      "Running RectGrad explanation method\n",
      "Running RectGrad PRR explanation method\n",
      "Running Saliency Map explanation method\n",
      "Running Guided BP explanation method\n",
      "Running SmoothGrad explanation method\n",
      "Running Gradient * Input explanation method\n",
      "Running IntegGrad explanation method\n",
      "Running DeepLIFT explanation method\n",
      "Done!\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAFDCAYAAAAqHPVGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztnXm4ndP1xz8rAzG7VyJCJEGJmWhKEWPMs1Khhih+lCqtsUqJtuaGFlVNK+apZi2tqbRVY9ASUjU0IcaoKIJoWL8/1j7u67g399x7zznvue/9fp7nPPe8wz3veqfvXnvttfc2d0cIIUT3p1feBgghhKgOEnQhhCgIEnQhhCgIEnQhhCgIEnQhhCgIEnQhhCgIEvQegpltbGZuZv3T8r5m9n4NjzfVzI6q1e9njuNmtmutj1NLzOwoM5uatx31xMzuM7Pz87ajaEjQGwQzW8vMPjGzv7WybVgSrpFl6y8xs99XeIgHgEHAf6pgbtaGcWY2uZVNXwEuqOax2mAQ8Luu/EC6jp4+c8zsJTP7pZk1VcnGhiUVvKVz/8DMJpvZgZ34jZoX3qJ9JOiNwwGEAK5qZitV84fNrK+7f+zur3udepK5+wx3/6AOx3nd3WdX4afuJgqHYcS92J76FEiNwI+Ic18duBn4lZmNydck0Rkk6A2Amc0HfAOYAFwP7F+2y7/T30eTJ3WfmY0DxgLbZjysjTPe/B5m9icz+xA4qDzkkjn29mb2LzP7yMzuNbNlM9u+4H1nQzVmti9wErBKxoZ907bPeW1mNsTMbjKz99LnRjMbXH4sM9vdzF5I+9xcbm8r1+6zkEvm3Hcxs7uSx/mMmW0+9zsAwOxUOEx39zuBa4Etyo51hJk9aWazzOwVM/uNmS1afm3MbHQ6l1npmi5T9jvHmNnrad/LgAXLtvcysx+a2ctmNtvMnjKzHTPbS+e5u5n92cw+NLMnzGx1M1vVzB5Ix76//Nht8F469+fd/QTgOWCndKzlzOyWZO8sM3vczLbL2HIfMBQ4q/QMZLZ9NT2Ds8zsv+n7kpnj9jKzU83sLTN708x+ambSpC6gi9cY7ApMc/engMuBfcysb2b72unvVoQn9TXgp8BvafEsBxFhlRKnER7myoTX1RrzEoL8TWBdoDdwo5lZhXZfC4wHns3YcG35TuklvQUYCGySPksCN5cdaxgwBtiZENMRwCkV2pLlFOBcYA3gUeAaM1tw7v/yOXuXJa71/8o2fQp8F1iFKIDXBs4r22de4DhgP+KaLgpcmPnt3YCfENd9LeLaHVH2G4cDRwPHAqsBNxH3Zc2y/U4GziCu0zvA1cme45Nt/Yjr0FE+AkrP34LAH4DNiet5Q7JlxbT9a8B0Wrz8Qek81wDuBZ4H1ge+SjwbfTLH2ROYA6wHHEpcW9UMuoK765PzB7gPOCp9N2AqsGtm+zDAgZFl/3cJ8PuydaV9jyxbv3Fa3z8t75uW18/sMxT4BNgsLY8DJpf9zr7A+5nlL+yT1k/NnNPm6XeHZbYvSwhk9lgfAYtk9jkeeL6da+ela5U594My25dK60bN5TcuIYTlfeDDtL8D32vn2FsBs4FeZdd0eGafPdM+lpYfAH5d9jt3A1Mzy68AJ7byjFwxl/PcLq37Wlv3qo1zyN6nPplzOHgu//MQcEJrv5FZdyXwYDvP/INl6+4CfpPHO1iUjzz0nDGzLwGjgKsAPJ7sK/li2KWjTKpgn0+BR0oL7j4NeJXw6qvJSsCr7j41c6wXWznWNHf/b2b5VWDxThzvybLfoILf+QuwJi1e9+2UebdmtmkK5Uw3s/eAG4F5gCUyu81292fLjj8PUGpgXQl4sOzYny2b2cJE7aW8cfx+vnhfsuf5Rvr7VNm6BcxsfubOKSmM9iHwC+As4FfJngXM7MwUupqZ9hsJDGnnN0cAf2pnnyfLljt7v0WiT/u7iBpzABHqeCkTfTAAM1va3V/u5O/OqnC/uTWSflqyJUPf1nbsAtnjl4c4nM6FBT/7HXf3dF3b+50P3P359P0wM7sX+CFRc8DMhgK3Ab8GTiSyhdYiwhzzZH5nTivnUMnxK6H8Xv2vlW2trWvv2GcDFwEfAK8lp6LET4mayFFEbP0D4DI+f86dpVr3WyR08XLEzPoQDZvHEd5h6bMG4b18M+36cfrbu+wnPm5lXUfoRUt8HjMbQniHU9KqGcDAsjh3eRy3EhumAEua2bDMsZZNx3qmM4bXgZOBYzONeCMJEfueuz/o7v8i7O8oU4h4cpbPlt39XcJTXb9sn1HU7lr9x6NB9NUyMS8d9zJ3v8HdnyTi5cuV7dPaM/AEsGltzBVtIUHPl22B/kRMdXL2A1wDfDOJ6ZtEdXhLMxtoZouk/59KpDkON7P+ZQ2plTAH+JmZrZsa3C4FniZiuhBxzmbgBynbYX+iATfLVGCoRR59fzObt5Xj3E0UUFea2UiLfPorgcdpv1qeC+5+HyGgJ6RVzxHvy3fNbBkz24NoxOsoPwfGmtn/mdnyZnYcsE7ZPmcBR1lkKq1gZj8CNiC85XrzL2DndH9XA64gGluzTAU2MLOlrCUr6SxghJlNMLM10jN6QHIaRI2QoOfL/sC97t5aZ5/riMavzd19DnAYEZ55lcgYgaj+TyHi5TP4olfXHrOJjJDLgIeJ5+FrJS/N3acABwMHEoK8OXBq2W/cQMSb70k27FF+kPR7O6bt96bP68BOrXiEjcR4YH8zG5q808OJjJRniHvR4c407n4tEcY5hfBiVyNCHlnOJQTxTGAykfWzi7v/o3On0SWOIByKvxLZLg+l71lOBJYGXiDuMe7+d2AzYMX0Pw8Du/PFMIuoItbY75MQQohKkYcuhBAFQYIuhBAFQYIuhBAFQYIuhBAFQYIuhBAFQYIuhBAFQYIuhBAFQYIuhBAFoa6Dc/Xv39+HDRtWz0MKIUS357HHHnvL3Qe0t19dBX3YsGFMmlTJqK5CCCFKmNm0SvZTyEUIIQqCBF0IIQqCBF0IIQqCBF0IIQqCBF0IIQqCBF0IIQqCBF0IIQqCBF0IIQpCXTsWCVFUYi7vytHUj6IWSNCFqAKtCbSZSbhFXVHIRQghCoIEXQghCoIEXQghCoIEXQghCoIEXQghCoIEXQghCoIEXQghCoIEXQghCoIEXQghCoIEXQghCoIEXQghCoIEXQghCoIEXQghCoIEXQghCoIEXQghCoIEXQghCoIEXQghCoIEXQghCoIEXQghCoIEXQghCoIEXQghCoIEXQghCkK7gm5mS5vZvWb2jJk9bWaHp/XNZnaXmT2X/jbV3lwhhBBtUYmHPgc40t1XBr4KfNvMVga+D9zj7ssD96RlIYQQOdGuoLv7a+7+ePr+HjAFWArYEbg07XYpsFOtjBRCCNE+HYqhm9kwYATwMDDQ3V9Lm14HBrbxPwea2SQzmzRjxowumCqEEGJuVCzoZrYgcAPwXXd/N7vN3R3w1v7P3Se4+0h3HzlgwIAuGSuEEKJtKhJ0M+tLiPmV7n5jWv2GmQ1K2wcBb9bGRCGEEJVQSZaLARcBU9z97MymW4Gx6ftY4JbqmyeEEKJS+lSwz/rA3sBTZvb3tO4HwOnAb81sf2AasFttTBRCCFEJ7Qq6u98PWBubR1fXHCGEEJ1FPUWFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIg9MnbAJEPZtah/d29RpYIIaqFBL2H0pZAm5nEW4huikIuQghRECToQghRECToQghRECToQghRECToQghRENoVdDObaGZvmtnkzLpxZvaKmf09fbaprZlCCCHaoxIP/RJgq1bWn+Pua6bP7dU1SwghREdpV9Dd/S/A23WwRQghRBfoSgz9UDN7MoVkmtraycwONLNJZjZpxowZXTicEEKIudFZQf8lsBywJvAaML6tHd19gruPdPeRAwYM6OThhBBCtEenBN3d33D3T9z9U+DXwNrVNUsIIURH6ZSgm9mgzOLOwOS29hVCCFEf2h2cy8yuBjYG+pvZdOAkYGMzWxNwYCpwUA1tFEIIUQHtCrq779HK6otqYIsQQoguoJ6iQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQnSQ5uZmzKzdD1DRfmZGc3NzzmclikCfvA0Qorsxc+ZM3L2qv1kqAIToCu166GY20czeNLPJmXXNZnaXmT2X/jbV1kwhhBDtUUnI5RJgq7J13wfucfflgXvSshBCiBxpV9Dd/S/A22WrdwQuTd8vBXaqsl1CCCE6SGcbRQe6+2vp++vAwLZ2NLMDzWySmU2aMWNGJw8nhBCiPbqc5eLROtRmC5G7T3D3ke4+csCAAV09nBBCiDborKC/YWaDANLfN6tnkhBCiM7QWUG/FRibvo8FbqmOOUIIITpLJWmLVwMPAsPNbLqZ7Q+cDmxuZs8Bm6VlIYQQOdJuxyJ336ONTaOrbIsQQoguoK7/QghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghRECToQghREDQFnRA9iI5MdVftafZE7ZGgC9GDaE2kzUziXRAUchFCiIIgQRdCiIIgQRdCiIIgQRdCiIIgQRdCiIIgQRdCiIKgtEUhRMOhfPnOIUEXQjQcypfvHAq5CCFEQZCgCyFEQZCgCyFEQZCg9wCam5sxs4o+QEX7NTc353xWQohy1CjaA5g5c2bVG5M6koUghKgP8tCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgqKeoEB3ET1oYxi1S/d8Uoot0SdDNbCrwHvAJMMfdR1bDKCEaGTv53ZoMpeDjqvqTogdSDQ99E3d/qwq/I8QX6OiYMZoAQfRkFHIRDY1mrhGicrraKOrAnWb2mJkd2NoOZnagmU0ys0kzZszo4uGEEEK0RVcFfZS7rwVsDXzbzDYs38HdJ7j7SHcfOWDAgC4eTgghRFt0SdDd/ZX0903gJmDtahglhBCi43Ra0M1sATNbqPQd2AKYXC3DhBBCdIyuNIoOBG5KWQh9gKvc/Y9VsUoIIUSH6bSgu/uLwBpVtEUIIUQXUNd/IYQoCBJ0IYQoCBJ0IYQoCBJ0IYQoCBJ0IYQoCBJ0IYQoCBJ0IYQoCBJ0IYQoCBJ0IQpKc3MzZtbuB6hoPzOjubk557MSc0PjoQtRUGbOnFmTmZVE4yIPXQghCoIEXQghCoIEXQiRK9WO9ffkOL9i6EKIXKl2rL8nx/kl6D0AP2lhGLdI9X9TCNFQSNB7AHbyuzXJdvBxVf1JIUQXUQxdCCEKgjx0ITpBteO0TU1NVf090TORoAvRQSoNX5lZ1UNdQswNhVxEw6Cu6kJ0DXnoomFQV3UhuoY8dCGEKAgSdCGEKAgSdCGEKAgSdCGEKAhqFBVC5Eq1h6ao57AUHWl0r0cKqwRdiILSXcbwqfbQFPUclqI1u/PsfyBBF6KgaAyfnodi6EIIURAk6EIIURAk6EIIURC6VQy90VqUhRCikehWHrq7f+Ezt/VCCFENKh04Lu+5T7uVh94d6OhgUCp8hGh8usvAcV3y0M1sKzN71syeN7PvV8uo7lIatkZrtQXVJIQQ9aDTHrqZ9QZ+AWwOTAceNbNb3f2ZrhrVXUrD5uZmZs6cWVUbmpqaePvtt7tilhCih9KVkMvawPPu/iKAmV0D7Ah0WdC7Sw+37lLwCCF6Bl0R9KWAlzPL04F1yncyswOBAwGGDBlS0Q/bye92wazWaWpq4u1x1f3N7lLwQPeYA7M7Xc9y2rq+ba2vV7itO9x3qK6dPfnZrHmjqLtPACYAjBw5sqKnuNvElsf9N28LKkLXs/Y04jVuRJtao1vY2U2eza40ir4CLJ1ZHpzWCSGEyIGuCPqjwPJmtoyZzQPsDtxaHbOEEEJ0lE6HXNx9jpkdCtwB9AYmuvvTVbNMCCFEh+hSDN3dbwdur5ItQgghukC36vovhBCibSToQghRECToQghRECToQghREKyeSf1mNgOYVuWf7Q+8VeXfrAWys3p0BxtBdlabnmznUHcf0N5OdRX0WmBmk9x9ZN52tIfsrB7dwUaQndVGdraPQi5CCFEQJOhCCFEQiiDoE/I2oEJkZ/XoDjaC7Kw2srMdun0MXQghRFAED10IIQQSdNFDME0FJXoAPUrQzWyhvG2oFt1BoBrMxkWg4WxqFzNbuv29RHfGzPpW67d6jKCb2RDgATP7wjR53QkzWxjAG7jxIyOa/XI1JGFmWwPXmtmlwDe6S8FuZksAJ5jZEXnb0hnMrJ+ZLZa+DzKzeet8/GFm9t16HrOjmNlw4Dul97qr9AhBNzNz95eAC4HzzWz1vG3qDGa2DXCFmV1pZqPr/YJUiru7mW0FXG5mp5rZznl5xma2LTAOOAOYAmwEfCUPWzrBu8R8A8ua2UF5G9MRzKwXsCYw1swOAc4C6jPBawu9gWMbtUA0sxWAq4EZ7l6ViZR7hKBnvNlngNeAiWa2Zo4mdZgkTD8Cfk50K94HWChta6gwgpl9FTgOuBb4hBDRI3Owowm4CHjU3f/k7qcDBqxXb1s6g7t/ADSlz15mdkDOJlWMu39KvGsbEgXqXe4+Iwl9zTGzvu7+AnAAcISZHV+P41ZK8szvJZ7Ny9O6Ll+bHiHoAGZ2OHAmcB3wGHCJmX05X6sqI1W9LwDucfd73P1w4t5tBy0FViMIu5kNBsYDD7j7dYRn/EdgpbStbrj7TGAssLaZ7ZdWvwd8Wk87OouZjQW+S+Q1Pwislbzdhqb0HLr7NOBPwD3Ayma2ahL6qojX3HD3/5nZDoSg/xb4gZmNq+UxK8XMlgeuAW4G1jSzXSEKwS6/w+7eIz7A+cCo9L03cBTwMDAib9sqsH1BYC/gemC3tO4qooS/AzgMWDtvO5NdixOCPg1YK7P+LmDDOtmwBhFWWS4tbwY8Dvw5XcM+eV+nCs6hF3ASsG9abiIKpzuAb+dtXwX2DwcmAQOImuRPgPOIgauGAluS+sHU4NiWjnkf8LW0bpn0TI7L+boslbToG2l5e+BJYNes/Z39/UJ66OWlXPIGmoCvA7j7J8CdhLCf06ix6NSQ1AuYBVxJ1C72NLM/EOezH+EBDQUuzqOxr3StzWxFM/sSMMvdjyS8ylPNbLPkkQwmvONa27MlcCMhfjeY2TLufjdwNLAscKfHfLi9G6FGU6LcFg9P9lVg/3QOM4nCaF6ittOUg5kV4+7PAvcDvydsPh94B7gM+BvxnNSkYd+D9wihfC+1of0b+BZwYl4NpSnM8nPgZne/Ktn6O+D4ZFfJU/dOP5t5l+S1KJ0z3zcBvkxkWyxBeGk/Stu+QTTUDMrb5jbOYxuimn0rUe1uBuYDxgD/IpXwmf0XzsHG3hlbpxDhlceARdL6M4CZhFe5bvn9qYE9OxAiUqqJnQ2sBgxMy6OBR4Dv5H1/y+zOPrNbpnu8ONGIeAJROK6Szu8PpfNpxA/hCTdllk9Lz0Qz4TlvAWxQq2sILAn0Td+PBH4HNKflLxOhjo1zuC4rAQ8B+7WxffukT3t06Th5PwA1vIBHAn8FJgK/IargS6eH6xrgOWDVvO1sw/btiHDQVwgv/FekkApRldyN8EL3y/xPzYSyFfsWynz/CvAU4f3uRnjhLwP90/bvJVu/VCs7k1D0SQXdHWndIOB9wiN8Fdgxrd8a+AuRl163a1bheXyHKMTPSNe05JB8PxVU9wFr5G1nG7b3BhYA7gZOLhP13wL/oMbOU7q3jwCnkEIrRPjv9+kdeoEU9qvz+9Kc7uf5ablXuqfDyvbbmUjcWKKz9uX+IFTxopV7Obel7+ekku+XwOrpYi4CDMjb5tbOgYiXTwGuz6w/KX36AL3Sur2JlKeF6mzjMsA/gf3T8kBgVWBTosUeIrvl9fQgNxFe5lVETamWHvriwItEIX4XcGhavzvwITA8Lc+f971u5Zn9KuGAzJOE/UXgNmDLtH0hYIG8bW7rHIB50t9hRK3yeFo847FEw+j6NbRjXeBpwhM+DXiWSFM2YC3CA67Z8ediV9/0GUcU1KOBW4Dxbey/eJeOl/cDUc2HKn0fQaRKDSVauO8ivMdbiWyLLfO2d243P/1dnfB0TkrLvyZK7oeAi4FdiMaVBXOwcQtgNvBv4ODM+nHA0en7XoRHVgp9DCi93DW8730yx/on8FDZvhOBdfK+x23YvlP6uyTRznNXWv4Z8AqwSd72tnMu2yaROoOoXTYRoY7TiHDhw8BKtbqGqRDZBlgR2JxojF2XCE9dTE4FeLoOl6dCZiEiajAFuCmzT+9qHrMQjaJeurvR8eZ8otr1MnGDT3X3F4kGkn8CT+Rl59wws82AX5vZMYQn+21gtJk9TojUCCIP/THgcOAjd3+/3na6+51EdfESYB8zOzpteg1YPnXiOJSI8d+f/meGu79dA3PmBTCzPh4NnX3cfQYwChhkZmel7WOI3PNXa2BDp8g8s7sAp5jZ4u7+KuGI/CPt9ihxv6fkY2X7mNnKwMFECt7TwA+J0ME+wNvAysQ7WNVzSA2dbmajgR8TDs+LRPvDke7+YFpeAlihmsfuAH2A55N9SxDadAnwqpmtB5GgUdXG+bxL9y6WgH0z3/cl4o+jMuuOA94EfgBMBpbN2+Y2zmNrwqv4FtER5lzC61iFKIh+XLZ/vzrbNxTYKLO8F5GCNhx4ADiIaLA9HLiU5HHW0B4DvgS8UbqntDTQljz1xYmY6SNEzHzlvO9zsmuBzPeRhJNRiu8bsAHRiHwNESpcLm+b53IPViJ6s56QWb8KEV5Zrnz/GtgwghDJLdJyPyJefyiwcbrvK+Z8nQYSGVY3EwXLgsARREx/dNWPl/eD0YULtRFR8pXiomsCHwNnlO13ENGhqCFe6FbOY1mio8veaXkIkV++c1pelWgM+2nmf+rZoLMIMJVInTwoIz6XAnsSjXaPUJYbXUsbaalqjyNCP4PTcmui/jCwet73OdmzRSoI90nLw4lUxL+RGhGBRZPQH5e3GFV4TlcR7SXZENJlwJdrcKwhwFbAMml5AlF72S1z7zcmwqt3A1/P4Xr0bWVdVtSXJxqPjyOct6qGInN/IDp50bYiPNeDs6VcEvX3SI1hmfVVjVPV4Hx+nER7WFq+BBib2b4GEf/vn5N93wZmEJ1DTgeuILzx8Wn7+sDfiQbTXnWwpyToqxLtCi8AQ7L3OiPqNbenQpu3JcIn2wBfyawfSNTILgMWy9vOCq/7asC2mfVXEo2QaxEF/EvZc6zSsVckauCnAOuV7nW6dhcTtchSwsACpPYl6uv8LE543qu0sm2JJOrXp3NZsFQwVdWGvB+STly0tYm41AZl69dJf1cnPIaj87a1g+d1AhG+mAjcRFlYhZRFUGebemW+H06kXi1DZC1cQvS8XCJtX6TOth2WrtcOSVBeyxSIfTL75Z6amJ7JKeUiB+yS/vYn0utuAhbN2942zqEk5psn8X4EuIGWkNelRA35Z1TZOydCFZMpy9Em8vSNyGaZQAOEpwiv+zJaqV0Rta9jiTTKmmQs5f6gdOKC7c0XwyoXJq/g/9LySKIq3tQIL/RczqV32fLxfD69bt5GsjHZ93jGviE52nUBmVg9kVHxHDA072vWiq2jgAvT91Im0+lEqK2U0bIkUQNaMm97y2yfJ/N9lSRGpT4F5yUBKw2xcC7wdGb/qrx7RBrnUdnfJGrnN5KGECCySSZS5/aljI3Z9+Q8IqV4xWRbyeY1iQbjwbWyoztmuXwEzA+Qum+vQsR3DwZ2NbOvufskooSc6elKNgpmtraZnQKftXD3Lm1z91OIKuUEM1vJ3WfnZSd8lknwmY3JvuuJscXX8BiSuOaDgrXx+7OJRrES56W/t5hZn0bq1k9kKS0Hnw0atQLR9b0XMMfMfu6R4TIu/W0IzGxR4A4zWyut2p4I/62Ylr9LdN76iZmt4O6HAS+Y2f2lLJQqmbIUET/H3d3M1iXGYvo7IZBfIZIifubuH1XpmBWTeU8WSzZ+h3Aof0Ska7qZbUQUQO+4+/SaGZNHadaJ0i/rJWxOpCSWvISFaImdnULEqXrRoJ458TI8Bvwws67cUz+VaNTpk9d5JDuPJnkYfD78chKRnVHzPHg+39j2DWBXQlgGEFkuh6T7vSdwIjnWGuZi9zxEp6HjMuvmz5zT6eXPQKN8iAyxR4iwRz+iF+gEWvoY9CI67Y3I/E9Vahm0eLZbEcN0ZDOEFkh/xwN7NcB12oZoB7uA1D+DaBubSBQ2L5FCbDW1I+8LUcGFWpQYLW8JIq1vRHqopgHLZ/bbm2g0adTUxOwLviMRUz06s65c1PNqAC29RDuTwgSZbVlRr0u8MmPPoUT3920I7/zLRBX2ISJ++y8aMJOJlnFtNk92npTZ9g2iHaChs1mI4Rsep6Ux7/tJuDYu26/UEF01J4QImy6W7v33ykR9BJGamOtIo0QN4WaiU9WeqYA7I20rdQ4rZa3V1EHL/WGp4GItQHhef00v7dJp/UlEteYywmP4F620Ljfah6gqXkf0/vxcjnm5qOdk33zp78KpgDykbHupNlTbBzPTE5bIHvgt0ZHoCKJL/Lwle5PINMRQDkn0Sg2FhxDV7GOJcYQ2JVJS7yVivpNpoPGE5nZPM6I+PF3vk4gxkpqociZRphBfgRhNdD0iZPUg0c4wjqipPUcm2yana7ZE0p6JaXnedI2uIjkYJMez1u+Mu3924RqObAxA7O85AAAUQUlEQVTOzEYSqXL3Et2Lp3v0DFyfCLnMBzzh7lPzsrcSzGwgkSO7i7tPT8NpXkwMp3lmvtZBGv72cOAP7n67mW1IjD1xGvCxp8kJ6mDHkkSe7hRC+GYThfY7RCx1L3f/IE328IhHm0nuWMwLeQwxhs2TRK/F04jG5D8RDYivE52H3gVed/dX8rG2bdJEJK94mTiY2feIcXH2JcKei3v0wq6FDTsQHdgGErNenUw8D6OJEMxU4EF3/2Mtjt8RzOxgQpd2dPd707qrgRs9JnmpG33qebBKKRPzvYA5RKeMw4jGz+uInpXTvJYNDF0k0z0520D0ES0z5jxPeHA/MLM57n52jjYOJbygR4AfmdkIwqtchkgH/Ge9bHL3V83sISKk8nV3n2hmrxBV/YWSmH8D2J8oIBsCd3/XzG4hRkncEfiNu99pZs8QMeBvA78ovfSNQmpA7k144f2ITJUbyxs23f0cM5uH6MW6bg3FfCmiM+AYouD7KlHLOdfdrwaurnKja0dsK70v6xHpqH8n0mbfJ4bu+DExdMNaRC/WutKQWS4ZMf8eUW192iOjYjwRgtnJzM4lJjBYND9L26bsgRsK4O5vEDHfG8xsQY+JNmYSGRo352Fnejh3IOK70z3mNxxDxHaNiFUfY2bz18OeTHbKPMQ4IIcl8T6ZiE3+1cwuIMRnbCMU6Ga2WKpFQgjjC0Qj2D4p+2M6ESpaDfimmfXNydRW8WAO0WN1JLCome1GDEFcvm/JE/2g2nZk7n1f4A13/4fHxBR3EeHV48xs+5LN1T5+JaT3ZRsi3NSPqGHvnt6b8UTHolOJGYj+ZnWaQ7VEI4dcFqelx+QMi0lf/2cxv+YOxMvxa3d/Mk8728NiLtNtic4Y9xMe5YlEo+NtwNeATdODm4d9qxEP5UHu/li555PEfmeiAfetOtm0BxG62JOo3s9PDM17qZltQFTBX22UEJvFjExHEY13/YgCcVkizrsI4ZU/l57d3g0aZhlLXOexwHRgksek2vU4dsnrHZicHszsYmIAuoPT8v7AOsS9PwF4OycPfWFCuH9MzML1S2IE19fT9jFEdtgh7v5IvWsSDRNyaePElyQaGGYQYReIjgMTzKxXvWK6nSV5lrsQMb/bCI9zkLsfZ2Z3Ew0oF+Yl5on5gWeTmPcBHPjEzJo88vhvtZiyazQxznnVMbNNgNnu/kBatTxwg7s/Y2bHAQcCByXP9hrPYZTJdniB6CU5GjjT3WcBT5nZfERhfqyZne7uz+dpZDvc4O7vJyf5CSLMQT3esyTmWwNHm1mpZngO8B0zu5NwOL6fPrsQjbB5eejvmtlLRAhtKLCDu7+eHJ/p7n6tmTUDZ5vZFkRHwbrRECGXspj5SsmTeZu4keub2crppu8FnGtmi3QDMR9MeGe7E+OyzyFyUnexGCL3MXe/3d2fq7Nd5R1uZgFNZjbA3ed4dJBYL9m5gJkNILqmP15Ds/oCr6T7DpGnv7ZF56VP3P2XRLvDcBrECclWpdOzeDkRBlojNdbi7o8QmRmTqcN8ql3kQ4s5aWe6+0Okjlr1eM/M7MtEzPwQImtmDSLEchhx/VYgBuCaThovqNY2ZWwrzZk7NGmTEeG0wcSwwNPMbG1C4OcHSM/rtu7+Qb0LnoZ4OTJifjSR2lWa4PVfRHfjy83sQaJhdCd3/29etrZFWaF0AJG+diLhhW/g7punbfsRaXi92/qtWpLxhtalJe3zCeASM7ucqNKeRnSOmAXMMrP1PSbdrSqla5YaDgcB/zSzPYkU1fWA3VPB2Js0Toi7v1NtOzpKsvvT9H0MkWL5EjGpw5vAAWb2LtHxaTXgV+7+bl72lmMxKfonHplipZrYJ8R7d03arZ5CND8x/2t/Iqd7d3d/z8yGu/tJyeYNCa99v1JYph6k92VHooPVy0Sb1xVEQbNbClUNJ4YmuD/zf7loVG4x9FZitZsQPem2MLPrgA/dfR8z60+Mfd0ETGmUuGlbmNm+RHZGKW66EJGRM4EYcP9wYvKHunbxzsQpVyFe2huJkJYRIY39iJDQEsCl7n5HveJ/ZjbE3V9KDXE/Bv6PeHm2JKrYc4BjG629JBXc3yWyGcYSg2vdSctwqYMJB6RuGULtkcJqo4h7/z4Rlz6zXgJk0bi+sUda7Gii4+DTxHX7iBjA7L8pXLEbIZTvmNlXiXaTl+pkZ+l9WRH4BbATUdv+jruvbmaLEI7ZUOA1d3+63vHyVvH8EvLnLVveisg9PpqYOqrUwWVEvW3r4HmsCWyWWf4d8aIskVk3kujKfyc5TvJL5D9fTcuUZ4MJMbqIlq7odRvciChMViVytLdL63YlYtKj0/L85DDVXgV2L0gUjKWhXJdP1/HwtLwgMDBvW9uwfQ1iEopXS88udRxmmBDIR4iUv9L1O5roPLZp+vyDiE/X+/r0Kvu7AuGdlybwLg1ENjLve9mq/Tk9VFsQ3sxJtAwhumJ6yP5Cy4h036GGQ01W4Ty2JmLLZ/P5sSx+xxfntJwvb2EiGrpeI0IXpZd7KaL2cAMRgqt7b1UiVnoTsHVa3oXoRLRF3vc4Y+MXevml+348sHBaXic9w3UdSrij55AKybvTc3oodZpzM3P85Yhw332ZbcsStZy/EL0st2/rutfQvhWInqjnEo7OUKIGcQ1Ry14h7bcZEaas+njmXf3UPeRiZlsROcWXEVWWJYm8zTeJzgMLAf8hGkUPAvZ098l1NbIC0nmcA3zToxGptL40v+UtxNjWG+VoY6nauBrwATGmxBDgduA0d78o7bcUIULP1NiezzImUohtBXf/VVo+mOiMc56732aRbzzFGyAzpKx9ZB3iJX+YqFWuSXibtxBjeewP7OY5jPo3NzLPwmAitj+H8NQPIrIzTrHoobuYuz9Vw+MvSGR+LEm8P72AMR4pyfO6+2wz6+1prk2vk0BZ9Nq+gahlfUqEeUsZaisQc6T+lXiPDiFCgL+vh20dop6lB9El+lNaSt/BRIeWUrVrAeICnkGMQNeIgy0ZkZVxPmnAncy2nxPZDkum5T8T3ehzsTP93Z4QnJ8QoazBRIPoPykbp6XG9owi4vRrpeUDiKyfsZl9xhOZDFWfa7FK5/A9IkxwK+Hd7kpUx39DdH55gBxDahXYvx0x/sn1wDlp3UZELvW1RDZOzUKcRI32j0SyQGlilJuJMOAORCx9CHX0ypMNKxMe9w5l639IDALYP12nI4mEgU3T9oYb0TWPh2rbdONK1dTbiA435xIl32KNerHKzuMCMjOoEB2E7iNSLSfSMrzn0nW2KzvQ/mCi919zehjvJ8bfgAi/TKuHfamQfoLwcjZJ6/oTHYcmEJkLpcLnViJXP/d7XHYOyxKx/tJzewARCx5OdCZahQaMmWfsX57oqLdRei7uoWXSjWWIjlxb1uC4pREY1ybakPYgnJ7xpcKPCF1dSZqvNodrMwr4NLM8X+b7zwinsyGmMmz3XHK6gFsTnsJ5hNe4M5Fp8UgSw4XzvjBt2F2ar7LU8eGMzLYvZb7fQqahtI72lQa1Ko1EuFS6xmOSsJfGkC81ONY81psE5HnSFIGZ9aURCfcgQkC3Ed7vl2ptU4V2W9nywkmQNs2sm0CML5K7vXM5j17puXiQCCmUhvNdhIijXzm38+7CcZfJHGsZIlxxQlpegpi7YDxpWj5aGuXzGv9/a6IxvuRQ9kt/x5Rfo0b+5PmgbUaEXwZm1vUip3HAK7C3iTRuNTHwzggiJn1C2X67EJ5cXb1MIle7D+ExDgaWSutvIGKmpUmUN00vd10adIi0vsPL1p1FtJMckJaHEQV6w4k50YBXymw4mWjAXT0tf5OYlabhapOtFEjbp0J9O1oK/EWJRsiqh4nS+/1OOkZvIhz5T1q88gGEZ/4zoCnv65VsKol6c2bdTkRNrG8j3ucvnEMDXMBnSGGARv4QqYenEvHS59K6VYnUr/HESHqHEXHIusb+ibznH5DGgyeqtJcTHvomxIBBlxMhjqeoQzoYLTH884CflN3zS4D1ic44uVSzKzyHY4BH00u+J9E4dh6Rwz+RGM61YcYzb+Xab5EKz6PSs7ADEWrZJiPqNctqIkJtz9HiqR9PZDOVCsQBpPlpG+WTns8X0/fhqRDaKm+7KrY/dwMis+FxGjRGxee9tdOJFvoDM+uWIjy3M9On7pNsEA1JVxFpoEsRIxVOJNolvpS2n0o06m1Rfl41tm000WBYagztS5pSkBhkaY962NGJez0PMfl4P2LEyeeJ8NDCRDx4b2JY4dztbuNctk6F0bZEWLM0AcOeRIbO9vV4BlLh8TwR4jFiPJZc+2NUeO0+INr6tsnbno58GmK0RYuhZBttwKXydLWNiHFP1gVWIibb+JO7/8fMFnD3WfUYyKgtG81sGSJ+/hbRYPsmUZv4L3CKp9Hg6o2ZLUB0GpkfuN5jfJPSiIqHAnt7jcbV7ixmdihxj5clsnDeTD0VLyEaEn+Wp32tkXpUL+RpoDczO5FIDV6FyNYY4+7T0rYxwFR3f7hOtm1DhFxGEuObnwDc7u6P1eP4nSH1Yl3Y3W/K25aO0BCC3uiY2WFELG1fjy7qBxHC/lsi5WkxojrpXscLmhHzXu7+qZkNSXa8TaRVvklkD/yX6LKcS250ynPfn4jfP0HUcnYleqzWNPe9Esxsfk/je6eC+1RiEpUNiPGEznX318xsFCFMmxGztzfEy2NmvYnC/CpipqHZZnYGUZMwYJ/03G5PxIcvzcHGrYgMsBW9AcdiaouG6M7fASTo7ZBGUvsl0Yvxzcz6AwnvZxSRdvePOttVEvPSlFwPEHnvfYkQ0FtE7PwNYKV621eOxVCyaxGTJb9C9BKs60iTrWFm2xI2nQlsSPRO/rG7/zENCLUt0Xj/C49pA/vlVTC2hpktRqTZTU8dg75F1Mx6Efnmt7n7Samg+jXwLXf/U062bgvMcvf78jh+T0CCXkZGKEt/tyC82+3T9r7u/r/SvkS1LJ+R1cLrGU/E9g8h2iLOJTz0nxLzV57o7rPzsK/RMbPtiPS5E939FjNbmkihvN9bJlYYRcTOZxADh3m9w2ptYWb9aOldfTGRVXIWMZnK9UTc+lyim/1ywA/d/bZ8rG2hu3m93QkJeoaymPnCHoPZDySq2Vd46uprMQTu0sCP6hxiWZIYjncakUZ5OtGrdiiRffFHYsyYM4l4/+LegMMmNAIWY69fDRzj7o9m2kE2IGLPZ7v7eWnfdYEXsjW0RiF1Wf8m8D8iBbAfMI7IILqEKIgWI8ZHmpqLkaJuSNBbIY0r8hVCOB8l0peGEnneTxLV8q97HYdFTcN43kC8rH/0GC96aSLEcgURGhhMdGr6PTEU8ax62dfdMLMmorv7kURq3feBjYkQ1SCiM8zF7n5iXjbOjUy7ycZEo/MKxHAEZxNjmf+QaICcWM/nVORLQ8xY1EhYTLBQmpR4H6LR8zJixLXZhGjuXmcxH0ZUoc929+uI4Xlx95cJj6yvu88kvLSngQsk5u3yDnAHEZp6nujcdAURoniAqJVtajFjU8ORxHxVop3kGCLX3IgY+sdEKKk/MWGJ6CE0xIxFjYLFnJWlcS02JCakONfdPzazj939oZzif5sA97j7RRZTn62eGmv/Q3jjz5nZw6QxW+SRtU9qH/kVId5LA7eU2hpSFtM9wPgGj/UOBF5y96eBp83sDaJNZTEiU+dgtZ/0LHq0oKcGr9WIRqMHPWZKmUZktbzl7pul/Y4ivN+f5/SCv0hMa7YlMbbEfEQv1ceJ3oBjCdF/3d2fVKNTZaS+Dw+mDwBm9nUie+nfjXYNW7mvDwPvm9nuwLXJ4XiA6FzWT2Le8+ixgp7E8afEzCijiEmJf0LkHf8buMNinssNid51e+ZlKxHHv45oAH2eCAdMJjzLIwmH887Szo0mRN2BdK/HENPfjfEGGIc9SybrakuiwPnE3X9uZrcTHXbWMrM7iFz/QxshJVTUnx7ZKGpmmxLjMA9PHUa2J8ZoPzBt355oIFuD8MyP8RoM+t9RzKzZ3d/OLG9ExEq/TnjnPe9mVomUJ78p8GyjiXmJlMd9BpGiegsRPz+HaLTfi5hP4Fp3vzU3I0Wu9FRBX53osbi3u1+V1j1AZAk8Bdzt7h9ZzI7e1xtsWIIU69+cGGz/B42QWyxqS8o5v5bIXlmSNMojMbjd/h6zZM3v7h8o5NZz6ZEhlxRnXge4K4n2ICIuvRqRrni+mY0nskUaKg6ZxHxt4Ahi6F6JecFJnnkfwgsfTHRwWofoUPQ28LaZHVEavkBi3nPpkYIO4O6TUi/QO4GZ7r5saVvqQfh3d2+4lC+PuRcfAfZy99fljRUbMxtJZF0dm/oeALxMjPo4jBir50Y9AwJ6eB66uz9KzKjTZGZ7Z9b/3t2n52fZ3HH3/3kaPVEvcrEws4XMrDl9H0SMsT/LWyYin03Mu/pLYrq+S9z9r2kYCtHD6ZEx9HKSF/QIEYu8OG97RM8kdeM/i0hHvJYIp+xEZN5MKD2bqddwE/H+PpCTuaIBkaAnzGwE8IG7P5u3LaLnYWYrE5MRX0iEUGam9X2B3Yma5J/d/fL8rBSNTo+NoZfj7k/kbYPomZjZQsTgahe6+0WZ9fsA77r75Wb2KbBNGsOl7uOZi+6BBF2I/PmQiItfX1phZvuSZnoys8Hufr6Z9SF6BwvRKj26UVSIvEmNmQsSk3+sn1m3ANGDeV1gz9RQemUjdHATjYsEXYgc8eAdYsrAXc1srZS5dGGKow8lBmHr7e5z8rRVND4SdCEagxuB14AD09AUlgaPu4CY/m5GrtaJboGyXIRoENLsWLsBBxODxi0DnO7uN+dqmOg2SNCFaDCSsH8CzOvur6g3sKgUCboQQhQExdCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIgSNCFEKIg/D987N2nhFVj4QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "with DeepExplain(session=sess, graph=sess.graph) as de:\n",
    "        \n",
    "    new_saver = tf.train.import_meta_graph(logdir + 'model.meta')\n",
    "    new_saver.restore(sess, tf.train.latest_checkpoint(logdir))\n",
    "    \n",
    "    X = tf.get_collection('tensors')[0]\n",
    "    logits = tf.get_collection('tensors')[1]\n",
    "    yv = tf.reduce_max(logits, 1)\n",
    "    yi = tf.argmax(logits, 1)\n",
    "\n",
    "    heavy_methods = ['SmoothGrad', 'IntegGrad', 'DeepLIFT']\n",
    "\n",
    "    attribution_methods = [\n",
    "                           ('RectGrad'         , 'rectgrad'),\n",
    "                           ('RectGrad PRR'     , 'rectgradprr'),\n",
    "                           ('Saliency Map'     , 'saliency'),\n",
    "                           ('Guided BP'        , 'guidedbp'),\n",
    "                           ('SmoothGrad'       , 'smoothgrad'),\n",
    "                           ('Gradient * Input' , 'grad*input'),\n",
    "                           ('IntegGrad'        , 'intgrad'),\n",
    "                           ('DeepLIFT'         , 'deeplift')\n",
    "                          ]\n",
    "\n",
    "    attribution_methods = collections.OrderedDict(attribution_methods)\n",
    "    \n",
    "    attributions = collections.OrderedDict()\n",
    "    \n",
    "    for k, v in attribution_methods.items():\n",
    "        \n",
    "        print('Running {} explanation method'.format(k))\n",
    "        \n",
    "        if k in heavy_methods:\n",
    "            batch_size = 500\n",
    "        else:\n",
    "            batch_size = 1000\n",
    "        \n",
    "        attribution = batch_run(lambda x: de.explain(v, yv, X, x), data_test[0], batch_size)\n",
    "        \n",
    "        if 'RectGrad' in k:\n",
    "            attribution = np.where(attribution > 0, attribution, 0.0)\n",
    "            attributions[k] = preprocess(attribution, 0, 100)\n",
    "        else:\n",
    "            attributions[k] = preprocess(attribution, 95, 100)\n",
    "    \n",
    "    print('Done!')\n",
    "\n",
    "# Plotting\n",
    "noise = [np.sum(v[:,:8,:8], axis=(1,2)) for v in attributions.values()]\n",
    "\n",
    "plt.figure()\n",
    "plt.boxplot(noise, showfliers=False)\n",
    "plt.xticks(np.arange(1, 9), attributions.keys(), rotation=45, ha='center')\n",
    "plt.title('Attribution in Random Patch', fontsize=14, pad=10)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}