examples/integrated_gradients_mnist.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Integrated gradients for MNIST"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook we apply the integrated gradients method to a convolutional network trained on the MNIST dataset. \n",
    "Integrated gradients defines an attribution value for each feature of the input instance (in this case for each pixel in the image) by integrating the model's gradients with respect to the input along a straight path from a baseline instance $x^\\prime$ to the input instance $x.$\n",
    "\n",
    "A more detailed description of the method can be found [here](https://docs.seldon.io/projects/alibi/en/latest/methods/IntegratedGradients.html). Integrated gradients was originally proposed in Sundararajan et al., [\"Axiomatic Attribution for Deep Networks\"](https://arxiv.org/abs/1703.01365)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TF version:  2.5.0\n",
      "Eager execution enabled:  True\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Activation, Conv2D, Dense, Dropout\n",
    "from tensorflow.keras.layers import Flatten, Input, Reshape, MaxPooling2D\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from alibi.explainers import IntegratedGradients\n",
    "import matplotlib.pyplot as plt\n",
    "print('TF version: ', tf.__version__)\n",
    "print('Eager execution enabled: ', tf.executing_eagerly()) # True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loading and preparing the MNIST data set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 28, 28, 1) (60000, 10) (10000, 28, 28, 1) (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "train, test = tf.keras.datasets.mnist.load_data()\n",
    "X_train, y_train = train\n",
    "X_test, y_test = test\n",
    "test_labels = y_test.copy()\n",
    "train_labels = y_train.copy()\n",
    "                         \n",
    "X_train = X_train.reshape(-1, 28, 28, 1).astype('float64') / 255\n",
    "X_test = X_test.reshape(-1, 28, 28, 1).astype('float64') / 255\n",
    "y_train = to_categorical(y_train, 10)\n",
    "y_test = to_categorical(y_test, 10)\n",
    "print(X_train.shape, y_train.shape, X_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train a convolutional neural network on the MNIST dataset. The model includes 2 convolutional layers and it reaches a test accuracy of 0.98. If `save_model = True`, a local folder `./model_mnist` will be created and the trained model will be saved in that folder. If the model was previously saved, it can be loaded by setting `load_mnist_model = True`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "load_mnist_model = False\n",
    "save_model = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "235/235 [==============================] - 16s 65ms/step - loss: 0.5084 - accuracy: 0.8374 - val_loss: 0.1216 - val_accuracy: 0.9625\n",
      "Epoch 2/6\n",
      "235/235 [==============================] - 14s 60ms/step - loss: 0.1686 - accuracy: 0.9488 - val_loss: 0.0719 - val_accuracy: 0.9781\n",
      "Epoch 3/6\n",
      "235/235 [==============================] - 17s 70ms/step - loss: 0.1205 - accuracy: 0.9634 - val_loss: 0.0520 - val_accuracy: 0.9841\n",
      "Epoch 4/6\n",
      "235/235 [==============================] - 18s 76ms/step - loss: 0.0979 - accuracy: 0.9702 - val_loss: 0.0443 - val_accuracy: 0.9863\n",
      "Epoch 5/6\n",
      "235/235 [==============================] - 16s 69ms/step - loss: 0.0844 - accuracy: 0.9733 - val_loss: 0.0382 - val_accuracy: 0.9872\n",
      "Epoch 6/6\n",
      "235/235 [==============================] - 14s 59ms/step - loss: 0.0742 - accuracy: 0.9768 - val_loss: 0.0364 - val_accuracy: 0.9875\n"
     ]
    }
   ],
   "source": [
    "filepath = './model_mnist/'  # change to directory where model is saved\n",
    "if load_mnist_model:\n",
    "    model = tf.keras.models.load_model(os.path.join(filepath, 'model.h5'))\n",
    "else:\n",
    "    # define model\n",
    "    inputs = Input(shape=(X_train.shape[1:]), dtype=tf.float64)\n",
    "    x = Conv2D(64, 2, padding='same', activation='relu')(inputs)\n",
    "    x = MaxPooling2D(pool_size=2)(x)\n",
    "    x = Dropout(.3)(x)\n",
    "    \n",
    "    x = Conv2D(32, 2, padding='same', activation='relu')(x)\n",
    "    x = MaxPooling2D(pool_size=2)(x)\n",
    "    x = Dropout(.3)(x)\n",
    "    \n",
    "    x = Flatten()(x)\n",
    "    x = Dense(256, activation='relu')(x)\n",
    "    x = Dropout(.5)(x)\n",
    "    logits = Dense(10, name='logits')(x)\n",
    "    outputs = Activation('softmax', name='softmax')(logits)\n",
    "    model = Model(inputs=inputs, outputs=outputs)\n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer='adam',\n",
    "                  metrics=['accuracy'])\n",
    "    \n",
    "    # train model\n",
    "    model.fit(X_train,\n",
    "              y_train,\n",
    "              epochs=6,\n",
    "              batch_size=256,\n",
    "              verbose=1,\n",
    "              validation_data=(X_test, y_test)\n",
    "              )\n",
    "    if save_model:\n",
    "        if not os.path.exists(filepath):\n",
    "            os.makedirs(filepath)\n",
    "        model.save(os.path.join(filepath, 'model.h5'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculate integrated gradients"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The IntegratedGradients class implements the integrated gradients attribution method. A description of the method can be found [here](https://docs.seldon.io/projects/alibi/en/latest/methods/IntegratedGradients.html).\n",
    "\n",
    "In the following example, the baselines (i.e. the starting points of the path integral) are black images (all pixel values are set to zero). This means that black areas of the image will always have zero attribution.\n",
    "The path integral is defined as a straight line from the baseline to the input image. The path is approximated by choosing 50 discrete steps according to the Gauss-Legendre method."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize IntegratedGradients instance\n",
    "n_steps = 50\n",
    "method = \"gausslegendre\"\n",
    "ig  = IntegratedGradients(model,\n",
    "                          n_steps=n_steps, \n",
    "                          method=method)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate attributions for the first 10 images in the test set\n",
    "nb_samples = 10\n",
    "X_test_sample = X_test[:nb_samples]\n",
    "predictions = model(X_test_sample).numpy().argmax(axis=1)\n",
    "explanation = ig.explain(X_test_sample, \n",
    "                         baselines=None, \n",
    "                         target=predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'name': 'IntegratedGradients',\n",
       " 'type': ['whitebox'],\n",
       " 'explanations': ['local'],\n",
       " 'params': {'method': 'gausslegendre',\n",
       "  'n_steps': 50,\n",
       "  'internal_batch_size': 100,\n",
       "  'layer': 0}}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Metadata from the explanation object\n",
    "explanation.meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['attributions', 'X', 'baselines', 'predictions', 'deltas', 'target'])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Data fields from the explanation object\n",
    "explanation.data.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get attributions values from the explanation object\n",
    "attrs = explanation.attributions[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualize attributions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sample images from the test dataset and their attributions.\n",
    "\n",
    "* The first column shows the original image. \n",
    "* The second column shows the values of the attributions.\n",
    "* The third column shows the positive valued attributions.\n",
    "* The fourth column shows the negative valued attributions.\n",
    "\n",
    "The attributions are calculated using the black image as a baseline for all samples."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAApYAAAGaCAYAAAC12APsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAABIEElEQVR4nO3deZxcVZn/8e/TW9JZyAaGrEQBBWSQJQQU0MjuwiKgMKITRkFxQP2JAyIuMCwKOm4MCqIyLCqgghqFAdlFZAuoaAQkIGQhAbJBlk7S3fX8/ri3oc5NdVVX16muqu7Pm1e96KfurVsnVadOPXXvc881dxcAAABQqaZaNwAAAACDA4klAAAAoiCxBAAAQBQklgAAAIiCxBIAAABRkFgCAAAgikGTWJrZlWZ2fvr3fmb2ZD+3c5mZfSlu69AozOx4M/tdXuxmtl3E7a81szfE2h76zszOMrMfFlkevPf1Ln+sMrPZZrY44rb7PYYiZGb/Z2Zzat2Ovsofo/K/VyNtm+/XIWBAE0sze9bMOtKO+0LaaUfFfh53v9fd39SH9pxgZn/IPPZkdz8vdpsKPPdl6evQc9toZmuq/byDkZndbWarzGxY3n3PmtmBefGMNElsKbYtd/+Jux8csV0nZrY/yt2fibH9wS72eOHuX3H3E9Ntb9YfYr73fVWoHYXGpUJijlXZH1B9HUPrUdpvXjSzkXn3nWhmdw/Ac59jZj/Ov8/d3+XuV1X7uUu1o9B4VEisMaqW36+orVrssTzM3UdJ2l3STElfzK5Q6st/MEg/YKN6bpKulfTzWrer0ZjZDEn7SXJJh1e4rUHf7xpQyfFiKDKz5lq3oc41S/p0rRvRSBj/EI27D9hN0rOSDsyLvy7pt+nfLukUSU9J+md633sl/VnSakl/lLRL3mN3k/SopDWSrpd0naTz02WzJS3OW3eapBslvSRphaRLJO0oaYOkbklrJa1O172yZztpfJKkBZJWSporaXLeMpd0ctrm1ZK+K8n68bqMTP8d7xjI92Mw3CR9WdJ9kr6Z15eukZST1JG+t2dIWpi+X2vT21slnZA+9ltpvzg/ve8Pmff4U5KekbQ87bNN6bJzJP04b90Z6fotki5I+9aG9PkuydvedunfYyRdnfbL55QkTT3bPkHSHyT9t6RVkv4p6V15z3VC2qY16bLja/1eVOG9fVa9jxeHS5qffu7ulrRj3nqfk7QkfW2elHRA9v0q0h/+kC6/VNJ/Z9rza0mnpX9PlnRD+t79U9Knivw73iPpT5JekbRI0jl5ywq1o7dx6VJJN0taJ+lA5Y1VSsc8SWel/fTZ/D6RvkYnZvpPz7/192kb1qXPeaw2H0N3TLexOn3dD89bdqWSse+m9DV/UNK26TJT8vl6Mf33/1XSzgPQb85UMmaPTe87UdLdeevsIOm2dJ0nJX0gb9kESb9J2/uwknEhf0z4Tvo+viLpEUn7pfcfKmmTpM70dfxL/msvaVj6+u2ct62tlIxTr0vjXr/zCvw7+9wOFR+Pst+7+WPUlZIuS1+rNZLukbRNumxGum5Ltp+pyt+vkrZL2/Kykv5+fa3HK255fXNAnyzvi0JJsjdf0nlp7GnnHS+pXUni+KKkvZT8+pyTPn6YpDYlX8SfkdQq6Zj0Q7RZYpk+9i9KBreRkoZL2jdddoLyBoz0vivztrN/2ml3T5/3fyT9Pm9dl/RbSWMlTVfyJXNoumx6+mGY3ofX5d+UJAllJ6VD/ZYOSv8haY+0D0zM9rU0LjQIniCpS9InlSSD7dk+kT7mrrRfTpf0D6Vf0CqSWKbx3cr7Ms/bXs+gfbWSZGV0+th/SPpoXts6lQy8zZI+Iel5JV/UI5V8mbwpXXeSpDfX+r2ownv76nuovPFC0huVJEEHKfn8n5H2gzZJb1LyZTs57z3pSXJefb+K9IeeZOvt6XZ6vsjGKUkAJis50vOIkh81bZLekH5+D+nl3zFb0r+kj9tF0guSjuxLO/Luu1LJl+g+6XaGa/PEskvJD6xhkt6RvkY9fSToi9nnyO+XedvrGUNb09f3rPTfu7+SJONNeW1bIWmWks/RTyRdly47JH2txirpuztKmjQQ/UbJzoSe1+fVxFLJ52eRpH9P27ubknF+p3T5delthKSd0nXzX6sPKUk+WyR9VtIyScMLjQnZ117SFZIuyFt2iqRb0r97/c7r5d/Z73Zk3vdXv3ezfSF9b9co+TwMU5LM9nxGZqiXxLJEP47x/XqtpC/otc/CvrUer7i9dqvFofBfmdlqJXtj7pH0lbxlX3X3le7eIeljkr7v7g+6e7cnNSobJe2d3lolfdvdO939F0p+WRYyS8mXwenuvs7dN7h7yfql1PGSrnD3R919o6TPS3previ1x4XuvtrdFypJQHaVJHdf6O5j0/tLmSPpak8/MegbM9tX0jaSfubuj0h6WtIHy9zM8+7+P+7elfa7Qi5K++VCSd+W9K/9bnQqPZR5nKTPu/sad39W0jckfThvtefc/Qfu3i3pKiUJ5MR0WU7SzmbW7u5L3X1+pW2qU4XGi2Ml3eTut7l7p5K9uu2S3qZkD8kwSTuZWau7P+vuT/fjee9V8sW2XxofI+l+d39e0p6StnL3c919kyf1aD9Q8n5uxt3vdve/unvO3R9T8qX4jn606dfufl+6nQ29rPMld9/o7vco2YP4gX48T9bekkYpGes2ufudSr7w8z8Hv3T3h9y9S0liuWt6f6eSH047KEnSH3f3pRHa1BdflvRJM9sqc/97JT3r7v+bfu7/pGTv8/vTz+XRks529/Xu/ncln71XufuP3X1F+thvKOlvfa1H/anCfvLB9D6p+HfeZipsR778791CbnL336ffgV9Q8h04rR/Pk9Xv71cl/WobJT8gy/lOxwCoRWJ5ZJpwbePu/5HpzIvy/t5G0mfNbHXPTclei8npbUkmEXuul+ebpuQLuqsfbZ2cv113X6vkl/mUvHWW5f29XskA3GdmNl3J3oGr+9G+oW6OpN+5+/I0/ml6XzkWlV4lWOc5Jf2iUlsq+XGU32+fUy99y93Xp3+Ocvd1SpKrkyUtNbObzGyHCG2qR4XGi+znMqfkPZri7gsk/T8le2xeNLPrzKzs9ysdW67Ta8nTB5UkTFL6hZYZm87Sa0l/wMz2MrO7zOwlM3tZyfu2ZbltUum+uirtGz1i9dXJkhalr3P+tkuOg2kSeomSw5gvmtnlZrZFhDaV5O5/U5IAn5lZtI2kvTLv3/GStlZyaLpF4WsdvO5m9p9m9riZvZw+doz6/n7eJWlE2idmKEmUfpnXrt6+8zZTYTvylepXry5PvwNX9tamMlXy/XqGkj3gD5nZfDP7SIT2IJJ6m24oP1FcpOSQwdi82wh3v1bSUklTzMzy1p/eyzYXSZreS2FyqT2Ezyv5sEuS0rMMJyip34rlw5Luc84ULouZtSvZG/MOM1tmZsuUlEa8xczeos3f297e677sJc7/dT5dSb+QkkONI/KWbV3GtpfrtV/d+dvuU99y91vd/SAlezGfULLHbKjIfi5NyXu0RJLc/afu3rM32yVdVGAbfXnfr5V0jJlto+Tw5A3p/YuU1KPlj02j3f3dvWznp0rqx6a5+xglNWs9Y1ehdvS3r47LPxNa5fXVYp6XNM3M8r8vyumrF7v7HkoOK79R0ullPHelzlZSTpKfrCySdE/m/Rvl7p9Qcri1S9LUvPVf/fyb2X5KkpoPSBrn7mOVlCgUez9flR59+JmSHyz/qqRmuGc2kGLfeYF+tqO//Sr/3z9KyWHz55X0Kan3flW171d3X+buJ7n7ZEkfl/Q9izgtHCpTb4llvh9IOjn9ZWdmNtLM3mNmoyXdr+TD/ykzazWzo5Qc8i7kISWJ6IXpNoab2T7pshckTTWztl4ee62kfzezXdOpbL4i6cH0sGUs/6ak7gTlOVLJYc+dlPzq31VJ/da9Sl7TF5TUvvV4Scnh4/7MIXm6mY1LD/98WsnJYlJSZP92M5tuZmOUHMrJl23Dq/K+YC4ws9Fp8nKapB8XWj+fmU00syPSgXijkuL4XImHDSY/k/QeMzvAzFqV1JdtlPRHM3uTme2ffl43KKmLLPTalOwP6SHS5ZJ+KOlWd1+dLnpI0hoz+5yZtZtZs5ntbGZ79rKp0ZJWuvsGM5ulsFyjUDtKjUvF/JeZtaWJx3v12kwTf5Z0lJmNSL+AP5p5XK99VcnJOOslnZGOt7MlHaZkj25RZrZnOoa3KklENmgA+2q6B/t6JSfg9fitpDea2YfTf09r2s4d08/ljZLOSV+rHZSMJz1GK/nueUlSi5l9WVL+HtgXJM3IJOFZP1VyxOF4vXYYXCr+nZfVn3YUe4+LebeZ7Zv2x/MkPeDui9z9JSVJ4IfSz8BHJG2beb6qfL+a2fvNrCf5X6UkiR1KY2Bdq9vE0t3nKfmleYmSjrNASTGw3H2TpKPSeKWSD+mNvWynW8kguJ2SMzAXp+tL0p1KTghYZmbLCzz2dklfUrKnYqmSD03BOqqsNNlYa8mh7t7WeauSX8ZMM1S+OZL+15Na1mU9NyX95XhJX5X0xfSQ0n+mh5IvkHRfel/BuqVe/FrJCQh/VlK39iNJcvfblHxpPZYu/23mcd9RssdrlZldXGC7n1TyZfuMkhrCnyop7i+lSUkS+ryS/v8OJSf3DAnu/qSSExf+R0nid5iSaYk2KakzuzC9f5mk12nzhF9l9IefKjkR5Kd5j+1WkrTtquSM8J7kc0wv2/gPSedaMk/tl5UkxsXaUXRcKmKZkrHyeSWH7U929yfSZd9ScqbwC0pqBn+Seew5kq5K2xDUZaav62GS3pX+W78n6d/ytl3MFkoSplVKDnuuUHJ2/0A6V8kJO5KkdA/hwUrG8ueVvG4XKek7knSqkvdymZIZJq5V8sNFkm6VdIuSE+2eU5Io5x9K7hnLV5jZo4Ua4+4PKvncT5b0f3n39/qdV0B/2lFqPOrNT5Xs+V2p5CTJD+UtO0nJHugVkt6s5Ez2HlX7flVS5/ygma1VcjTg0xz1qx89ZzwCAIAMM7tI0tbuXm79NjAk1e0eSwAABpqZ7WBmu6SHo2cpKRv4ZanHAUgw0z4AAK8ZreTw92QlpQPfUFIOA6APOBQOAACAKDgUDgAAgCiKHgo3M3ZnNjB3t9JrVV9HRwf9qIG1t7fXvB/RhxpbPfQhiX7U6OqlH6E49lgCAAAgChJLAAAAREFiCQAAgChILAEAABAFiSUAAACiILEEAABAFCSWAAAAiILEEgAAAFGQWAIAACAKEksAAABEQWIJAACAKEgsAQAAEAWJJQAAAKIgsQQAAEAUJJYAAACIgsQSAAAAUZBYAgAAIAoSSwAAAERBYgkAAIAoSCwBAAAQBYklAAAAoiCxBAAAQBQklgAAAIiCxBIAAABRkFgCAAAgChJLAAAAREFiCQAAgChaavnkxxxzTBCfdNJJQfz8888H8YYNG4L4Jz/5SRAvW7YsiBcsWFBpE9EActoUxOv9xSB2dQfxSJsUxM1qCzfomdAqax/qX6V9qCnbh4B+2GyoYSxCA2KPJQAAAKIgsQQAAEAUJJYAAACIwty994VmvS+M4JlnngniGTNmVLS9NWvWBPH8+fMr2l6lFi9eHMRf+9rXgnjevHlVfX73+qjI6ejoqGo/2uDLg3jh+oeDuMvD+rlXNr4QxDuPf1cQv9y5JIjNmoN4RMuYIB5uE4K4TaPD51dYG9ztYZx9fJc6wufPVF41a3hmedi+2Nrb22vej+hD9KEYqt2PUF310o9QHHssAQAAEAWJJQAAAKIgsQQAAEAUNZ3HMjtv5S677BLEjz/+eBDvuOOOQbz77rsH8ezZs4N47733DuJFixYF8bRp0/rcVknq6uoK4pdeeimIJ00K57bLWrhwYRBXu8ZyqBjRNT6Id7SDwxWaw7KczlHhnISrc+F8p5u6w/q0XGYOw1EtYT3b75d8P4jfMG5mEG874oAgbrLWsHld4e+7ls72IN7Qnm1PZ/j4KtfHDQXDbcsgfuPId/WyZiI3KnwPat2HsjWSrRoVxJ1am2kPfagamjpzQWydmZLOpnAs6h5eX/t2mrrC9lrm39PdTj9BafXVqwEAANCwSCwBAAAQBYklAAAAoqjpPJaxjRs3Loh33XXXIH7kkUeCeM899yxr+9lrlf/jH/8I4mxN6PjxYe3fKaecEsSXXnppWc9frqEyj2Vsliu9TlHZn2uZ7VkufDlyLeHbtMafC+IXOp4M4u3bwxrSar/L9TB3XKP1oVor1Ye2G5GpQ66yeuhDUuP1o1qPRaVkM4ShMBahNPZYAgAAIAoSSwAAAERBYgkAAIAoBlWN5UA7+uijg/hnP/tZEP/tb38L4ne+851BvHLlyuo0LEWNZWPIKbwO9crusHZ3RNNWQTxSE4N4KNQ10YeKK7cPjbCwD1VbPfQhiX4UGzWWKIQ9lgAAAIiCxBIAAABRkFgCAAAgippeK7zRvO51rwvi733ve0Hc1BTm6eeee24QV7umEo0hWyTU4cuD+IlVdwfxvlv+RxBTJIascvsQIG0+FpU7ttRHFT/qDXssAQAAEAWJJQAAAKIgsQQAAEAU1FiWIXut7622CueGW7VqVRA/+WR4fV5AkjrVEcTdmTkIdxg3O4ibwsXKtVWhUWgoXWX2IfYhoJByayqbNoUXG8+10a+wOXoFAAAAoiCxBAAAQBQklgAAAIiCGssi9tlnnyA+88wzi65/5JFHBnH2WuEYmpo6w7qkzpY1Qbxq48Ig3mb424M411yddqFxdXp5fQiQNh+Lcq3l7VuiphJ9QS8BAABAFCSWAAAAiILEEgAAAFFQY1nEu9/97iBubW0N4jvuuCOI77///qq3CY0nW8fUrHAiyunDw1reSq/fi8Gv2Yr3IaCQcmsqGYvQH+yxBAAAQBQklgAAAIiCxBIAAABRUGOZp729PYgPPfTQIN60Kbwe79lnnx3EnZ2d1WkYGkq2LmmdvxDED7xwdRC/c+vTgpg6JmStL7MPAVLlNZKMRegP9lgCAAAgCvZYAgAARDBy2nDv3pArvWIZNi7vvNXdDy29Zn0gsQQAAIggtyGn1x+1ZdRtPnH50rgbrDISyzynn356EO+2225BfMsttwTxH//4x6q3CY1no14J4i6tC+J9t/54EJu4GDhCm+hDiIAayRowqakpW906tJBYAgAARGJDO68ksQQAAIjBJDUN8dOih/g/HwAAIBKTrMmi3vr0tGaHmtmTZrbAzM4ssHyYmV2fLn/QzGak97ea2VVm9lcze9zMPl/pSzCk91i+5z3vCeIvfelLQfzKK2Gd07nnnlv1NqHxdXlYDzfMxoVxbosgdn7eIaNUH2pT2IeA/rDMycuMRXEM9B5LM2uW9F1JB0laLOlhM5vr7n/PW+2jkla5+3ZmdpykiyQdK+n9koa5+7+Y2QhJfzeza9392f62h24EAAAQiTXFvfXBLEkL3P0Zd98k6TpJR2TWOULSVenfv5B0gJmZknO8RppZi6R2SZukzNmDZRrSeywBAABiMZOa4p+9s6WZzcuLL3f3y/PiKZIW5cWLJe2V2car67h7l5m9LGmCkiTzCElLJY2Q9Bl3X1lJY0ksAQAAIunjXsZyLHf3mdG3mpglqVvSZEnjJN1rZre7+zP93eCQSiwnTJgQxBdffHEQNzeHc8HdfPPNQfzAAw9Up2FoMLlM1BXEo7omBnG2jql7WFUahYZSvA+NsEkD2RgMUk2dYT/bfCyiGq4aanBW+BJJ0/Liqel9hdZZnB72HiNphaQPSrrF3TslvWhm90maKanfiSW9CgAAIAKzmtRYPixpezN7vZm1STpO0tzMOnMlzUn/PkbSne7ukhZK2j9pu42UtLekJyp5DYbUHksAAIBqGugr76Q1k6dKulVSs6Qr3H2+mZ0raZ67z5X0I0nXmNkCSSuVJJ9Scjb5/5rZfCXTcP6vuz9WSXtILAEAACKpxZV33P1mSTdn7vty3t8blEwtlH3c2kL3V2JQJ5bZmsnstb5f//rXB/HTTz8dxNl5LTE0ZceIpo2Z5S2tQby65bkg3sKmV6FVaGxNmagtiF/JTCG3eR+iimko2nwsytTqtoRr5FrpJwPNjCvvDOrEEgAAYOD0/Wo5gxWJJQAAQAzssSSxBAAAiKUK81g2lEGdWG677bZBvMceexRd/7TTTgvibM0lhibPxBuHhUWWrWoP4jE+o+jjgS51BHFLpg9tYTMGsDVoFNmxJDsPZfYArGUe4EP7CO2AMFXlyjsNZVAnlgAAAAPG2GNJYgkAABAJNZYAAAComEmcFV7rBsS0zTbbBPHvfve7ouuffvrpQfzb3/42epvQeLJ1SZtsbRC/uOlvQTyx7V+CuNVGVqVdaFydKq8PtYg+hPJrJLM1mEO81K82OCt8cCWWAAAAtTTUE3oSSwAAgAhMA3+t8HpDYgkAABADZ4UPrsTyYx/7WBBPn178Gs333HNPELsz4yCkjbY6iF/pXhzE81+6M4inTNk7iOlF2KTVQVxuHwKkyuedZCyqDfZYAgAAoGImY4L0WjcAAABgUDD2WJJYAgAARMDJOw2eWO67775B/MlPfrJGLUEja96QC+Lc8O4g3pQLr+v810UPBfG7tgwfn71+L4aenMrrQwdPqXqT0ACyY1H38PLGkuaNjEX1oGmIn73T0IklAABA3TBjj2WtGwAAADAYcCicxBIAACAazgpvYPvtt18Qjxo1quj6Tz/9dBCvXbu2lzUxlORaw0GgRcODeExLWAD39jcdEcTWGdY1ibqmIa/cPgRIm49F5WIsqj0zqakGFws3s0MlfUdSs6QfuvuFmeXDJF0taQ9JKyQd6+7Ppst2kfR9SVtIykna09039LctDZ1YAgAA1I+Br7E0s2ZJ35V0kKTFkh42s7nu/ve81T4qaZW7b2dmx0m6SNKxZtYi6ceSPuzufzGzCZI6K2kPP2cAAABisORQeMxbH8yStMDdn3H3TZKuk5Q9LHKEpKvSv38h6QAzM0kHS3rM3f8iSe6+wt27VQH2WAIAAERQpZN3tjSzeXnx5e5+eV48RdKivHixpL0y23h1HXfvMrOXJU2Q9EZJbma3StpK0nXu/rVKGjuoE8u//OUvQXzAAQcE8cqVKweyOahTG5pXB/GfVvw8iMcOmxzEe46bE8Rd7Pgf8jZqVRCX24cASfLmyhKSrlGD+iu9YVShxnK5u8+MvdFUi6R9Je0pab2kO8zsEXe/o78b5BsRAAAgBjNZ5FsfLJE0LS+emt5XcJ20rnKMkpN4Fkv6vbsvd/f1km6WtHslLwGJJQAAQAQ9h8Jj3vrgYUnbm9nrzaxN0nGS5mbWmSup51DJMZLudHeXdKukfzGzEWnC+Q5Jf1cF2G8OAAAQyUCfFZ7WTJ6qJElslnSFu883s3MlzXP3uZJ+JOkaM1sgaaWS5FPuvsrMvqkkOXVJN7v7TZW0x5KEtZeFZr0vRN1z97qYpbWjo4N+1MDa29tr3o/oQ42tHvqQRD9qdPXSj4qZuN1IP/ZrO0fd5v8c/dAjVayxjI49lgAAAFFwrXASSwAAgEi4pCMAAAAqllzSkcSyV/VSo4fG1gh1Mahv9CHEQD/CQKjFtcLrCXssAQAAIjD1+TKMgxaJJQAAQAwcCiexBAAAiIXEEgAAABUzSU1GjSUAAAAqZGZqaR7aqdXQ/tcDAABEY5wVXusGAAAADAYmqdmaa92Mmho0abWZXWlm56d/72dmT/ZzO5eZ2Zfitg4AAAx6Zmpqao56azQDmlia2bNm1mFma83shTQZHBX7edz9Xnd/Ux/ac4KZ/SHz2JPd/bzYbSrw3HPM7BEze8XMFpvZ18yMPcgAADSwJmuOems0tdhjeZi7j5K0u6SZkr6YXWGIJFgjJP0/SVtK2kvSAZL+s5YNAgAA/WdpjWXMW6OpWYvdfYmk/5O0sySZmZvZKWb2lKSn0vvea2Z/NrPVZvZHM9ul5/FmtpuZPWpma8zseknD85bNNrPFefE0M7vRzF4ysxVmdomZ7SjpMklvTfegrk7XffWQehqfZGYLzGylmc01s8l5y9zMTjazp9I2ftesb1Puu/ul6Z7VTelr8RNJ+/TjpQQAAPXATM3WHPXWaGqWWJrZNEnvlvSnvLuPVLL3bicz203SFZI+LmmCpO9Lmmtmw8ysTdKvJF0jabykn0s6upfnaZb0W0nPSZohaYqk69z9cUknS7rf3Ue5+9gCj91f0lclfUDSpHQb12VWe6+kPSXtkq53SPrY6WmyOb2PL8nbJc3v47oAAKDOmDTkayxrccj5V2bWJellSTdJ+kresq+6+0pJMrOPSfq+uz+YLrvKzM6StLckl9Qq6dvu7pJ+YWan9fJ8syRNlnS6u3el9/2hl3Wzjpd0hbs/mrbp85JWmdkMd382XedCd18tabWZ3SVpV0m3uPtCSWP78iRm9hElZQEn9rFdAACg7hgTpNfgOY9099t7WbYo7+9tJM0xs0/m3demJEl0SUvSpLLHc71sc5qk5/KSynJMlvRoT+Dua81shZK9ns+mdy/LW3+9pLJORjKzI5XsFT3Q3Zf3o40AAKAOmKkh9zLGVG8nyeQnioskXeDuF2RXMrN3SJpiZpaXXE6X9HSBbS6SNN3MWgokl15g/XzPK0lwe553pJLD8ktKPK5PzOxQST+Q9B53/2uMbQIAgFqxhqyLjKme99f+QNLJZraXJUaa2XvMbLSk+yV1SfqUmbWa2VFKDnkX8pCkpZIuTLcx3Mx6TpJ5QdLUtGazkGsl/buZ7Wpmw5Qctn8w7zB4v6X1mz+RdLS7P1Tp9gAAQG31XCs85q3R1G2L3X2epJMkXSJplaQFkk5Il22SdFQar5R0rKQbe9lOt6TDJG0naaGkxen6knSnkhNmlpnZZoeh00P2X5J0g5LkdFtJx/Wl/enJO2uLnLzzJUljJN2crrfWzP6vL9sGAAD1iAnSLSxTBAAAQH/MePNWfvZPC05S028f2fX7j7j7zGLrpKV135HULOmH7n5hZvkwSVdL2kPSCknH5h99TXeC/V3SOe7+35W0t95qLAEAABqSydQ8wHsZ02kVvyvpICVHZR82s7nu/ve81T4qaZW7b2dmx0m6SK8dvZWkbyqZW7xiJJYAAAAR9NRYDrBZkha4+zOSZGbXSTpCyR7IHkdIOif9+xeSLuk5ATqdneafktbFaAyJJQAAQAxm1aiL3NLM5uXFl7v75XnxFIXTNS5WcrEZFVrH3bvM7GVJE8xsg6TPKdnbGeWy0iSWAAAAUZia4k83tLxUjWUFzpH0rXSe7igbLJpYmhln9jQwd4/TSyrU0dFBP2pg7e3tNe9H9KHGVg99SKIfNbp66UfFJJd0HPBD4UuUXAymx1RtPt92zzqLzaxFyaw0K5Ts2TzGzL6m5GqBOTPb4O6X9Lcx7LEEAACIoiYTpD8saXsze72SBPI4SR/MrDNX0hwl84AfI+nO9AIz+/WsYGbnSFpbSVIpkVgCAABEYaZqHAovKq2ZPFXSrUqmG7rC3eeb2bmS5rn7XEk/knSNmS1QMv93n+bk7g8SSwAAgCiqcvJOSe5+s6SbM/d9Oe/vDZLeX2Ib58RoC4klAABABDWabqiukFgCAADEYAM/QXq9IbEEAACIINljSWIJAACAihmHwmvdAAAAgMGCPZYAAAComFXnyjsNhcQSAAAgBjMZiWXj+s//DK+X3t7eHsS77LJLEB9zzDFFt3fppZcG8f333x/E11xzTblNRAPIqTOIu7QujH1jZnlHEK/auDCIJw3fNYiHaYsg5ppyg0+1+1Bbpg8B/ZG9HiJjUXWwxxIAAAAVM5maRGIJAACACNhjCQAAgIpx8k6DJZbXX399EJeqmczK5XJFl3/84x8P4gMPPDCI77nnniBeuDCsi0JjsO6wsqij6aUgdnUH8UibHMSdmfq5LYe1BXG3bwjiTZk5zdo0KvN8aHQdXt0+1JnpQ62ZPoTGlB2LvDlbBRlXdqyh5rIaOHmnoRJLAACAekaNJQAAACpmxpV3SCwBAAAiocayjlVaU/nEE08E8a233hrEb3jDG4L4sMMOC+Jtt902iI8//vgg/upXv1pWe1ArYW3t2qZlQTzKw/q3ea9cHcTbjtk7iF9c/3QQd/mmosv32OroIG7zTH3c0P5x2yDCPrTOwz6UraF8ZHV1+1CrUWM5GGRrKi1zGoBXe2zInnbAWBQBJ+/UdWIJAADQKIyTd0gsAQAAYuHkHQAAAETAofC6SixnzpwZxO973/uKrj9//vwgPvzww4N4+fLlQbx27dogbmsL54574IEHgvgtb3lLEE+YMKFoe1AfsnOzbdL6zPLwQ9+0IZxzcM9RHw5iz9QhjR35xqLP9+aRJRqEhtNZog9l7TH234ouz/ahrJ0ooRwUyp0nMjsWeVtY9OiZDW5Wo5ndYKmJKxmbojPj5B1KdQEAAKJI9ljGvPXpWc0ONbMnzWyBmZ1ZYPkwM7s+Xf6gmc1I7z/IzB4xs7+m/9+/0legrvZYAgAANK6BP3nHkif8rqSDJC2W9LCZzXX3v+et9lFJq9x9OzM7TtJFko6VtFzSYe7+vJntLOlWSVMqaQ+JJQAAQASmmpy8M0vSAnd/RpLM7DpJR0jKTyyPkHRO+vcvJF1iZubuf8pbZ76kdjMb5u4b+9uYukosJ02aFMRmYQFItqbykEMOCeKlS5eW9Xyf/exng3innXYquv5NN91U1vZRG7nMdZoXdoS1s8ObwwK2kcMnBnG5c8dZV+Z6vyXqoDabrI6KlLqTvdZ3qT40oi3sQ9VHH2oE5V57Ozc8TEiqPxaVt330RVWuvLOlmc3Liy9398vz4imSFuXFiyXtldnGq+u4e5eZvSxpgpI9lj2OlvRoJUmlVGeJJQAAQKOq0sk7y919ZunV+s/M3qzk8PjBlW6LxBIAACCKmkw3tETStLx4anpfoXUWm1mLpDGSVkiSmU2V9EtJ/+buT6tCHD8BAACIxNQc9dYHD0va3sxeb2Ztko6TNDezzlxJc9K/j5F0p7u7mY2VdJOkM939vhj//rraY/mb3/wmiLfbbrsgXrNmTRCvXLmyouc77rjjgri1tbWi7aE+ZD+I27UfGK4Q+3q8ubCuKVMarM7mcA7EVg8nuszWQaH2sn1o22wfGmBdWhfELcpOlopGsNlHfYDHos3mvcwUgTIWVc5qsMcyrZk8VckZ3c2SrnD3+WZ2rqR57j5X0o8kXWNmCyStVJJ8StKpkraT9GUz+3J638Hu/mJ/21NXiSUAAEDjqs2Vd9z9Zkk3Z+77ct7fGyS9v8Djzpd0fsy2kFgCAABEYkO8ypDEEgAAIJqhXVNQ14nlc889F3V7p59+ehC/8Y3Fr9f74IMPFo3RGDabS67CH5NNmbniWl7pDOLukeHHqrU5Uw83tH/Moh+oqRwcaj0WbXaElrGoCow9lrVuAAAAwGBgSlLLoYzEEgAAIBr2WAIAACAC9lgOYu9973uD+Nxzzw3itra2IH7xxXDaps9//vNBvH59OB8hhoacwrqltuXhdaStqfggkmvKrN+3CW8xiGT7UJPKmzM3e+1y+hAkqWV5eEnnUmNRxfNkog9MFv9a4Q1lUCeWAAAAA4s9lgAAAIiAs8IBAAAQgVFjWesGVNPMmTODOFtTmXX99dcH8T333BO9TWg8ba+E8cYl4TXrW948IYhzw8Nfq0N7iIFUfk1lFjWVkDafp7LcsQgDZWi/7oM6sQQAABgozGNJYgkAABCJDfkjDCSWAAAAkXDyziDyq1/9KogPPvjgoutfffXVQfzFL34xdpPQgDq1Noib2kcEcdsbxgTxmmErg7hdW1anYWgY2T7UqlFF19/gy4N4uNGHsLlce7gnLDsWdVLfXRc4FA4AAIAITJy8AwAAgIolJ++QWAIAACACDoU3sEmTJgXx2972tiAeNmxYEC9fHtYxnX/++UG8dm1YF4WhwXJhPGp5OOdg5/hwkNg0LqxzytZUerymoUGVqqnMXvubmkpIm49FbZlrgXeOD+di7hwXxtl0hrGoBswkrhUOAACAGNhjCQAAgAhsyNdYDu1/PQAAQFQW+daHZzQ71MyeNLMFZnZmgeXDzOz6dPmDZjYjb9nn0/ufNLND+vmPflVD77G84YYbgnjChAm9rJn48Y9/HMRPP/109Dah8VhXWNi08XVhjeUaXxjE2ZpK08jqNAwNLOxTa3xRELdnaipb6ENQobFoWC9rFkZNZX0Y6D2WZtYs6buSDpK0WNLDZjbX3f+et9pHJa1y9+3M7DhJF0k61sx2knScpDdLmizpdjN7o7uHheBlYI8lAABABD3XCo/5Xx/MkrTA3Z9x902SrpN0RGadIyRdlf79C0kHmJml91/n7hvd/Z+SFqTb6zcSSwAAgChiHwY3SdrSzObl3T6WedIpkvIPiyxO7yu4jrt3SXpZ0oQ+PrYsDX0oHAAAoG64qlGTsNzdZ0bfapU0VGJ5+OGHB/Huu+9edP277747iM8+++zYTcIgsL5tVRBn62Oy9XCtmXo46pqwQSuCuFQfoqYSheTayjuIyLyV9chlPuDvxBJJ0/Liqel9hdZZbGYtksZIWtHHx5aFQ+EAAACxeORbaQ9L2t7MXm9mbUpOxpmbWWeupDnp38dIutPdPb3/uPSs8ddL2l7SQ2X/m/M01B5LAACAujbAOyzdvcvMTpV0q6RmSVe4+3wzO1fSPHefK+lHkq4xswWSVipJPpWu9zNJf5fUJemUSs4Il0gsAQAA4hn4Q+Fy95sl3Zy578t5f2+Q9P5eHnuBpAtitaWuE8vsvJRnnXVWELe2hvMNZv35z38OYq4FPjRl65BuW3pREM/a+tggHtu9TRDnWsItUMeE20v0oS1em3sYeFW5NZFNXeEajEUNwCUb4m9MXSeWAAAADYXEEgAAAFHU4FB4PSGxBAAAiGVo55X1nVh+9rOfDeI999yz6Pq/+tWvgph5KyFJTevDE9xyuTB+bMVvg3ivLU8I4jYfFcTepytsYTArtw+1KuxDGJqyY1H3iOai62drKrO1e4xFqEd1nVgCAAA0DFctJkivKySWAAAAsQztvJLEEgAAIBoSy/p12mmnlbX+qaeeGsTMWwlJsq5cEI9tnxjEt/3t50H89redEsS5uv6UoBZK9aF93x6ORYC0+ViUXCSld6XmsUSd4lA4AAAAYmCCdAAAAFTOxaHwWjcAAABg0CCxHDzGjx8fxJ2dnRVt7+WXXy66vey1yseMGVN0e2PHjg3icmtIu7vDOdA+97nPBfH69evL2t5QsXr0C0E8ceP2QXzEbh8L4lxLOCq4wrooy9RFlap6KneMyR5Gse7wDsvUXa0fnumnHtYWj7TJmfaE/55mtWaWI2utLw7iiSOK9yFlXmPPvKrZPlRrG7UqiMvtQ02ZPoTCurYo73Uqt6ay1mNR9/CmMp8hs/1M3JhjkVNjWesGAAAADAYmaiwr+3kBAAAApNhjCQAAEAuHwgePxx57LOr2fv7zcG66pUuXBvHEieFcdscee2zU5y9l2bJlQXzBBRcM6PM3itGaGsbDwzg3PKxdbe7IDAqZqefWjVgerm/Dg7gp87FqVmZ5pi6pq6UriG9dcmEQf+1H1wTx7LdvE8SnvP1bQTyiaasgXu9hjWm2Pq7NwutYt2msEBplYZ8ZlelDnulD2YNB2dqxDV5ZH8rKKaz/vnUxfagebXaINNMxsoubOzL9KjMWdY8Ma3XLTWdKzZOZvRa5Z2s+I8+rOSjSMc4KH1yJJQAAQC0N9RpLEksAAIBYOBQOAACAKIZ2XinzIpm1WW136N54441BfMQRR9SoJf3T1RXWzuVy2evEhubOnRvE8+bNK7r+vffeG8QPPPBAELtnK2Rqo6MjW7Q4sFrWhu+Dt4T1bxuHbwzits5hQbyhNZzTr8s7gni4hfOnZueF7FS4/gZfUbS93Qrb8/jKO4J4fWc4b+XP7w1rgc86PKyXmzbsrUWfz7ozdVXNYdze3l7zflTrPlRKV+Y9blF7EG/SK+H6JfpQdl7I7PbrrQ+VmmCkHvqQVPt+VGosys4D2dQZfmfkWsubyKXR5oXMzpNZj2NRKTP/ZTd/aO6dUbfZ/Ibxj7j7zP481szGS7pe0gxJz0r6gLuvKrDeHElfTMPz3f0qMxsh6eeStpXULek37n5mqedkuiEAAIAoXO5xbxU6U9Id7r69pDvSOJAmn2dL2kvSLElnm9m4dPF/u/sOknaTtI+ZvavUE5JYAgAAxJKLfKvMEZKuSv++StKRBdY5RNJt7r4y3Zt5m6RD3X29u98lSe6+SdKjUmaalQKosQQAAIjAXfJc9KKDLc0svzbucne/vI+PnejuPXMlLpM0scA6UyQtyosXp/e9yszGSjpM0ndKPWFdJ5ZHHXVUEJ9xxhlBnL1WdylvfvObg7jceSevuOKKIH722WeLrn/DDTcE8RNPPFHW8yGOrlFhN89eF9k9M1dcplsN3xTO0Wcexh3DwhrMnMI6qqbMdaFHZa673JS5pH1TZu66t405Kdx+Zu64/d9/uirh9XXZ6oZQsg9lKsHatEUYWxhnazCbyuxDpQ4+7T1h26LLK+1D6JvsWFRKtqayaVO4+yp7FkT3sHD9sue1zNR0Zsei3Iiw/eVey7yUbE1lw4p/VvjyYjWWZna7pK0LLPpC2Cz3/pw7Y2Ytkq6VdLG7P1Nq/bpOLAEAABpJFfZYFn8+9wN7W2ZmL5jZJHdfamaTJL1YYLUlkmbnxVMl3Z0XXy7pKXf/dl/aQ40lAABADC4p53FvlZkraU769xxJvy6wzq2SDjazcelJOwen98nMzpc0RtL/6+sTklgCAABEUXdnhV8o6SAze0rSgWksM5tpZj+UJHdfKek8SQ+nt3PdfaWZTVVyOH0nSY+a2Z/N7MRST1jX81iiMsxjiRjqYe44+lBjq4c+JNGPGl299KNi9njzW/z+a38XdZvD3rJ1v+exrAVqLAEAACKJsJexoZFYAgAAxNBTYzmEkVgCAABEMtBnhdcbEksAAIBYOBQOAACASrk7eyxr3QAAAIBBo/Lrezc0EksAAIBIOCscAAAAleOscBJLAACAaEgsAQAAEAOHwgEAAFA5Fyfv1LoBAAAAgwPTDZFYAgAAxMKhcAAAAFTMuaQjiSUAAEAsJJYAAAColIuzwkksAQAAYnCXd3bXuhU1RWIJAAAQAzWWJJYAAABxuLx7aE9kWTSxdHcbqIZg8Gpvb6cfoSL0IcRAP0LVuaQciSUAAAAq5JK8e2gfCm+qdQMAAAAGBXd5Lhf1VgkzG29mt5nZU+n/x/Wy3px0nafMbE6B5XPN7G99eU4SSwAAgEi8Oxf1VqEzJd3h7ttLuiONA2Y2XtLZkvaSNEvS2fkJqJkdJWltX5+QxBIAACCGnhrLmLfKHCHpqvTvqyQdWWCdQyTd5u4r3X2VpNskHSpJZjZK0mmSzu/rE1JjCQAAEIVXY7qhLc1sXl58ubtf3sfHTnT3penfyyRNLLDOFEmL8uLF6X2SdJ6kb0ha39fGklgCAADE4KrGdEPL3X1mbwvN7HZJWxdY9IWgae5uZn3Oes1sV0nbuvtnzGxGXx9HYgkAABCBSxWfcFP2c7of2NsyM3vBzCa5+1IzmyTpxQKrLZE0Oy+eKuluSW+VNNPMnlWSL77OzO5299kqYtDUWJrZlWZ2fvr3fmb2ZD+3c5mZfSlu6wAAwKDnLnXn4t4qM1dSz1necyT9usA6t0o62MzGpSftHCzpVne/1N0nu/sMSftK+keppFIa4MTSzJ41sw4zW5tm0VemhaFRufu97v6mPrTnBDP7Q+axJ7v7ebHbVOC5h5nZt8zseTNbZWbfM7PWaj8vAACoHs951FuFLpR0kJk9JenANJaZzTSzH0qSu69UUkv5cHo7N72vX2pxKPwwd7/dzKYoyZK/qMzp72bW4u5dNWjbQDpT0kxJO0tqlvQbJa/F2bVsFAAA6Kfq1Fj2m7uvkHRAgfvnSToxL75C0hVFtvOsknylpJodCnf3JZL+T2lDzczN7JQ0q34qve+9ZvZnM1ttZn80s116Hm9mu5nZo2a2xsyulzQ8b9lsM1ucF08zsxvN7CUzW2Fml5jZjpIuk/TWdA/q6nTdVw+pp/FJZrbAzFamE4ROzlvmZnZyOqHoajP7rpn19ZJhh0m6OD29/yVJF0v6SJkvIwAAqBv1NUF6LdQssTSzaZLeLelPeXcfqWSCzp3MbDcl2fPHJU2Q9H1Jc9NDyG2SfiXpGknjJf1c0tG9PE+zpN9Kek7SDCWn0F/n7o9LOlnS/e4+yt3HFnjs/pK+KukDkial27gus9p7Je0paZd0vUPSx05Pk83pxV6GzN9TzWxMkfUBAEC9ckndHvfWYGpxKPxXZtYl6WVJN0n6St6yr/Yc1zezj0n6vrs/mC67yszOkrS3kreuVdK33d0l/cLMTuvl+WZJmizp9LzD63/oZd2s4yVd4e6Ppm36vKRVZjYj3S0sSRe6+2pJq83sLkm7SrrF3RdKGltk27dI+nT6mGZJn0rvH6HktQEAAA2mEfcyxlSLxPJId7+9l2X5E3RuI2mOmX0y7742JUmiS1qSJpU9nutlm9MkPdfPms3Jkh7tCdx9rZmtULLX89n07mV566+X1NeTkS5Qknj+WdJGST+QtJukF/rRTgAAUGPuXlc1lrVQb9MN5SeKiyRd4O5j824j3P1aSUslTcnUM/Z2yHmRpOlmViiJLrWP+XklCa4kycxGKjksv6TUP6QUd+9w91PdfYq7v0HSCkmPuPvQ7pEAADQwaizr1w8knWxme1lipJm9x8xGS7pfUpekT5lZa3qB9Fm9bOchJYnohek2hpvZPumyF5TUNbb18thrJf27me1qZsOUHLZ/MO8weL+Z2RQzm5z+2/aW9CVxRjgAAI2LGsv6TSzTU+FPknSJpFWSFkg6IV22SdJRabxS0rGSbuxlO91KzsDeTtJCJdfAPDZdfKek+ZKWmdnyAo+9XUnCd4OS5HRbScf1pf3pyTtri5y8s62kP0pap+TC8Ge6++/6sm0AAFCPOCvcwjJFAAAA9Mcuk7f335z0rajbnHHuYY8Uu1Z4veFa4QAAAFF4Q+5ljInEEgAAIIaeGsshjMQSAAAgAhfzWJJYAgAAxOAu7yKx7JWZDe39uQ3O3ft63fKq6ujooB81sPb29pr3I/pQY6uHPiTRjxpdvfSjolxDfoJ09lgCAADE4GKPZa0bAAAAMDhwSUcSSwAAgAicPZYklgAAAFFw8g6JJQAAQCzOPJYAAACoGIfCSSwBAACicE7eaap1AwAAAAaDnpN3Yt4qYWbjzew2M3sq/f+4Xtabk67zlJnNybu/zcwuN7N/mNkTZnZ0qedkjyUAAEAM9XfyzpmS7nD3C83szDT+XP4KZjZe0tmSZiq5KuUjZjbX3VdJ+oKkF939jWbWJGl8qScksQQAAIikzg6FHyFpdvr3VZLuViaxlHSIpNvcfaUkmdltkg6VdK2kj0jaQZLcPSdpeaknJLEEAACIoTon72xpZvPy4svd/fI+Pnaiuy9N/14maWKBdaZIWpQXL5Y0xczGpvF5ZjZb0tOSTnX3F4o9IYklAABADNU5FL7c3Wf2ttDMbpe0dYFFXwib5m5m5cyF1CJpqqQ/uvtpZnaapP+W9OFSDwIAAECFXAN/KNzdD+xtmZm9YGaT3H2pmU2S9GKB1ZbotcPlUpJM3i1phaT1km5M7/+5pI+Wag9nhQMAAMSQ7rGsl7PCJc2V1HOW9xxJvy6wzq2SDjazcelZ4wdLutXdXdJv9FrSeYCkv5d6QvZYAgAAxOB1d/LOhZJ+ZmYflfScpA9IkpnNlHSyu5/o7ivN7DxJD6ePObfnRB4lJ/pcY2bflvSSpH8v9YQklgAAADHU2ZV33H2Fkj2N2fvnSToxL75C0hUF1ntO0tvLeU4SSwAAgCjqbh7LAUdiCQAAEIG75N3lnHg9+JBYAgAAxFBnh8JrgcQSAAAgCq+3k3cGHIklAABADOyxJLEEAACIojpX3mkoJJYAAAAReP3NYzngSCwBAAAiIbEEAABABK6ck1gCAACgQu5SzpnHEgAAABF0s8dy6Nhuu+2CeMsttwzi973vfUE8e/bsIM7lws5y2WWXBfF9990XxAsWLOhPM1HnLHNVBcv8OH3R5gfx/Yt/FsQHbfPpIB6mseH2ZJlnbCq7jWhsy7v/FsT0IRRSaizKtWT7AarNORQ+tBJLAACAauJQOAAAAKJgjyUAAAAq5s6h8EGVWO68885BfOqppwbxUUcdFcTZGsty7bXXXkHc1dUVxE8++WQQ/+EPfwjiT386rJPatGlTRe1BHJYZE5o2hXdsHN4RxMs7w/d5ZPOEID5sm3OCeL2/EMRdFm5vYccDQTy1fY9w+2tGB3H36PBjPLQPwjSGLq0L4s36UEt1+9AwjetzW1E7pcai7uFh7exAf/Zb1oTfeYxFCQ6FAwAAoGIuDoWTWAIAAEThTDdU6wYAAAAMBskeSw6FN4xddtkliE855ZQgPvbYY4N4iy22KLq9JUuWBPG9994bxP/85z+D+IwzzgjiRx55JIhnzZoVxOPHjw/id7/73UH8l7/8JYiz82KiOnIKa1m7tD6IvSkcFFY0h/ORjvCxQbxkXTjn4LI1Yb/ZZ+oHw+f3sC7poZeuDZfnuoN4Rvu+Qbx61NIg3qJrahA7c9dVXck+lKkuW9GZ6UMtY4N4oPvQWl8UxKNsmlB/PDP9aLamsta6MjWVTV1hvx+SY5FzKLyhEksAAIB65XJ1ZX7YDTX19fMHAACggeU8F/VWCTMbb2a3mdlT6f8LTglhZnPSdZ4yszl59/+rmf3VzB4zs1vMrOR0OiSWAAAAEXh68k7MW4XOlHSHu28v6Y40DpjZeElnS9pL0ixJZ5vZODNrkfQdSe90910kPSbp1Ozjs+r6UPj3v//9IM5ey7vUPJR33HFHEP/1r38N4rPOOiuIN2zYUHR7b3vb24L4E5/4RBBfccUVQbzrrrsG8QsvhHPPffe73w3iG264IYhfeumlou1BX4UfzH92/D6Ip7TvFsQtGhHE07p2D+KmdeH2xo0Or0HvY4vXFS353NwgPuC/wvlMmzaEh1E2qTmIR2fq4awzc9ilJVwfMVTWhya37ll063uM3T68Y2yJ5mS62OzX7Vh0dVfYR6ipbEzNHeH72LQx7JfZmkdvLjEWnRGORdP/6z3h9jNjUee4tiDObt06M0nQUByL6q/G8ghJs9O/r5J0t6TPZdY5RNJt7r5SkszsNkmHSvqFkrd5pJmtkLSFpAUqoa4TSwAAgEZRpbPCtzSzeXnx5e5+eR8fO9Hde874XCZpYoF1pkjKP6NvsaQp7t5pZp+Q9FdJ6yQ9JemUAo8PkFgCAABEUZVLOi5395m9LTSz2yVtXWDRF4KWubuZ9TnrNbNWSZ+QtJukZyT9j6TPSzq/2ONILAEAACKoxZV33P3A3paZ2QtmNsndl5rZJEkvFlhtiV47XC5JU5UcMt813f7T6bZ+pgI1mlk1TSyHDx8exNl5Ik888cQgNgsrOrI1iJdeemkQf/3rXw/idevC6/OWa8KE8Pq9zc1h/cg555wTxLfccksQb7PNNhU9PwrL1vnkMvVkw1aE8cgtwvlF13wjrL2d+Ol3hNtrD5+hu72yuqGpXz0siLNHTbozdVHZf1/rsrAWuGtCmxBXtibRMnWuI1vCPjRc4dhQb2yzXoRqyL7K2V1DbSvC+U83ZT67L33j7iDefCwK++FAj0VZjEWF1dkE6XMlzZF0Yfr/XxdY51ZJX8k7Y/xgJXsmh0vaycy2cveXJB0k6fFST8geSwAAgAjc6+6SjhdK+pmZfVTSc5I+IElmNlPSye5+oruvNLPzJD2cPubcvBN5/kvS782sM338CaWekMQSAAAgkno6K9zdV0g6oMD98ySdmBdfIemKAutdJqmsywKSWAIAAERSZ4fCB1xNE8vZs2cH8emnnx7E2ZrK7LW9jz766CB+6KGHKmpPtmZy2rRwrrerr746iG+++eYgHjeu4IT2r8r+e6655pogXr16dV+aiRIWb3wgiMeNnx7Ejz0f1r4e+smwFjkX/fq24a/X7ubwOs/rPZzfdEz3lCBuWRM+3jN1VbnW8DoHpeq8UNpmfagt04deDPvQ1lPCuU7jy2Wi4n1o83kquRZGPcjWVGZN/OTbgzj+WBTKznOZHSuaN4b3tKwJ+x1j0ea8OmeFNxT2WAIAAESwWKtvPc1vLHnZwzItj7y9qiKxBAAAiMDdD611G2qN4yMAAACIoqZ7LLM1jd3d3b2smejqCus79tprryA+5phjgniHHXYour2Ojo4g3nHHHYvGy5eHe6MnTix0ZaTeZa8Vfv754eT1nZ2dZW0Piab1Yb+ZPGKPIN6QzJrwqs9d9KMgnn3xfwTx8s6ngjg7Z2G3h3PRjW96UxA/3/lIEG/VGvajESvDOqu2lvCCCdYd9nNlanOtK1tvF/abJrUKlZk8rL77UKtGBXGl1/6mD8WRHYu6R5Q3z2SubWD39bSuDPuhZ2o6rTtTFVliLMoaCjWV2Bx7LAEAABAFiSUAAACiILEEAABAFDWtsbzzzjuD+K677griAw8Mr6s+fXo4l9zFF18cxF5iUtJsDWe2xrOUUjWVuVxYb/LLX/4yiD/1qU8F8dKlS8t6fhTWPSJzLW+FdUPttlUQv2HHYUE8f/VNQfy6EdsG8SgL55VclVsQxGt8URC3WFhDma2Hy43IzEuZ6YarW54N4jH2BhXDr8MYMnON1nkfKuUVfyaItyjZh6ipjKHcmspay40IU4DsWFRqXkqgEL6TAAAAEAWJJQAAAKIgsQQAAEAUVqwu0cxqOg3V2LFjg/jMM8NrOu+zzz5BvGLFiiBeuHBhEA8bFtZFveUtbwniWbNm9aeZr7rsssuC+Kyzzgrigb4WuLvXRUlMR0dHjaczC+vnmrrCl+Xzdx4QxGNHjwzi/d54WBBv6F4bLn9dOIfh8EWZuewmjQji5rXhPJXrHl0WxCN3D+e17Bxb2/q39vb2mvejeutD2d/kX/jd/kFcaR9q1vB+tLF+1UMfkuqhH4WausLmxL42eOuidUHMWISBwB5LAAAAREFiCQAAgChILAEAABBFXddYVtvVV18dxB/60IeKrr9mzZogPu2004L4yiuvDOJS1z6vNmosB0bLK+F1li1Tjtc1JqxLWuGPB/GL658O4h1GvTde4yKoh7qmwd6HyrUyRx/qj8Hej8odi+rjG6Lv6qUfoTj2WAIAACAKEksAAABEQWIJAACAKIZUjeUZZ5wRxOeff34Qt7QUv3T68ccfH8TXXnttnIZVCTWWAyOnsK6ppSt7/d3M25AJ6/3FqYe6pqHWhwbbtbvroQ9Jg78fZWXnyWQswkBgjyUAAACiILEEAABAFCSWAAAAiKJ4UWGDO/HEE4P4i1/8YhCXqqmcP39+EN94441xGoaG0rwhnAzOMz/HOts2hndkulVrR/iAXHtzrKZhkOjWxqLLB1vNJfqn1FiUa8uMNZlrjzd3hHMrMxahGthjCQAAgChILAEAABAFiSUAAACiGFQ1lrNmzQrib3zjG0E8atSooo9fu3ZtEJ988slBvHFj8TooDE7ZOibrDmd7a8uF/WqNLQ7i0cOmhtuL1zQMEq0K+9BaD/vQKAv7EIamUmNR9trgm9VgDgtrKhmLUA3ssQQAAEAUJJYAAACIgsQSAAAAUQyqGsvDDjssiEePHl10/XXr1gXx4YcfHsT33XdfnIahoWx2Mdqm8J7sXHGv+HNBPKZ7erj+oPqUoRrWZPrQaNumRi1BPSl3LMrWTGavFZ6d1xKoBvZYAgAAIAoSSwAAAERBYgkAAIAoGrr6K1tDecYZZ5T1+J/85CdBfPfdd1faJDQgyxQmZeeGy9YlvexPB/Foy9ZUUseE4kr1IQxN5Y5FpTAWoRbYYwkAAIAoSCwBAAAQBYklAAAAojD33q8Wapat+Kit7LW+H3/88SCeMmVK0cc/9thjQbz33nsH8YYNGypoXf1x97oosOno6KirfpR9UTq0IohzvimI2+11QdyUC6+3m/15Vlf/2Aja29tr3o/qrQ9lbSizD5kyfWiQq4c+JNVfP8q+KOU2LnttcMYi1AP2WAIAACAKEksAAABEQWIJAACAKBpqHsv9998/iKdOnRrExepFJekzn/lMEA+2mkr0UaabtNiIIN6ksD4uW1Npuczcck2U/Qx1LSreh4ZaTSX6KPuVVWIoydZUMhahHrHHEgAAAFGQWAIAACAKEksAAABE0VA1luedd14Ql6qp/PrXvx7Ed911V/Q2ofHkrDuIO31NEN/23CVBfMiM04J4eNOE6jQMDcNVYR8SfQhSuTMNe3aeSmoqUYfYYwkAAIAoSCwBAAAQBYklAAAAomioGsvx48cHsVlYX/Liiy8G8be//e1qNwkNqFNhPZxZOMfgzhPfGcTtmXq4wXb9XZSv3D5ETSViqPTa4sBAYI8lAAAAoiCxBAAAQBQklgAAAIiioWosv/nNbxaNs/NcLl26tOptQuPJZeYg/OEjpwTxx/a4LIipY0JWuX0IiIGxCI2APZYAAACIgsQSAAAAUZBYAgAAIAordr1tM6Oko4G5l3sl2uro6OigHzWw9vb2mvcj+lBjq4c+JNGPGl299CMUxx5LAAAAREFiCQAAgChILAEAABBF0RpLAAAAoK/YYwkAAIAoSCwBAAAQBYklAAAAoiCxBAAAQBQklgAAAIiCxBIAAABR/H+Ko7PNz7BAUwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x504 with 13 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(nrows=3, ncols=4, figsize=(10, 7))\n",
    "image_ids = [0, 1, 9]\n",
    "cmap_bound = np.abs(attrs[[0, 1, 9]]).max()\n",
    "\n",
    "for row, image_id in enumerate(image_ids):\n",
    "    # original images\n",
    "    ax[row, 0].imshow(X_test[image_id].squeeze(), cmap='gray')\n",
    "    ax[row, 0].set_title(f'Prediction: {predictions[image_id]}')\n",
    "    \n",
    "    # attributions\n",
    "    attr = attrs[image_id]\n",
    "    im = ax[row, 1].imshow(attr.squeeze(), vmin=-cmap_bound, vmax=cmap_bound, cmap='PiYG')\n",
    "    \n",
    "    # positive attributions\n",
    "    attr_pos = attr.clip(0, 1)\n",
    "    im_pos = ax[row, 2].imshow(attr_pos.squeeze(), vmin=-cmap_bound, vmax=cmap_bound, cmap='PiYG')\n",
    "    \n",
    "    # negative attributions\n",
    "    attr_neg = attr.clip(-1, 0)\n",
    "    im_neg = ax[row, 3].imshow(attr_neg.squeeze(), vmin=-cmap_bound, vmax=cmap_bound, cmap='PiYG')\n",
    "    \n",
    "ax[0, 1].set_title('Attributions');\n",
    "ax[0, 2].set_title('Positive attributions');\n",
    "ax[0, 3].set_title('Negative attributions');\n",
    "\n",
    "for ax in fig.axes:\n",
    "    ax.axis('off')\n",
    "\n",
    "fig.colorbar(im, cax=fig.add_axes([0.95, 0.25, 0.03, 0.5]));"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}