{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "import innvestigate\n",
    "import innvestigate.utils as iutils\n",
    "from innvestigate.utils.visualizations import heatmap\n",
    "\n",
    "from keras.models import Model\n",
    "from keras.layers import concatenate, Input, Conv2D, MaxPooling2D, Conv2DTranspose, Activation, Cropping2D, Lambda, Flatten\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "import matplotlib.pyplot as plot\n",
    "import numpy as np\n",
    "import os\n",
    "%matplotlib inline  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets('MNIST_DATA', one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ConvBNRelu(input_tensor, filters):\n",
    "    out = Conv2D(filters, (3, 3), activation=None, padding='valid', use_bias=False)(input_tensor)\n",
    "    out = BatchNormalization(axis=-1)(out)\n",
    "    return Activation('relu')(out)\n",
    "\n",
    "\n",
    "def get_model(weights=None, flatten_output=False):\n",
    "    filters = 4\n",
    "    inputs = Input((28, 28, 1), name=\"input\")\n",
    "\n",
    "    conv1 = ConvBNRelu(inputs, filters)\n",
    "    conv2 = ConvBNRelu(conv1, filters)\n",
    "\n",
    "    pool1 = MaxPooling2D(pool_size=(2, 2))(conv2)\n",
    "    \n",
    "    conv3 = ConvBNRelu(pool1, 2*filters)\n",
    "    conv4 = ConvBNRelu(conv3, 2*filters)\n",
    "\n",
    "    up1 = Conv2DTranspose(filters, (2, 2), strides=(2, 2), activation='relu', padding='valid')(conv4)\n",
    "    up1 = concatenate([up1, Cropping2D(cropping=(4, 4))(conv2)])\n",
    "    \n",
    "    conv5 = ConvBNRelu(up1, filters)\n",
    "    conv6 = ConvBNRelu(conv5, filters)\n",
    "\n",
    "    logits = Conv2D(1, (1, 1), activation=\"sigmoid\")(conv6)\n",
    "    \n",
    "    if flatten_output:\n",
    "        logits = Flatten()(logits)\n",
    "\n",
    "    model = Model(inputs=[inputs], outputs=[logits])\n",
    "    model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\")\n",
    "    if weights is not None:\n",
    "        model.load_weights(weights)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "55000/55000 [==============================] - 66s 1ms/step - loss: 0.1222\n",
      "Epoch 2/2\n",
      "55000/55000 [==============================] - 58s 1ms/step - loss: 0.0526\n"
     ]
    }
   ],
   "source": [
    "model = get_model()\n",
    "\n",
    "\n",
    "batch_size = 32\n",
    "\n",
    "x = mnist.train.images\n",
    "count = len(x)\n",
    "x = np.reshape(x, (count, 28, 28, 1))\n",
    "y = np.where(x > 0.5, 1, 0)\n",
    "y = y[:, 8:-8, 8:-8, :]\n",
    "model.fit(x, y, batch_size=batch_size, epochs=2)\n",
    "model.save_weights(\"params\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAADZxJREFUeJzt3X+o1fUdx/HXe6YUFf1g6SSdN+2Xqz9c3WJRDNcyagQ2aNaFlquxu8Igw2AiQf7RIIZmg6C40WUG022xftxibGoEJq6lhnjbbCvCplOumqVXikJ974/7NW52v59zPOf7Pd9z7/v5ALnnfN/fH28Ovu73e+73x8fcXQDi+UbVDQCoBuEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxDUKa3cmJlxOSFQMne3euZras9vZjeZ2b/N7H0zW9zMugC0ljV6bb+ZjZP0H0lzJO2StElSl7v/K7EMe36gZK3Y818t6X13/8Ddv5D0B0lzm1gfgBZqJvznS9o57P2ubNpXmFm3mW02s81NbAtAwZr5g99IhxZfO6x39x5JPRKH/UA7aWbPv0vS1GHvp0ja3Vw7AFqlmfBvknSRmV1gZhMk3SGpr5i2AJSt4cN+dz9iZvdL+pukcZJ63f2fhXUGoFQNn+praGN85wdK15KLfACMXoQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8E1fAQ3ZJkZjskDUo6KumIu3cW0RSA8jUV/swP3H1/AesB0EIc9gNBNRt+l7TGzLaYWXcRDQFojWYP+691991mNlHSWjN7193XD58h+6XALwagzZi7F7Mis6WSDrv7ssQ8xWwMQC53t3rma/iw38xON7Mzj7+WdKOkdxpdH4DWauawf5KkF83s+HpWuftfC+kKQOkKO+yva2Mc9gOlK/2wH8DoRviBoAg/EBThB4Ii/EBQhB8Iqoi7+lCxu+++O7dW61TuRx99lKzPnDkzWd+4cWOyvmHDhmQd1WHPDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBjZnz/F1dXcn6FVdckaynzpW3u7PPPrvhZY8ePZqsT5gwIVn/7LPPkvVPP/00t9bf359cdt68ecn6vn37knWksecHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaBG1aO7ly9fnlt74IEHksuOGzeumU2jAq+//nqyXuvajoGBgSLbGTV4dDeAJMIPBEX4gaAIPxAU4QeCIvxAUIQfCKrmeX4z65V0i6S97n55Nu1cSX+U1CFph6R57v5xzY01eZ5/586dubUpU6Ykl922bVuyXuu+9DLVerb9Sy+91KJOTt6cOXOS9bvuuiu31tHR0dS2a10HcPvtt+fWxvKzAIo8z/87STedMG2xpNfc/SJJr2XvAYwiNcPv7uslHThh8lxJK7PXKyXdWnBfAErW6Hf+Se6+R5KynxOLawlAK5T+DD8z65bUXfZ2AJycRvf8A2Y2WZKyn3vzZnT3HnfvdPfOBrcFoASNhr9P0vzs9XxJLxfTDoBWqRl+M1st6e+SLjGzXWb2c0mPSZpjZu9JmpO9BzCKjKr7+S+++OLc2mWXXZZcdt26dcn64OBgQz0hbfr06bm1V199NbnszJkzm9r2Qw89lFtLPRtitON+fgBJhB8IivADQRF+ICjCDwRF+IGgRtWpPowtt912W7L+/PPPN7X+/fv359bOO++8ptbdzjjVByCJ8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4IqfbguxHbffffl1q666qpSt33qqafm1q688srkslu2bCm6nbbDnh8IivADQRF+ICjCDwRF+IGgCD8QFOEHgqr53H4z65V0i6S97n55Nm2ppF9I2pfNtsTd/1JzYzy3vxSTJ0/Ord15553JZRcuXFh0O1+R6s2srsfLl+LQoUPJ+llnndWiTopX5HP7fyfpphGmr3D3Wdm/msEH0F5qht/d10s60IJeALRQM9/57zezbWbWa2bnFNYRgJZoNPxPSZohaZakPZKW581oZt1mttnMNje4LQAlaCj87j7g7kfd/ZikZyRdnZi3x9073b2z0SYBFK+h8JvZ8D/h/ljSO8W0A6BVat7Sa2arJc2W9E0z2yXpEUmzzWyWJJe0Q9IvS+wRQAlqht/du0aY/GwJvYR1ww03JOu17j3v7u7OrU2fPr2hnsa63t7eqluoHFf4AUERfiAowg8ERfiBoAg/EBThB4Li0d0FuPDCC5P1p59+Olm//vrrk/Uyb3398MMPk/WPP/64qfU//PDDubXPP/88ueyTTz6ZrF9yySUN9SRJu3fvbnjZsYI9PxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ExXn+Oj344IO5tQULFiSXnTFjRrJ++PDhZP2TTz5J1p944oncWq3z2Rs3bkzWa10HUKaDBw82tfzg4GBu7ZVXXmlq3WMBe34gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrz/HW65pprcmu1zuP39fUl68uX5452Jklav359sj5azZo1K1mfNm1aU+tPPS/g3XffbWrdYwF7fiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IquZ5fjObKuk5Sd+SdExSj7v/1szOlfRHSR2Sdkia5+7NPeS9jd177725tW3btiWXffTRR4tuZ0yoNd7BpEmTmlr/unXrmlp+rKtnz39E0iJ3nynpe5IWmNl3JC2W9Jq7XyTptew9gFGiZvjdfY+7v529HpS0XdL5kuZKWpnNtlLSrWU1CaB4J/Wd38w6JH1X0j8kTXL3PdLQLwhJE4tuDkB56r6238zOkPRnSQvd/VC948eZWbek7sbaA1CWuvb8ZjZeQ8H/vbu/kE0eMLPJWX2ypL0jLevuPe7e6e6dRTQMoBg1w29Du/hnJW1398eHlfokzc9ez5f0cvHtASiLuXt6BrPrJL0hqV9Dp/okaYmGvvf/SdK3Jf1X0k/c/UCNdaU3hlCWLVuWrC9atChZr/VI85tvvjm39uabbyaXHc3cva7v5DW/87v7Bkl5K/vhyTQFoH1whR8QFOEHgiL8QFCEHwiK8ANBEX4gKB7djVL19/fn1i699NKm1r1mzZpkfSyfyy8Ce34gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrz/ChVR0dHbu2UU9L//Q4ePJisr1ixopGWkGHPDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBcZ4fTenq6krWTzvttNza4OBgctnu7vQob9yv3xz2/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QlLl7egazqZKek/QtScck9bj7b81sqaRfSNqXzbrE3f9SY13pjaHtjB8/Pll/6623kvXUs/lXr16dXPaee+5J1jEyd7d65qvnIp8jkha5+9tmdqakLWa2NqutcPdljTYJoDo1w+/ueyTtyV4Pmtl2SeeX3RiAcp3Ud34z65D0XUn/yCbdb2bbzKzXzM7JWabbzDab2eamOgVQqLrDb2ZnSPqzpIXufkjSU5JmSJqloSOD5SMt5+497t7p7p0F9AugIHWF38zGayj4v3f3FyTJ3Qfc/ai7H5P0jKSry2sTQNFqht/MTNKzkra7++PDpk8eNtuPJb1TfHsAylLPX/uvlfRTSf1mtjWbtkRSl5nNkuSSdkj6ZSkdolK1TgWvWrUqWd+6dWtube3atbk1lK+ev/ZvkDTSecPkOX0A7Y0r/ICgCD8QFOEHgiL8QFCEHwiK8ANB1bylt9CNcUsvULp6b+llzw8ERfiBoAg/EBThB4Ii/EBQhB8IivADQbV6iO79kj4c9v6b2bR21K69tWtfEr01qsjeptU7Y0sv8vnaxs02t+uz/dq1t3btS6K3RlXVG4f9QFCEHwiq6vD3VLz9lHbtrV37kuitUZX0Vul3fgDVqXrPD6AilYTfzG4ys3+b2ftmtriKHvKY2Q4z6zezrVUPMZYNg7bXzN4ZNu1cM1trZu9lP0ccJq2i3paa2f+yz26rmf2oot6mmtnrZrbdzP5pZg9k0yv97BJ9VfK5tfyw38zGSfqPpDmSdknaJKnL3f/V0kZymNkOSZ3uXvk5YTP7vqTDkp5z98uzab+RdMDdH8t+cZ7j7r9qk96WSjpc9cjN2YAyk4ePLC3pVkk/U4WfXaKveargc6tiz3+1pPfd/QN3/0LSHyTNraCPtufu6yUdOGHyXEkrs9crNfSfp+VyemsL7r7H3d/OXg9KOj6ydKWfXaKvSlQR/vMl7Rz2fpfaa8hvl7TGzLaYWXfVzYxgUjZs+vHh0ydW3M+Jao7c3EonjCzdNp9dIyNeF62K8I/0iKF2OuVwrbtfIelmSQuyw1vUp66Rm1tlhJGl20KjI14XrYrw75I0ddj7KZJ2V9DHiNx9d/Zzr6QX1X6jDw8cHyQ1+7m34n6+1E4jN480srTa4LNrpxGvqwj/JkkXmdkFZjZB0h2S+iro42vM7PTsDzEys9Ml3aj2G324T9L87PV8SS9X2MtXtMvIzXkjS6viz67dRryu5CKf7FTGE5LGSep191+3vIkRmNl0De3tpaE7HldV2ZuZrZY0W0N3fQ1IekTSS5L+JOnbkv4r6Sfu3vI/vOX0NltDh65fjtx8/Dt2i3u7TtIbkvolHcsmL9HQ9+vKPrtEX12q4HPjCj8gKK7wA4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8Q1P8Bp+YC7BbcNBcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x14552e1d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAADFpJREFUeJzt3X+o3XUdx/HXq91GzYzp0pjbxTnQpQRpHsMSIvcD7AetP0ocLC2Ci5I/FkGsEP23PzIWmMEwS0gmucSGSCVrMcQYHqcw5y2UW+pty92Wd+YUt7l3f5wTrOvkXs/38/1+z3w/HyD33uPh/X2z7Xm/55x77jmOCAHI5X1tLwCgeYQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEIjTR5s0aJFMTo62uQhGzEy0ugf45yUekam7SJzjh8/PlRz3njjjSJzJGliYqLInGPHjhWZExGz/qU1+i92dHRU27dvb/KQjVi4cGHbK7zNW2+9VWTO/Pnzi8wpFdrrr79eZM6ePXuKzJGkq6++usicqampInPmgpv6QEKEDyRE+EBChA8kVCl821fZ/qvt521vLLUUgHoNHL7teZJ+Kunzki6StM72RaUWA1CfKmf8T0l6PiImIuKIpPslrS2zFoA6VQl/iaSXTvh6sn/Z/7E9Zrtru3vw4MEKhwNQSpXwT/bsoLc9XSwiNkdEJyI6ixYtqnA4AKVUCX9S0onPv10qaV+1dQA0oUr4T0g63/Z5tudLukbStjJrAajTwM/Vj4hjtm+U9HtJ8yTdExF7i20GoDaVfkknIh6R9EihXQA0hGfuAQkRPpAQ4QMJucn3zut0OtHtdhs7HtCEHTt2FJmzbt26yjMOHjyoo0ePzvoKPJzxgYQIH0iI8IGECB9IiPCBhAgfSIjwgYQIH0iI8IGECB9IiPCBhAgfSIjwgYQIH0iI8IGECB9IiPCBhCq9yu67dfjwYe3atavynMcee6zANtJDDz1UZA5mt2bNmiJzrr322iJzli1bVmSOJF155ZVF5mzZsqXyjOuvv35O1+OMDyRE+EBChA8kRPhAQoQPJDRw+LZHbe+wPW57r+1bSi4GoD5Vfpx3TNJ3I2K37dMlPWn70Yh4ttBuAGoy8Bk/IvZHxO7+5/+RNC5pSanFANSnyH1828skXSKp+rNzANSucvi2PyTpN5I2RMSrJ/n/Y7a7trvT09NVDweggErh236/etHfFxEPnuw6EbE5IjoR0Vm4cGGVwwEopMqj+pb0c0njEfHjcisBqFuVM/4Vkr4uaaXtp/v/faHQXgBqNPCP8yLiMUmzvg83gOHDM/eAhAgfSIjwgYQcEc0dzG7uYHhPWr58eZE5Dz/8cJE5knThhRcWm1VVp9NRt9ud9bE3zvhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8kVOXdcoHGTUxMFJlz2223FZkjSQ888ECxWU3hjA8kRPhAQoQPJET4QEKVw7c9z/ZTtsu9UDmAWpU4498iabzAHAANqRS+7aWSvijp7jLrAGhC1TP+Jknfk3S8wC4AGjJw+La/JOlARDw5y/XGbHdtdwc9FoCyqpzxr5D0Zdt/l3S/pJW2fzXzShGxOSI6EdGpcCwABQ0cfkR8PyKWRsQySddI+mNErC+2GYDa8HN8IKEiv6QTEX+S9KcSswDUjzM+kBDhAwkRPpAQ4QMJET6QEOEDCRE+kBDhAwkRPpAQ4QMJET6QEOEDCRE+kBDhAwkRPpAQ4QMJET6QEOEDCRE+kBDhAwkRPpAQ4QMJFXl5baApN9xwQ5E5l112WZE5pyrO+EBChA8kRPhAQoQPJFQpfNsLbW+1/Rfb47Y/XWoxAPWp+qj+TyT9LiK+anu+pAUFdgJQs4HDt/1hSZ+V9A1Jiogjko6UWQtAnarc1F8uaUrSL2w/Zftu26cV2gtAjaqEPyLpk5J+FhGXSDosaePMK9kes9213a1wLAAFVQl/UtJkROzqf71VvW8E/yciNkdEJyI6FY4FoKCBw4+If0p6yfaK/kWrJD1bZCsAtar6qP5Nku7rP6I/Iemb1VcCULdK4UfE05K4CQ+cYnjmHpAQ4QMJET6QEOEDCfEKPAUsXry42Kz169cXmbNhw4Yic4ZNqT9r20XmnKo44wMJET6QEOEDCRE+kBDhAwkRPpAQ4QMJET6QEOEDCRE+kBDhAwkRPpAQ4QMJET6QEOEDCRE+kBDhAwmdkq/As3r16iJzLr300iJzxsbGisyRpOXLlxebBbwTzvhAQoQPJET4QEKEDyRE+EBClcK3/R3be20/Y3uL7Q+UWgxAfQYO3/YSSTdL6kTExyXNk3RNqcUA1KfqTf0RSR+0PSJpgaR91VcCULeBw4+If0j6kaQXJe2XdCgi/jDzerbHbHdtdwdfE0BJVW7qnyFpraTzJJ0j6TTbb3vjt4jYHBGdiOgMviaAkqrc1F8t6W8RMRURRyU9KOkzZdYCUKcq4b8o6XLbC9x769FVksbLrAWgTlXu4++StFXSbkl7+rM2F9oLQI0q/XZeRNwu6fZCuwBoCM/cAxIifCAhwgcSavQVeC644ALdddddleesXLmywDZS74cR700vvPBCkTmvvPJKkTml3HrrrUXmvPnmm0XmSNKdd95ZZM6KFSuKzJkLzvhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8k1OhLb51++ulatWpV5TmvvfZagW2k6enpInM2bdpUZI4k7dtX5n1HH3/88SJzSr2E13vZoUOH2l7hXeOMDyRE+EBChA8kRPhAQrOGb/se2wdsP3PCZWfaftT2c/2PZ9S7JoCS5nLG/6Wkq2ZctlHS9og4X9L2/tcAThGzhh8ROyX9e8bFayXd2//8XklfKbwXgBoNeh//oxGxX5L6H88utxKAutX+4J7tMdtd292pqam6DwdgDgYN/2XbiyWp//HAO10xIjZHRCciOmedddaAhwNQ0qDhb5N0Xf/z6yT9tsw6AJowlx/nbZH0Z0krbE/a/pakH0paY/s5SWv6XwM4Rcz6SzoRse4d/lf137YB0AqeuQckRPhAQoQPJET4QEKNvgLP9PS0tm3bVnnOHXfcUWAbaefOnUXm4NRz8cUXF5t17rnnFpvVFM74QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQo6I5g5mT0l6YZarfUTSvxpYZ67YZ3bDtlPmfc6NiFnfsqrR8OfCdjciOm3v8T/sM7th24l9ZsdNfSAhwgcSGsbwN7e9wAzsM7th24l9ZjF09/EB1G8Yz/gAajY04du+yvZfbT9ve+MQ7DNqe4ftcdt7bd/S9k6SZHue7adsPzwEuyy0vdX2X/p/Tp9ueZ/v9P+unrG9xfYHWtjhHtsHbD9zwmVn2n7U9nP9j2c0vddMQxG+7XmSfirp85IukrTO9kXtbqVjkr4bERdKulzSt4dgJ0m6RdJ420v0/UTS7yLiY5I+oRb3sr1E0s2SOhHxcUnzJF3Twiq/lHTVjMs2StoeEedL2t7/ulVDEb6kT0l6PiImIuKIpPslrW1zoYjYHxG7+5//R71/1Eva3Mn2UklflHR3m3v0d/mwpM9K+rkkRcSRiJhudyuNSPqg7RFJCyTta3qBiNgp6d8zLl4r6d7+5/dK+kqjS53EsIS/RNJLJ3w9qZYjO5HtZZIukbSr3U20SdL3JB1veQ9JWi5pStIv+nc97rZ9WlvLRMQ/JP1I0ouS9ks6FBF/aGufGT4aEful3glF0tkt7zM04fsklw3Fjxtsf0jSbyRtiIhXW9zjS5IORMSTbe0ww4ikT0r6WURcIumwWrwJ27/fvFbSeZLOkXSa7fVt7TPshiX8SUmjJ3y9VC3cTJvJ9vvVi/6+iHiw5XWukPRl239X767QStu/anGfSUmTEfG/W0Fb1ftG0JbVkv4WEVMRcVTSg5I+0+I+J3rZ9mJJ6n880PI+QxP+E5LOt32e7fnqPShT/f20K7Bt9e6/jkfEj9vcRZIi4vsRsTQilqn35/PHiGjtjBYR/5T0ku0V/YtWSXq2rX3Uu4l/ue0F/b+7VRqeB0G3Sbqu//l1kn7b4i6SejfXWhcRx2zfKOn36j0ae09E7G15rSskfV3SHttP9y/7QUQ80uJOw+YmSff1v1lPSPpmW4tExC7bWyXtVu8nMk+phWfM2d4i6XOSPmJ7UtLtkn4o6de2v6XeN6ivNb3XTDxzD0hoWG7qA2gQ4QMJET6QEOEDCRE+kBDhAwkRPpAQ4QMJ/Reqbtxs5iXMCQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x14569e5c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAACwpJREFUeJzt3V+oXXeZh/HnO0mKpo5YSRRNyqRCqS0pQ+1BqgUZGoU6ivFiBlqodETIjX+qCBLnprdeiOiFCKGtFiwtQyxYpKilKjIwBE/aQpom0lKd9thojsioeJO0vnNxtpCeSTmZs9fea5++zwfC/pPNXi85ec5v7X9rp6qQ1MvfjT2ApPkzfKkhw5caMnypIcOXGjJ8qSHDlxoyfKkhw5ca2j7Pje3atav27ds3z01KM3f8+PGxR3iVqspGt5lr+Pv27WN5eXmem5RmLtmws4Xjrr7UkOFLDRm+1JDhSw1NFX6SW5P8MslzSQ4PNZSk2dp0+Em2Ad8EPgxcB9ye5LqhBpM0O9Os+O8Fnquq56vqHPAQcHCYsSTN0jTh7wFevODyyuS6V0lyKMlykuXV1dUpNidpKNOEf7F3LfyfA/hV1ZGqWqqqpd27d0+xOUlDmSb8FeDKCy7vBV6abhxJ8zBN+L8Ark5yVZLLgNuAR4YZS9Isbfq9+lX1cpLPAD8CtgH3VdXJwSaTNDNTfUinqh4FHh1oFklz4jv3pIYMX2rI8KWG5nogDun1aKjvn5znAT1c8aWGDF9qyPClhgxfasjwpYYMX2rI8KWGDF9qyPClhgxfasjwpYYMX2rI8KWGDF9qyPClhgxfasjwpYZaH4Fnnkc86e7EiROD3M/+/fsHuZ9FNMSRfJaWli7pdq74UkOGLzVk+FJDhi81ZPhSQ5sOP8mVSX6a5FSSk0nuGnIwSbMzzct5LwNfrKonkvw9cDzJY1X1zECzSZqRTa/4VXWmqp6YnP8zcArYM9RgkmZnkMf4SfYBNwDHhrg/SbM1dfhJ3gR8D/h8Vf3pIn9/KMlykuXV1dVpNydpAFOFn2QHa9E/UFUPX+w2VXWkqpaqamn37t3TbE7SQKZ5Vj/AvcCpqvracCNJmrVpVvybgU8AtyR5avLnnweaS9IMbfrlvKr6T8CPt0lbkO/ckxoyfKkhw5camusReI4fP+5Rb5q6/vrrxx7hVYY42s1W5oovNWT4UkOGLzVk+FJDhi81ZPhSQ4YvNWT4UkOGLzVk+FJDhi81ZPhSQ4YvNWT4UkOGLzVk+FJDhi81ZPhSQ3M99Ja0KIY8BNxWPIyXK77UkOFLDRm+1JDhSw1NHX6SbUmeTPKDIQaSNHtDrPh3AacGuB9JczJV+En2Ah8B7hlmHEnzMO2K/3XgS8BfB5hF0pxsOvwkHwXOVtXxDW53KMlykuXNbkvSsKZZ8W8GPpbk18BDwC1Jvrv+RlV1pKqWqmppim1JGtCmw6+qL1fV3qraB9wG/KSq7hhsMkkz4+v4UkODfEinqn4G/GyI+5I0e674UkOGLzVk+FJDhi81ZPhSQ4YvNWT4UkOGLzVk+FJDhi81ZPhSQ4YvNWT4UkOGLzVk+FJDhi81ZPhSQ4YvNWT4UkOGLzVk+FJDhi81NMjhtaWt5pVXXhl7hFG54ksNGb7UkOFLDRm+1NBU4Sd5S5KjSU4nOZXkfUMNJml2pn1W/xvAD6vqX5JcBuwcYCZJM7bp8JO8GfgA8G8AVXUOODfMWJJmaZpd/XcBq8C3kzyZ5J4klw80l6QZmib87cB7gG9V1Q3AX4DD62+U5FCS5STLU2xL0oCmCX8FWKmqY5PLR1n7RfAqVXWkqpaqammKbUka0KbDr6rfAi8muWZy1QHgmUGmkjRT0z6r/1nggckz+s8Dn5x+JEmzNlX4VfUU4C68tMX4zj2pIcOXGjJ8qSHDlxryCDwLZvv2YX4k58+fH+R+9Prkii81ZPhSQ4YvNWT4UkOGLzVk+FJDhi81ZPhSQ4YvNWT4UkOGLzVk+FJDhi81ZPhSQ4YvNWT4UkOGLzXU+gg8O3bsGOR+zp3zu0K1tbjiSw0ZvtSQ4UsNGb7UkOFLDU0VfpIvJDmZ5OkkDyZ5w1CDSZqdTYefZA/wOWCpqvYD24DbhhpM0uxMu6u/HXhjku3ATuCl6UeSNGubDr+qfgN8FXgBOAP8sap+vP52SQ4lWU6yvPkxJQ1pml39K4CDwFXAO4HLk9yx/nZVdaSqlqpqafNjShrSNLv6HwR+VVWrVXUeeBh4/zBjSZqlacJ/Abgpyc4kAQ4Ap4YZS9IsTfMY/xhwFHgCODG5ryMDzSVphqb6dF5V3Q3cPdAskubEd+5JDRm+1JDhSw3N9Qg8N954I8vLvo9Hm7f2AtJiqaqxR/h/c8WXGjJ8qSHDlxoyfKkhw5caMnypIcOXGjJ8qSHDlxoyfKkhw5caMnypIcOXGjJ8qSHDlxoyfKkhw5caMnypobkeeuv16tprrx3svk6fPj3YfUmvxRVfasjwpYYMX2rI8KWGNgw/yX1JziZ5+oLr3prksSTPTk6vmO2YkoZ0KSv+d4Bb1113GHi8qq4GHp9clrRFbBh+Vf0c+MO6qw8C90/O3w98fOC5JM3QZh/jv72qzgBMTt823EiSZm3mT+4lOZRkOcny6urqrDcn6RJsNvzfJXkHwOT07GvdsKqOVNVSVS3t3r17k5uTNKTNhv8IcOfk/J3A94cZR9I8XMrLeQ8C/wVck2QlyaeArwAfSvIs8KHJZUlbxIYf0qmq21/jrw4MPIukOfGde1JDhi81ZPhSQ4YvNbQlj8CTZOwRpC3NFV9qyPClhgxfasjwpYYMX2rI8KWGDF9qyPClhgxfasjwpYYMX2rI8KWGDF9qyPClhgxfasjwpYYMX2ooVTW/jSWrwH9vcLNdwO/nMM6lcp6NLdpMnef5h6ra8Cur5hr+pUiyXFVLY8/xN86zsUWbyXk25q6+1JDhSw0tYvhHxh5gHefZ2KLN5DwbWLjH+JJmbxFXfEkztjDhJ7k1yS+TPJfk8ALMc2WSnyY5leRkkrvGngkgybYkTyb5wQLM8pYkR5Ocnvw7vW/keb4w+Vk9neTBJG8YYYb7kpxN8vQF1701yWNJnp2cXjHvudZbiPCTbAO+CXwYuA64Pcl1407Fy8AXq+pa4Cbg0wswE8BdwKmxh5j4BvDDqno38I+MOFeSPcDngKWq2g9sA24bYZTvALeuu+4w8HhVXQ08Prk8qoUIH3gv8FxVPV9V54CHgINjDlRVZ6rqicn5P7P2n3rPmDMl2Qt8BLhnzDkms7wZ+ABwL0BVnauq/xl3KrYDb0yyHdgJvDTvAarq58Af1l19ELh/cv5+4ONzHeoiFiX8PcCLF1xeYeTILpRkH3ADcGzcSfg68CXgryPPAfAuYBX49uShxz1JLh9rmKr6DfBV4AXgDPDHqvrxWPOs8/aqOgNrCwrwtpHnWZjwL/YtmAvxckOSNwHfAz5fVX8acY6PAmer6vhYM6yzHXgP8K2qugH4CyPuwk4eNx8ErgLeCVye5I6x5ll0ixL+CnDlBZf3MsJu2npJdrAW/QNV9fDI49wMfCzJr1l7KHRLku+OOM8KsFJVf9sLOsraL4KxfBD4VVWtVtV54GHg/SPOc6HfJXkHwOT07MjzLEz4vwCuTnJVkstYe1LmkTEHytp3cd8LnKqqr405C0BVfbmq9lbVPtb+fX5SVaOtaFX1W+DFJNdMrjoAPDPWPKzt4t+UZOfkZ3eAxXkS9BHgzsn5O4HvjzgLsLa7NrqqejnJZ4AfsfZs7H1VdXLksW4GPgGcSPLU5Lp/r6pHR5xp0XwWeGDyy/p54JNjDVJVx5IcBZ5g7RWZJxnhHXNJHgT+CdiVZAW4G/gK8B9JPsXaL6h/nfdc6/nOPamhRdnVlzRHhi81ZPhSQ4YvNWT4UkOGLzVk+FJDhi819L+mQ3+1ANWApAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1449f5518>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = get_model(\"params\")\n",
    "sample = x[0:1]\n",
    "plot.imshow(sample.squeeze(), cmap=\"gray\")\n",
    "plot.show()\n",
    "\n",
    "plot.imshow(sample[:, 8:-8, 8:-8].squeeze(), cmap=\"gray\")\n",
    "plot.show()\n",
    "\n",
    "p = model.predict(sample)\n",
    "p = p.squeeze()\n",
    "#p = np.pad(p, (8, 8), mode=\"constant\", constant_values=0)\n",
    "i, j = np.unravel_index(np.argmax(p), p.shape)\n",
    "plot.imshow(p.squeeze(), cmap=\"gray\")\n",
    "plot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Neuron output at (10, 11) =  0.99998176\n",
      "Flattened model neuron output at (10, 11) =  0.99998176\n",
      "relevance at index =  -8.912141\n",
      "analysis.sum() =  3.5663242\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAADC9JREFUeJzt3V+IXPUZxvHnMSZBklEiWW38V62EUhEayxoK1mIRRUsheqEkF5KCNF4oVPCikhu9qUipWi+KEGswgn8qqDUX0iaEohWKuBGJsWmrxK1uE5JIik5AkWzeXuyJrHHnnHHmzJzZfb8fWGbm/M7Z8zrm2TMzvznndUQIQD6nNV0AgGYQfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSZ0+zJ21Wq0YGxsb5i6BVI4cOaJ2u+1u1u0r/LZvkPSopEWS/hARD5atPzY2pgceeKCfXQIosXnz5q7X7fllv+1Fkn4v6UZJl0naYPuyXn8fgOHq5z3/WknvR8T+iPhC0nOS1tVTFoBB6yf850v6aNbjqWLZV9jeZHvC9kS73e5jdwDq1E/45/pQ4WvnB0fElogYj4jxVqvVx+4A1Kmf8E9JunDW4wskHeivHADD0k/435S02vYltpdIWi9pez1lARi0nqf6IuK47bsk/UUzU31bI+Ld2ioDMFB9zfNHxCuSXqmpFgBDxNd7gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSKqvLr22JyW1JU1LOh4R43UUBWDw+gp/4ScR8XENvwfAEPGyH0iq3/CHpB22d9veVEdBAIaj35f9V0XEAdvnSNpp+58R8drsFYo/CpskaeXKlX3uDkBd+jryR8SB4vawpJckrZ1jnS0RMR4R461Wq5/dAahRz+G3vcx26+R9SddL2ltXYQAGq5+X/edKesn2yd/zTET8uZaqAAxcz+GPiP2Svl9jLQCGiKk+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKV4be91fZh23tnLTvb9k7b7xW3KwZbJoC6dXPkf1LSDacsu1fSrohYLWlX8RjAPFIZ/oh4TdLRUxavk7StuL9N0k011wVgwHp9z39uRByUpOL2nPpKAjAMA//Az/Ym2xO2J9rt9qB3B6BLvYb/kO1VklTcHu60YkRsiYjxiBhvtVo97g5A3XoN/3ZJG4v7GyW9XE85AIalm6m+ZyX9XdJ3bU/Zvl3Sg5Kus/2epOuKxwDmkdOrVoiIDR2Grq25FmBojh8/3tf2p59eGZ2Rxzf8gKQIP5AU4QeSIvxAUoQfSIrwA0nN//kKLFi2S8cjonR88eLFPf/uEydOlI4vBBz5gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiAp5vkxsqrm8avm6stUzeNPT0+XjnNKL4B5i/ADSRF+ICnCDyRF+IGkCD+QFOEHkpr/k5Vo1CDn4vv12WefdRxbunRp6bZVdfd7rYFRwJEfSIrwA0kRfiApwg8kRfiBpAg/kBThB5KqnOe3vVXSzyQdjojLi2X3S/qFpCPFapsj4pVBFYnR1c98eNU58VW/u91ul46feeaZHcempqb62vfKlStLx+eDbo78T0q6YY7lj0TEmuKH4APzTGX4I+I1SUeHUAuAIernPf9dtvfY3mp7RW0VARiKXsP/mKRLJa2RdFDSQ51WtL3J9oTtiar3aACGp6fwR8ShiJiOiBOSHpe0tmTdLRExHhHjrVar1zoB1Kyn8NteNevhzZL21lMOgGHpZqrvWUnXSFppe0rSfZKusb1GUkialHTHAGsEMACV4Y+IDXMsfmIAtWAe6ue89apr5y9ZsqR0fGxsrHR89erVHcc++OCD0m1PO638RTHn8wOYtwg/kBThB5Ii/EBShB9IivADSXHpbgxU2ZRX1Sm9+/fvLx2/8sorS8fLpvrWr19fuu2LL75YOj4fpvKqcOQHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaSY50df+mnBXdZCW5JWrCi/NOTVV19dOr579+6OY4sWLSrdtuqU3oVg4f8XApgT4QeSIvxAUoQfSIrwA0kRfiApwg8kxTw/GlN1ae6lS5eWju/Zs6d0/NVXX+04tnz58tJtL7rootLxY8eOlY7PBxz5gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiCpynl+2xdKekrStySdkLQlIh61fbakP0q6WNKkpFsj4n+DKxULzeeff146ft5555WO79ixo3S87Jz8ZcuWlW5bVdtC0M2R/7ikeyLie5J+KOlO25dJulfSrohYLWlX8RjAPFEZ/og4GBFvFffbkvZJOl/SOknbitW2SbppUEUCqN83es9v+2JJV0h6Q9K5EXFQmvkDIemcuosDMDhdh9/2ckkvSLo7Ij79Btttsj1he6LdbvdSI4AB6Cr8thdrJvhPR8TJDoaHbK8qxldJOjzXthGxJSLGI2K81WrVUTOAGlSG3zOXZ31C0r6IeHjW0HZJG4v7GyW9XH95AAalm1N6r5J0m6R3bL9dLNss6UFJz9u+XdKHkm4ZTIlYqKpadE9NTZWOV7Xo/uSTTzqOHT16tHTbqtOJF4LK8EfE65I6XZz92nrLATAsfMMPSIrwA0kRfiApwg8kRfiBpAg/kBSX7kZjqtp7n3XWWaXjk5OTpePT09Mdx84444zSbTPgyA8kRfiBpAg/kBThB5Ii/EBShB9IivADSTHPj8ZEROl41eWzq1p8V/3+7DjyA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBSzPNjoMrm2staaFdt242q6wVkx5EfSIrwA0kRfiApwg8kRfiBpAg/kBThB5KqDL/tC23/1fY+2+/a/mWx/H7b/7X9dvHz08GXi4UkIkp/MFjdfMnnuKR7IuIt2y1Ju23vLMYeiYjfDq48AINSGf6IOCjpYHG/bXufpPMHXRiAwfpG7/ltXyzpCklvFIvusr3H9lbbKzpss8n2hO2JdrvdV7EA6tN1+G0vl/SCpLsj4lNJj0m6VNIazbwyeGiu7SJiS0SMR8R4q9WqoWQAdegq/LYXayb4T0fEi5IUEYciYjoiTkh6XNLawZUJoG7dfNpvSU9I2hcRD89avmrWajdL2lt/eQAGpZtP+6+SdJukd2y/XSzbLGmD7TWSQtKkpDsGUiHmNU6rHV3dfNr/uqS5/g++Un85AIaFb/gBSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeS8jAvkWz7iKT/zFq0UtLHQyvgmxnV2ka1LonaelVnbd+OiLFuVhxq+L+2c3siIsYbK6DEqNY2qnVJ1NarpmrjZT+QFOEHkmo6/Fsa3n+ZUa1tVOuSqK1XjdTW6Ht+AM1p+sgPoCGNhN/2Dbb/Zft92/c2UUMntidtv1N0Hp5ouJattg/b3jtr2dm2d9p+r7ids01aQ7WNROfmks7SjT53o9bxeugv+20vkvRvSddJmpL0pqQNEfGPoRbSge1JSeMR0ficsO0fSzom6amIuLxY9htJRyPiweIP54qI+NWI1Ha/pGNNd24uGsqsmt1ZWtJNkn6uBp+7krpuVQPPWxNH/rWS3o+I/RHxhaTnJK1roI6RFxGvSTp6yuJ1krYV97dp5h/P0HWobSRExMGIeKu435Z0srN0o89dSV2NaCL850v6aNbjKY1Wy++QtMP2btubmi5mDucWbdNPtk8/p+F6TlXZuXmYTuksPTLPXS8dr+vWRPjn6v4zSlMOV0XEDyTdKOnO4uUtutNV5+ZhmaOz9EjoteN13ZoI/5SkC2c9vkDSgQbqmFNEHChuD0t6SaPXffjQySapxe3hhuv50ih1bp6rs7RG4LkbpY7XTYT/TUmrbV9ie4mk9ZK2N1DH19heVnwQI9vLJF2v0es+vF3SxuL+RkkvN1jLV4xK5+ZOnaXV8HM3ah2vG/mSTzGV8TtJiyRtjYhfD72IOdj+jmaO9tJME9NnmqzN9rOSrtHMWV+HJN0n6U+Snpd0kaQPJd0SEUP/4K1Dbddo5qXrl52bT77HHnJtP5L0N0nvSDpRLN6smffXjT13JXVtUAPPG9/wA5LiG35AUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5L6P7ZBlCbXTBREAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x14c40f208>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "relevance at index =  -0.8532734\n",
      "analysis.sum() =  3.5663233\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAADMpJREFUeJzt3V+IXPd5xvHn0VqS7VGMV6SyhWPXajChJlClLCLgYlSMg1MKci4SoougQqiCHWMLYqjxTXxTMCF/GkMJKPUSBRKlgcS1LkQbYxfcQAheCxMrVVwLIUeqFq2C/E+DjazVm4s9Cmt553fGM2fmzOr9fmCZmfOes+dlpGfPzPzOnJ8jQgDyWdN2AwDaQfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyR11Th31ul0Ynp6epy7BFJ5/fXX1e123c+6Q4Xf9j2SvitpStK/RcTjpfWnp6f14IMPDrNLAAVPPPFE3+sO/LLf9pSkf5X0WUm3S9pp+/ZBfx+A8RrmPf82SUcj4lhEnJf0E0k7mmkLwKgNE/6bJJ1Y9vhktex9bO+2PWd7rtvtDrE7AE0aJvwrfajwge8HR8TeiJiJiJlOpzPE7gA0aZjwn5R087LHH5N0arh2AIzLMOF/QdJttrfYXifpi5IONNMWgFEbeKgvIi7YfkDSf2lpqG82In7bWGcARmqocf6IOCjpYEO9ABgjTu8FkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gqaFm6bV9XNLbkhYlXYiImSaaAjB6Q4W/8rcR8YcGfg+AMeJlP5DUsOEPSb+w/aLt3U00BGA8hn3Zf0dEnLK9SdIztn8XEc8vX6H6o7Bbkq6//vohdwegKUMd+SPiVHW7IOkpSdtWWGdvRMxExEyn0xlmdwAaNHD4bXdsf+TSfUmfkXS4qcYAjNYwL/tvkPSU7Uu/58cR8Z+NdAVg5AYOf0Qck/RXDfYCYIwY6gOSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kVRt+27O2F2wfXrZso+1nbL9a3U6Ptk0ATevnyP8DSfdctuwRSc9GxG2Snq0eA1hFasMfEc9LOnvZ4h2S9lX390m6t+G+AIzYoO/5b4iIeUmqbjc11xKAcRj5B362d9uesz3X7XZHvTsAfRo0/Kdtb5ak6nah14oRsTciZiJiptPpDLg7AE0bNPwHJO2q7u+S9HQz7QAYl36G+vZL+pWkT9g+afvLkh6XdLftVyXdXT0GsIpcVbdCROzsUbqr4V5wBbI98LYRsWr3vRpwhh+QFOEHkiL8QFKEH0iK8ANJEX4gqdqhPmAYpSGzqamp4rZ1Q3V1w3GLi4vFenYc+YGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcb50Zr33ntvqO3Xrl1brK9Z0/vYVneOAF/pBXDFIvxAUoQfSIrwA0kRfiApwg8kRfiBpBjnR1HdeHfdd+5L9Q0bNhS3PX/+fLFe57rrrutZq+v7qqvK0VhY6DlJ1arBkR9IivADSRF+ICnCDyRF+IGkCD+QFOEHkqod57c9K+nvJS1ExCerZY9J+kdJZ6rVHo2Ig6NqEu2pGw8vfWdeKn/nvm4sfdOmTcV6XW9HjhzpWTtx4kRx21tuuaVYX79+fbG+GvRz5P+BpHtWWP6diNha/RB8YJWpDX9EPC/p7Bh6ATBGw7znf8D2b2zP2p5urCMAYzFo+L8n6eOStkqal/StXiva3m17zvZct9sdcHcAmjZQ+CPidEQsRsRFSd+XtK2w7t6ImImImU6nM2ifABo2UPhtb1728HOSDjfTDoBx6Weob7+k7ZI+avukpK9L2m57q6SQdFzSV0bYI4ARqA1/ROxcYfGTI+gFE6huHH9xcbFYf/PNN3vWrr322uK2hw4dKtbvvPPOYn3Pnj09awcPlkenz5w5U6y/8847xfpqwBl+QFKEH0iK8ANJEX4gKcIPJEX4gaS4dPcVru5rr3WX5q4byqsbrivt/+LFi8Vt77vvvmL9ueeeK9ZfeeWVnrWjR48Wtx1m+m9pdUzxzZEfSIrwA0kRfiApwg8kRfiBpAg/kBThB5JinP8KUBpLH3a8uW48+9133y3Wr7766p616enypR/vuuuuYn379u3F+sMPP9yzNjs7W9z2SpiCuw5HfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IinH+K8Aovzte97vrrhdw7ty5nrW6awGcPVueH/b+++8v1vfv39+z9tZbbxW3HfY6CKsBR34gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSKp2nN/2zZJ+KOlGSRcl7Y2I79reKOnfJd0q6bikL0TE66NrFZOobrx7/fr1PWtvvPFGcduHHnqoWN+yZUux/tprrxXrJXXj/FeCfo78FyR9LSL+UtKnJX3V9u2SHpH0bETcJunZ6jGAVaI2/BExHxGHqvtvSzoi6SZJOyTtq1bbJ+neUTUJoHkf6j2/7VslfUrSryXdEBHz0tIfCEmbmm4OwOj0HX7bGyT9TNKeiCifGP3+7XbbnrM91+12B+kRwAj0FX7ba7UU/B9FxM+rxadtb67qmyWteMXDiNgbETMRMdPpdJroGUADasPvpY89n5R0JCK+vax0QNKu6v4uSU833x6AUennK713SPqSpJdtv1Qte1TS45J+avvLkn4v6fOjaRGTbJghsRtvvLFYP3PmTLG+cePGYv3UqVM9a9dcc01x27rpw68EteGPiF9K6vUvXL6wOoCJxRl+QFKEH0iK8ANJEX4gKcIPJEX4gaS4dDeK6sbx68bDL1y40LN27Nix4rZ15wHMz88X66XpwTOM49fhyA8kRfiBpAg/kBThB5Ii/EBShB9IivADSTHOj6K6S3NPTU0V62vW9D6+rFu3rrht6RwBqXxZcOnKmEZ7lDjyA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBSjPNjKMN8L35xcbHBTvBhceQHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaRqw2/7Ztv/bfuI7d/afqha/pjt/7f9UvXzd6NvF5MmIoo/tnv+oF39nORzQdLXIuKQ7Y9IetH2M1XtOxHxzdG1B2BUasMfEfOS5qv7b9s+IummUTcGYLQ+1Ht+27dK+pSkX1eLHrD9G9uztqd7bLPb9pztuW63O1SzAJrTd/htb5D0M0l7IuItSd+T9HFJW7X0yuBbK20XEXsjYiYiZjqdTgMtA2hCX+G3vVZLwf9RRPxckiLidEQsRsRFSd+XtG10bQJoWj+f9lvSk5KORMS3ly3fvGy1z0k63Hx7AEaln0/775D0JUkv236pWvaopJ22t0oKScclfWUkHWKilS7NLXH57EnWz6f9v5S00qDswebbATAunOEHJEX4gaQIP5AU4QeSIvxAUoQfSIpLd2MojOOvXhz5gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApj3Oc1vYZSa8tW/RRSX8YWwMfzqT2Nql9SfQ2qCZ7+/OI+LN+Vhxr+D+wc3suImZaa6BgUnub1L4kehtUW73xsh9IivADSbUd/r0t779kUnub1L4kehtUK721+p4fQHvaPvIDaEkr4bd9j+1XbB+1/UgbPfRi+7jtl6uZh+da7mXW9oLtw8uWbbT9jO1Xq9sVp0lrqbeJmLm5MLN0q8/dpM14PfaX/banJP2fpLslnZT0gqSdEfG/Y22kB9vHJc1EROtjwrbvlHRO0g8j4pPVsm9IOhsRj1d/OKcj4p8mpLfHJJ1re+bmakKZzctnlpZ0r6R/UIvPXaGvL6iF562NI/82SUcj4lhEnJf0E0k7Wuhj4kXE85LOXrZ4h6R91f19WvrPM3Y9epsIETEfEYeq+29LujSzdKvPXaGvVrQR/psknVj2+KQma8rvkPQL2y/a3t12Myu4oZo2/dL06Zta7udytTM3j9NlM0tPzHM3yIzXTWsj/CvN/jNJQw53RMRfS/qspK9WL2/Rn75mbh6XFWaWngiDznjdtDbCf1LSzcsef0zSqRb6WFFEnKpuFyQ9pcmbffj0pUlSq9uFlvv5k0mauXmlmaU1Ac/dJM143Ub4X5B0m+0tttdJ+qKkAy308QG2O9UHMbLdkfQZTd7swwck7aru75L0dIu9vM+kzNzca2ZptfzcTdqM162c5FMNZfyLpClJsxHxz2NvYgW2/0JLR3tp6crGP26zN9v7JW3X0re+Tkv6uqT/kPRTSbdI+r2kz0fE2D9469Hbdi29dP3TzM2X3mOPube/kfQ/kl6WdLFa/KiW3l+39twV+tqpFp43zvADkuIMPyApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSf0R4vrMg2IyyGEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x14c6a0cc0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "relevance at index =  0.7234432\n",
      "analysis.sum() =  3.5663233\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAADKdJREFUeJzt3V+MHXUZxvHnaWmXdO1FAambiqKGGIHE1mwaE4zBGASNSfHCQi9ITYjlAhIlXkC4kYSYEKOiF8akSmNNFDVRpBeNSIgJmAhhIUSqVfmTWmo3XQUCtJRT2n292KlZyp6Zw5mZM2f7fj/J5syZ38yZd0/22ZlzfjPzc0QIQD4rui4AQDcIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpM4Z5cYmJiZicnJylJsEUjl27Jh6vZ4HWbZW+G1fI+kHklZK+klE3F22/OTkpK6++uo6mwRQ4sEHHxx42aEP+22vlPRDSZ+XdKmkbbYvHfb1AIxWnc/8myU9FxEvRMQJSb+UtKWZsgC0rU74N0h6cdHzQ8W8t7G9w/aM7Zler1djcwCaVCf8S32p8I7rgyNiZ0RMR8T0xMREjc0BaFKd8B+SdNGi5++XdLheOQBGpU74n5B0ie0P2V4t6XpJe5opC0Dbhu7qi4iTtm+R9KAWuvp2RcRfG6sMQKtq9fNHxF5JexuqBcAIcXovkBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSdUapdf2AUmvSzol6WRETDdRFID21Qp/4TMR8d8GXgfACHHYDyRVN/wh6Q+2n7S9o4mCAIxG3cP+KyLisO0LJT1k++8R8cjiBYp/Cjskac2aNTU3B6Aptfb8EXG4eJyTdL+kzUssszMipiNiemJios7mADRo6PDbnrS99vS0pM9J2tdUYQDaVeewf72k+22ffp1fRMTvG6kKQOuGDn9EvCDp4w3WAmCE6OoDkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJFUZftu7bM/Z3rdo3nm2H7L9bPG4rt0yATRtkD3/TyVdc8a82yU9HBGXSHq4eA5gGakMf0Q8IunlM2ZvkbS7mN4t6dqG6wLQsmE/86+PiFlJKh4vbK4kAKPQ+hd+tnfYnrE90+v12t4cgAENG/4jtqckqXic67dgROyMiOmImJ6YmBhycwCaNmz490jaXkxvl/RAM+UAGJVBuvruk/RnSR+1fcj2jZLulnSV7WclXVU8B7CMnFO1QERs69P02YZrwVkoIoZe13Zr26567ar2Or/XuOAMPyApwg8kRfiBpAg/kBThB5Ii/EBSlV19QB11uuvm5+drvXadrr6zoSuvCnt+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iKfn50pqqvfeXKlaXtp06dGvr1q/rxV6wo3y+eDecBsOcHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaTo50epqv7sVatWlbafOHGib1vVCE5vvPFGrW2vXbu2b9vcXN9BpiRJa9asqbXtt956q7R9HLDnB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkKvv5be+S9EVJcxFxeTHvTklflfSfYrE7ImJvW0WiPVX9+HX7+Y8dO9a3reqa+XPOKf/zrOpLf+211/q2Vd0LYHJysta2l4NB9vw/lXTNEvPviYiNxQ/BB5aZyvBHxCOSXh5BLQBGqM5n/lts/8X2LtvrGqsIwEgMG/4fSfqIpI2SZiV9t9+CtnfYnrE90+v1htwcgKYNFf6IOBIRpyJiXtKPJW0uWXZnRExHxHTVhRwARmeo8NueWvT0S5L2NVMOgFEZpKvvPklXSrrA9iFJ35R0pe2NkkLSAUk3tVgjgBZUhj8iti0x+94WakEHqvrx695bv6x93bry74mrziHYsGFDafttt93Wt23r1q21XvvgwYOl7csBZ/gBSRF+ICnCDyRF+IGkCD+QFOEHkuLW3We5ukNRz8/Pl7YfPXq0tL3sstzjx4+Xrrtp06bS9ltvvbW0/brrruvbdtlll5WuOzs7W9qe5ZJeAGchwg8kRfiBpAg/kBThB5Ii/EBShB9Iin7+s1zVJblVqi7ZrTqPoOzuTVNTU33bJOnRRx8tba86x2D//v192/buLb/h9LnnnlvaXnZb8OWCPT+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEU/P0rVvbX3m2++2bft+eefL133xIkTpe133XVXaXvZrcGrbgv+0ksvlbZXDfG9HLDnB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkKvv5bV8k6WeS3idpXtLOiPiB7fMk/UrSxZIOSNoaEa+0VyrGUZ1xASYnJ0vXrepLr9p22XkEVfcCWL16dWl71TkIy8Ege/6Tkr4RER+T9ElJN9u+VNLtkh6OiEskPVw8B7BMVIY/ImYj4qli+nVJ+yVtkLRF0u5isd2Srm2rSADNe1ef+W1fLGmTpMclrY+IWWnhH4SkC5suDkB7Bg6/7fdI+o2kr0fEwDcws73D9oztmV6vN0yNAFowUPhtr9JC8H8eEb8tZh+xPVW0T0maW2rdiNgZEdMRMV12M0cAo1UZfi9ctnWvpP0R8b1FTXskbS+mt0t6oPnyALRlkEt6r5B0g6RnbD9dzLtD0t2Sfm37RkkHJX25nRKxnJV11508ebJ03arutscee6y0vexIc/369aXrvvJKea911dDmy0Fl+CPiT5L6XbT92WbLATAqy//fF4ChEH4gKcIPJEX4gaQIP5AU4QeS4tbdqKXq1t1lQ3wfP368dN2qS3rPP//80vayS35fffXV0nXPhn78Kmf/bwhgSYQfSIrwA0kRfiApwg8kRfiBpAg/kBT9/KilzhDeVeuWDe8t1euLrzo/IQP2/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFP38qKWqr76sve2+dvryy7HnB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkKsNv+yLbf7S93/ZfbX+tmH+n7X/bfrr4+UL75WLc2G7tZ8WKFaU/Veuj3CAn+ZyU9I2IeMr2WklP2n6oaLsnIr7TXnkA2lIZ/oiYlTRbTL9ue7+kDW0XBqBd7+ozv+2LJW2S9Hgx6xbbf7G9y/a6PuvssD1je6bX69UqFkBzBg6/7fdI+o2kr0fEa5J+JOkjkjZq4cjgu0utFxE7I2I6IqYnJiYaKBlAEwYKv+1VWgj+zyPit5IUEUci4lREzEv6saTN7ZUJoGmDfNtvSfdK2h8R31s0f2rRYl+StK/58gC0ZZBv+6+QdIOkZ2w/Xcy7Q9I22xslhaQDkm5qpUIArRjk2/4/SVqq03Rv8+UAGBXO8AOSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyTlqiGWG92Y/R9J/1o06wJJ/x1ZAe/OuNY2rnVJ1DasJmv7YES8d5AFRxr+d2zcnomI6c4KKDGutY1rXRK1Daur2jjsB5Ii/EBSXYd/Z8fbLzOutY1rXRK1DauT2jr9zA+gO13v+QF0pJPw277G9j9sP2f79i5q6Mf2AdvPFCMPz3Rcyy7bc7b3LZp3nu2HbD9bPC45TFpHtY3FyM0lI0t3+t6N24jXIz/st71S0j8lXSXpkKQnJG2LiL+NtJA+bB+QNB0RnfcJ2/60pKOSfhYRlxfzvi3p5Yi4u/jHuS4ibhuT2u6UdLTrkZuLAWWmFo8sLelaSV9Rh+9dSV1b1cH71sWef7Ok5yLihYg4IemXkrZ0UMfYi4hHJL18xuwtknYX07u18Mczcn1qGwsRMRsRTxXTr0s6PbJ0p+9dSV2d6CL8GyS9uOj5IY3XkN8h6Q+2n7S9o+tilrC+GDb99PDpF3Zcz5kqR24epTNGlh6b926YEa+b1kX4lxr9Z5y6HK6IiE9I+rykm4vDWwxmoJGbR2WJkaXHwrAjXjeti/AfknTRoufvl3S4gzqWFBGHi8c5Sfdr/EYfPnJ6kNTica7jev5vnEZuXmpkaY3BezdOI153Ef4nJF1i+0O2V0u6XtKeDup4B9uTxRcxsj0p6XMav9GH90jaXkxvl/RAh7W8zbiM3NxvZGl1/N6N24jXnZzkU3RlfF/SSkm7IuJbIy9iCbY/rIW9vbQwiOkvuqzN9n2SrtTCVV9HJH1T0u8k/VrSByQdlPTliBj5F299artSC4eu/x+5+fRn7BHX9ilJj0p6RtJ8MfsOLXy+7uy9K6lrmzp43zjDD0iKM/yApAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyT1P5MjxToqQRaIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1479c6b70>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pixel_x = 11\n",
    "pixel_y = 10\n",
    "model = get_model(\"params\")\n",
    "p = model.predict(sample)\n",
    "print('Neuron output at (%d, %d) = ' % (pixel_y, pixel_x), p[0, pixel_y, pixel_x, 0])\n",
    "index = np.ravel_multi_index((0, pixel_y, pixel_x, 0), p.shape)\n",
    "flatten_model = get_model(\"params\", flatten_output=True)\n",
    "flat_p = flatten_model.predict(sample)\n",
    "print('Flattened model neuron output at (%d, %d) = ' % (pixel_y, pixel_x), flat_p[0, index])\n",
    "for _ in range(3):\n",
    "    analyzer = innvestigate.analyzer.LRPEpsilon(flatten_model, input_layer_rule=(0, 1), neuron_selection_mode=\"index\")\n",
    "    analysis = analyzer.analyze(sample, index)\n",
    "    print(\"relevance at index = \", analysis[0, pixel_y+8, pixel_x+8, 0])\n",
    "    print(\"analysis.sum() = \", analysis.sum())\n",
    "    #analysis = heatmap(analysis)\n",
    "    plot.imshow(analysis.squeeze(), cmap='gray', interpolation='nearest')\n",
    "    plot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}