diff --git a/captum/attr/__init__.py b/captum/attr/__init__.py index d343ff24d4..52c90f3536 100644 --- a/captum/attr/__init__.py +++ b/captum/attr/__init__.py @@ -5,6 +5,12 @@ from ._core.input_x_gradient import InputXGradient # noqa from ._core.saliency import Saliency # noqa from ._core.noise_tunnel import NoiseTunnel # noqa +from ._core.layer_conductance import LayerConductance # noqa +from ._core.layer_gradient_x_activation import LayerGradientXActivation # noqa +from ._core.layer_activation import LayerActivation # noqa +from ._core.neuron_conductance import NeuronConductance # noqa +from ._core.neuron_gradient import NeuronGradient # noqa +from ._core.neuron_integrated_gradients import NeuronIntegratedGradients # noqa from ._models.base import ( InterpretableEmbeddingBase, TokenReferenceBase, @@ -19,6 +25,12 @@ "DeepLift", "InputXGradient", "Saliency", + "LayerConductance", + "LayerGradientXActivation", + "LayerActivation", + "NeuronConductance", + "NeuronGradient", + "NeuronIntegratedGradients", "NoiseTunnel", "InterpretableEmbeddingBase", "TokenReferenceBase", diff --git a/captum/attr/_core/conductance.py b/captum/attr/_core/layer_conductance.py similarity index 98% rename from captum/attr/_core/conductance.py rename to captum/attr/_core/layer_conductance.py index 99fdbd098a..7956e8d1b2 100644 --- a/captum/attr/_core/conductance.py +++ b/captum/attr/_core/layer_conductance.py @@ -6,7 +6,7 @@ from .._utils.gradient import compute_layer_gradients_and_eval -class Conductance(LayerAttribution): +class LayerConductance(LayerAttribution): def __init__(self, forward_func, layer): r""" Args diff --git a/captum/attr/_core/neuron_conductance.py b/captum/attr/_core/neuron_conductance.py index c0a7404fdf..0544db8124 100644 --- a/captum/attr/_core/neuron_conductance.py +++ b/captum/attr/_core/neuron_conductance.py @@ -23,7 +23,7 @@ def attribute( neuron_index, baselines=None, target=None, - n_steps=500, + n_steps=50, method="riemann_trapezoid", ): r""" diff --git a/notebooks/Titanic_Basic_Interpret.ipynb b/notebooks/Titanic_Basic_Interpret.ipynb new file mode 100644 index 0000000000..89439d43b0 --- /dev/null +++ b/notebooks/Titanic_Basic_Interpret.ipynb @@ -0,0 +1,844 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Captum Introduction Tutorial - Titanic Data Analysis" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this notebook, we will demonstrate the basic features of the Captum interpretability library through an example model trained on the Titanic survival data. We will first train a deep neural network on the data using PyTorch and use Captum to understand which of the features were most important and how the network reached its prediction." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# Initial imports\n", + "import numpy as np\n", + "\n", + "import torch\n", + "\n", + "from captum.attr import IntegratedGradients\n", + "from captum.attr import LayerConductance\n", + "from captum.attr import NeuronConductance\n", + "\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "\n", + "from scipy import stats\n", + "import pandas as pd" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will begin by importing and cleaning the dataset. Download the dataset from http://biostat.mc.vanderbilt.edu/wiki/pub/Main/DataSets/titanic3.csv and update the cell below with the path to the dataset csv." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# Download dataset from: http://biostat.mc.vanderbilt.edu/wiki/pub/Main/DataSets/titanic3.csv\n", + "# Update path to dataset here.\n", + "dataset_path = \"titanic3.csv\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Read dataset from csv file.\n", + "titanic_data = pd.read_csv(dataset_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With the data loaded, we now preprocess the data by converting some categorical features such as gender, location of embarcation, and passenger class into one-hot encodings (separate feature columns for each class with 0 / 1). We also remove some features that are more difficult to analyze, such as name, and fill missing values in age and fare with the average values." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "titanic_data = pd.concat([titanic_data,\n", + " pd.get_dummies(titanic_data['sex']),\n", + " pd.get_dummies(titanic_data['embarked'],prefix=\"embark\"),\n", + " pd.get_dummies(titanic_data['pclass'],prefix=\"class\")], axis=1)\n", + "titanic_data[\"age\"] = titanic_data[\"age\"].fillna(titanic_data[\"age\"].mean())\n", + "titanic_data[\"fare\"] = titanic_data[\"fare\"].fillna(titanic_data[\"fare\"].mean())\n", + "titanic_data = titanic_data.drop(['name','ticket','cabin','boat','body','home.dest','sex','embarked','pclass'], axis=1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "After processing, the features we have are:\n", + "\n", + "* Age - Passenger Age\n", + "* Sibsp - Number of Siblings / Spouses Aboard\n", + "* Parch - Number of Parents / Children Aboard\n", + "* Fare - Fare Amount Paid in British Pounds\n", + "* Female - Binary variable indicating whether passenger is female\n", + "* Male - Binary variable indicating whether passenger is male\n", + "* EmbarkC - Binary variable indicating whether passenger embarked at Cherbourg\n", + "* EmbarkQ - Binary variable indicating whether passenger embarked at Queenstown\n", + "* EmbarkS - Binary variable indicating whether passenger embarked at Southampton\n", + "* Class1 - Binary variable indicating whether passenger was in first class\n", + "* Class2 - Binary variable indicating whether passenger was in second class\n", + "* Class3 - Binary variable indicating whether passenger was in third class\n", + "\n", + "(Reference: http://campus.lakeforest.edu/frank/FILES/MLFfiles/Bio150/Titanic/TitanicMETA.pdf)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We now convert the data to numpy arrays and separate the training and test sets." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# Set random seed for reproducibility.\n", + "np.random.seed(131254)\n", + "\n", + "# Convert features and labels to numpy arrays.\n", + "labels = titanic_data[\"survived\"].to_numpy()\n", + "titanic_data = titanic_data.drop(['survived'], axis=1)\n", + "feature_names = list(titanic_data.columns)\n", + "data = titanic_data.to_numpy()\n", + "\n", + "# Separate training and test sets using \n", + "train_indices = np.random.choice(len(labels), int(0.7*len(labels)), replace=False)\n", + "test_indices = list(set(range(len(labels))) - set(train_indices))\n", + "train_features = data[train_indices]\n", + "train_labels = labels[train_indices]\n", + "test_features = data[test_indices]\n", + "test_labels = labels[test_indices]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We are now ready to define the neural network architecture we will use for the task. We have defined a simple architecture using 2 hidden layers, the first with 12 hidden units and the second with 8 hidden units, each with Sigmoid non-linearity. The final layer performs a softmax operation and has 2 units, corresponding to the outputs of either survived (1) or not survived (0)." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "torch.manual_seed(1) # Set seed for reproducibility.\n", + "class TitanicSimpleNNModel(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.linear1 = nn.Linear(12, 12)\n", + " self.sigmoid1 = nn.Sigmoid()\n", + " self.linear2 = nn.Linear(12, 8)\n", + " self.sigmoid2 = nn.Sigmoid()\n", + " self.linear3 = nn.Linear(8, 2)\n", + " self.softmax = nn.Softmax(dim=1)\n", + "\n", + " def forward(self, x):\n", + " lin1_out = self.linear1(x)\n", + " sigmoid_out1 = self.sigmoid1(lin1_out)\n", + " sigmoid_out2 = self.sigmoid2(self.linear2(sigmoid_out1))\n", + " return self.softmax(self.linear3(sigmoid_out2))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We train the network using the training data for 200 epochs." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/200 => Loss: 0.70\n", + "Epoch 21/200 => Loss: 0.56\n", + "Epoch 41/200 => Loss: 0.51\n", + "Epoch 61/200 => Loss: 0.49\n", + "Epoch 81/200 => Loss: 0.49\n", + "Epoch 101/200 => Loss: 0.49\n", + "Epoch 121/200 => Loss: 0.47\n", + "Epoch 141/200 => Loss: 0.48\n", + "Epoch 161/200 => Loss: 0.47\n", + "Epoch 181/200 => Loss: 0.47\n" + ] + } + ], + "source": [ + "criterion = nn.CrossEntropyLoss()\n", + "num_epochs = 200\n", + "\n", + "net = TitanicSimpleNNModel()\n", + "optimizer = torch.optim.Adam(net.parameters(), lr=0.1)\n", + "input_tensor = torch.from_numpy(train_features).type(torch.FloatTensor)\n", + "label_tensor = torch.from_numpy(train_labels)\n", + "for epoch in range(num_epochs): \n", + " output = net(input_tensor)\n", + " loss = criterion(output, label_tensor)\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + " if epoch % 20 == 0:\n", + " print ('Epoch {}/{} => Loss: {:.2f}'.format(epoch+1, num_epochs, loss.item()))\n", + "\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can now evaluate the training and test accuracies of our model." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train Accuracy: 0.8493449781659389\n" + ] + } + ], + "source": [ + "out_probs = net(input_tensor).detach().numpy()\n", + "out_classes = np.argmax(out_probs, axis=1)\n", + "print(\"Train Accuracy:\", sum(out_classes == train_labels) / len(train_labels))" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Test Accuracy: 0.816793893129771\n" + ] + } + ], + "source": [ + "test_input_tensor = torch.from_numpy(test_features).type(torch.FloatTensor)\n", + "out_probs = net(test_input_tensor).detach().numpy()\n", + "out_classes = np.argmax(out_probs, axis=1)\n", + "print(\"Test Accuracy:\", sum(out_classes == test_labels) / len(test_labels))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Beyond just considering the accuracy of the classifier, there are many important questions to understand how the model is working and it's decision, which is the purpose of Captum, to help make neural networks in PyTorch more interpretable." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The first question we can ask is which of the features were actually important to the model to reach this decision? This is the first main component of Captum, the ability to obtain **Feature Attributions**. For this example, we will apply Integrated Gradients, which is one of the Feature Attribution methods included in Captum. More information regarding Integrated Gradients can be found in the original paper here: https://arxiv.org/pdf/1703.01365.pdf" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To apply integrated gradients, we first create an IntegratedGradients object, providing the model object." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "ig = IntegratedGradients(net)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To compute the integrated gradients, we use the attribute method of the IntegratedGradients object. The method takes tensor(s) of input examples (matching the forward function of the model), and returns the input attributions for the given examples. For a network with multiple outputs, a target index must also be provided, defining the index of the output for which gradients are computed. For this example, we provide target = 1, corresponding to survival. \n", + "\n", + "The input tensor provided should require grad, so we call requires\\_grad\\_ on the tensor. The attribute method also takes a baseline, which is the starting point from which gradients are integrated. The default value is just the 0 tensor, which is a reasonable baseline / default for this task. \n", + "\n", + "The returned values of the attribute method are the attributions, which match the size of the given inputs, and delta, which approximates the error between the approximated integral and true integral." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "test_input_tensor.requires_grad_()\n", + "attr, delta = ig.attribute(test_input_tensor,target=1)\n", + "attr = attr.detach().numpy()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To understand these attributions, we can first average them across all the inputs and print / visualize the average attribution for each feature." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Feature Importances\n", + "age : -0.454\n", + "sibsp : -0.119\n", + "parch : -0.056\n", + "fare : 0.175\n", + "female : 0.154\n", + "male : -0.359\n", + "embark_C : 0.086\n", + "embark_Q : -0.001\n", + "embark_S : -0.082\n", + "class_1 : 0.062\n", + "class_2 : 0.021\n", + "class_3 : -0.159\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAssAAAGECAYAAADA7m3/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3de5hsVX3m8e8rBwgKgsgRUC4nMsSIRkk8EU28EIFEZUYw4i1EYaIh6BCTJ2MSMhhCNHEwxjEX40SiBrwL3sBLFDyKUUYNBwQEvIB4EATloqgIEtHf/LFXS9H26tOHqu463Xw/z1NP7dp71V5r7aruemvVqtqpKiRJkiT9tHtMuwGSJEnS5sqwLEmSJHUYliVJkqQOw7IkSZLUYViWJEmSOgzLkiRJUodhWZIkSeowLEtacknOTvLtJFtPuy2T0PrzgyQ3j1wePeY+1ySpJKsm1c4F1HlCkrcsVX3zSXJkkk9Nux2SZFiWtKSSrAEeCxTwlEWqY8kC5ohjqmrbkcunp9CGn8hgWf6Pn9LjJ0lzWpb/SCUta88FPgOcDBwxszLJo5J8I8kWI+uemuSitnyPJMcm+UqSG5OcmmTHtm1mFPZ5Sb4GfKytP63t8ztJ/j3JQ0b2fd8k70/y3STnJvmr0ZHMJD+f5Kwk30rypSTPuCudnW8/SQ5O8rnWhquSnDBy139v1zfNjFTPHvmdPfrcRrj/Osk5wC3AA5Nsn+QNSa5N8vXWzy1YgLbvFya5LMn3krwsyV5JPt3afGqSrVrZ/ZNcneR/JbkhyYYkh4/sa/skb0pyfZIrk7xkJsy3UeRzkrw6ybeAdwL/DDy69f2mjR2vkWNxRJKvtTYcN7J9i9a2r7S+nJdk9wU8Rk9Ocmm7z9eTvHghx07SymFYlrTUngu8tV1+I8nOAFX1GeD7wBNGyv4W8La2/CLgUODxwP2BbwP/NGvfjwceDPxGu/1vwN7A/YDzW50z/qnVtwtDaB8N7vcCzmp13w94NvDa0bC9EAvYz/fb8dgBOBh4QZJD27bHtesdNnGk+jnAUcB2wJXAKcDtwH8BfhH4deD5m9CNJwKPAB4F/AlwEnA4sDvw0NanGbsAOwEPYDieJyV5UNv2j8D2wAMZHqfnAv995L77AVcwHKffBo4GPt36vkMrM9/xmvEY4EHAAcDxSR7c1v9Ra+uTgXsDvwPcsoDH6A3A71XVdq2/H1vQUZO0YhiWJS2ZJI8B9gROrarzgK8wBOIZb6eFryTbMQSbt7dtvwccV1VXV9VtwAnAYbM+sj+hqr5fVbcCVNUbq+p7I+Uf3kY4twCeBvxFVd1SVZcyhMoZ/xXYUFX/WlW3V9X5wLuBw+bp3j8kualdzl/Ifqrq7Kr6fFX9uKouan19/IIOZt/JVXVJVd0O7Ag8CfjDdlyuA14NPGsT9veKqvpuVV0CXAycWVVXVNV3GN6M/OKs8n9eVbdV1SeADwLPaMf7mcCftcdjA/AqhmA/45qq+sd2nG6dqyELPF5/WVW3VtWFwIXAw9v65wMvqaov1eDCqrqRjT/WPwT2SXLvqvp22y7pbsSwLGkpHcEQtm5ot9/GyIhuu/2bGb7495vA+VV1Zdu2J/DemUAKfAH4EbDzyP2vmlloH7uf2D52/y6woW3aCVgNrBotP2t5T2C/kfB7E8No6i7z9O1FVbVDu/zSQvaTZL8kH29TE77DMJq60zx1LMTsfmwJXDtS/+sYRlAX6psjy7fOcXvbkdvfrqrvj9y+kuFTgJ2Ardrt0W0P6LR7Tgs8Xt8YWb5lpH27M7w5m21jj/XTGN60XZnkExnzi5uSlh+/RCFpSSTZBngGsEWSmUCzNbBDkoe3kb5Lk1zJMBo6OgUDhjD1O1V1zhz7XtMWa2T1bwGHAAcyBOXtGaZuBLieYWrCbsCXW/ndZ9X1iao66C51duH7eRvwGuBJVfWDJH/HHeGv5ij/feCeI7fnCu+j97sKuA3YqY00L7b7JLnXSGDeg2E0+gaGEdo9gUtHtn195L6z+ztX/+c7XhtzFbBXa8/s9d3HqKrOBQ5JsiVwDHAqd36uSFrhHFmWtFQOZRgJ3gfYt10eDHySYR7qjLcxzE9+HHDayPp/Bv46yZ4ASVYnOWSe+rZjCIo3MgTMl89sqKofAe8BTkhyzyQ/P6sNHwB+LslzkmzZLr88Mv91oTa2n+2Ab7Xg90juPCXleuDHDHN8Z1wAPC7JHkm2B/5svsqr6lrgTOBVSe6d4UuSeyUZd6rHfP4yyVZJHsswxeG0drxPZXj8tmuP4R8B8/1M3TeB3dK+QNjMd7w25vXAy5LsncHDktyXeR6j1o/Dk2xfVT8EvsvwHJZ0N2JYlrRUjgD+taq+VlXfmLkwjBQePjL3+O3A/sDHRqZrAPw9cAZwZpLvMfyixn7z1Pcmho/6v84wmvmZWduPYRht/gbw5lbvbQBV9T2GL8I9C7imlXkFw0j4gi1gPy8EXtr6czxDoJy57y3AXwPntOkBj6qqsxh+KeIi4DyGoLcxz2WYAnEpw8j6u4BdN6Ufm+AbrY5rGL5MeXRVfbFt+32GkfErgE8xvCl64zz7+hhwCfCNJDPPg+7xWoD/08qfyRB63wBss4DH6DnAhjaV52iGLx9KuhtJ1VyfdEnS3UuSVwC7VNURGy2sn5Jkf+AtVbXbtNsiSZPkyLKku6X227oPax/JPxJ4HvDeabdLkrR58Qt+ku6utmOYenF/4DqGnzI7faotkiRtdpyGIUmSJHU4DUOSJEnqmEhYTvLEJF9KcnmSY+fY/kdJLk1yUZJ1Mz/9JEmSJG3Oxp6G0U5j+mXgIOBq4Fzg2e30sTNlfg34bFXdkuQFwP5V9cz59rvTTjvVmjVrxmqbJEmStDHnnXfeDVW1eq5tk/iC3yOBy6vqCoAk72A4a9ZPwnJVfXyk/GdYwO9UrlmzhvXr10+geZIkSVJfO3vsnCYxDeMBDKcLnXF1W9fzPODfJlCvJEmStKgmMbKcOdbNObcjyW8Da4E5T7Wa5CjgKIA99thjAk2TJEmS7rpJjCxfDew+cns3hlOG3kmSA4HjgKdU1W1z7aiqTqqqtVW1dvXqOaeNSJIkSUtmEmH5XGDvJD+bZCvgWcAZowWS/CLwOoagfN0E6pQkSZIW3dhhuapuB44BPgJ8ATi1qi5J8tIkT2nFXglsC5yW5IIkZ3R2J0mSJG02JnK666r6EPChWeuOH1k+cBL1SJIkSUvJM/hJkiRJHYZlSZIkqcOwLEmSJHUYliVJkqQOw7IkSZLUYViWJEmSOgzLkiRJUsdEfmdZ0l2z5tgPTrsJG7XhxIOn3QRJkqbGkWVJkiSpw7AsSZIkdRiWJUmSpA7DsiRJktRhWJYkSZI6DMuSJElSh2FZkiRJ6jAsS5IkSR2GZUmSJKnDM/hJmojlcDZC8IyEkqRN48iyJEmS1GFYliRJkjoMy5IkSVKHYVmSJEnqMCxLkiRJHYZlSZIkqcOwLEmSJHUYliVJkqQOw7IkSZLUYViWJEmSOgzLkiRJUodhWZIkSeowLEuSJEkdhmVJkiSpw7AsSZIkdRiWJUmSpA7DsiRJktRhWJYkSZI6DMuSJElSh2FZkiRJ6jAsS5IkSR0TCctJnpjkS0kuT3LsHNsfl+T8JLcnOWwSdUqSJEmLbeywnGQL4J+AJwH7AM9Oss+sYl8DjgTeNm59kiRJ0lJZNYF9PBK4vKquAEjyDuAQ4NKZAlW1oW378QTqkyRJkpbEJKZhPAC4auT21W3dJktyVJL1SdZff/31E2iaJEmSdNdNIixnjnV1V3ZUVSdV1dqqWrt69eoxmyVJkiSNZxJh+Wpg95HbuwHXTGC/kiRJ0lRNIiyfC+yd5GeTbAU8CzhjAvuVJEmSpmrssFxVtwPHAB8BvgCcWlWXJHlpkqcAJPnlJFcDTwdel+SSceuVJEmSFtskfg2DqvoQ8KFZ644fWT6XYXqGJEmStGx4Bj9JkiSpYyIjy5KkzdeaYz847SYsyIYTD552EyTppziyLEmSJHUYliVJkqQOw7IkSZLUYViWJEmSOgzLkiRJUodhWZIkSeowLEuSJEkdhmVJkiSpw7AsSZIkdRiWJUmSpA5Pdy1JkiZiOZxa3dOqa1M5sixJkiR1GJYlSZKkDsOyJEmS1GFYliRJkjoMy5IkSVKHYVmSJEnqMCxLkiRJHYZlSZIkqcOwLEmSJHUYliVJkqQOw7IkSZLUYViWJEmSOgzLkiRJUodhWZIkSeowLEuSJEkdhmVJkiSpw7AsSZIkdRiWJUmSpI5V026AJEnS5mjNsR+cdhM2asOJB0+7CSueI8uSJElSh2FZkiRJ6jAsS5IkSR2GZUmSJKnDsCxJkiR1GJYlSZKkjomE5SRPTPKlJJcnOXaO7VsneWfb/tkkayZRryRJkrSYxg7LSbYA/gl4ErAP8Owk+8wq9jzg21X1X4BXA68Yt15JkiRpsU1iZPmRwOVVdUVV/SfwDuCQWWUOAU5py+8CDkiSCdQtSZIkLZpU1Xg7SA4DnlhVz2+3nwPsV1XHjJS5uJW5ut3+Sitzw6x9HQUcBbDHHns84sorrxyrbXfVSjpjz3LoC6ys/ng2pZXB55qWis81LYXl8DyD6T3XkpxXVWvn2jaJkeW5RohnJ/CFlKGqTqqqtVW1dvXq1RNomiRJknTXTSIsXw3sPnJ7N+CaXpkkq4DtgW9NoG5JkiRp0UwiLJ8L7J3kZ5NsBTwLOGNWmTOAI9ryYcDHatz5H5IkSdIiWzXuDqrq9iTHAB8BtgDeWFWXJHkpsL6qzgDeALw5yeUMI8rPGrdeSZIkabGNHZYBqupDwIdmrTt+ZPkHwNMnUZckSZK0VDyDnyRJktRhWJYkSZI6DMuSJElSh2FZkiRJ6jAsS5IkSR2GZUmSJKnDsCxJkiR1GJYlSZKkDsOyJEmS1GFYliRJkjoMy5IkSVKHYVmSJEnqMCxLkiRJHYZlSZIkqcOwLEmSJHUYliVJkqQOw7IkSZLUYViWJEmSOgzLkiRJUodhWZIkSepYNe0GSJtiw4kHT7sJkiTpbsSRZUmSJKnDsCxJkiR1GJYlSZKkDsOyJEmS1GFYliRJkjoMy5IkSVKHYVmSJEnqMCxLkiRJHYZlSZIkqcOwLEmSJHUYliVJkqQOw7IkSZLUsWraDZAk6e5sw4kHT7sJkubhyLIkSZLUYViWJEmSOgzLkiRJUodhWZIkSeoYKywn2THJWUkua9f36ZT7cJKbknxgnPokSZKkpTTuyPKxwLqq2htY127P5ZXAc8asS5IkSVpS44blQ4BT2vIpwKFzFaqqdcD3xqxLkiRJWlLjhuWdq+pagHZ9v3F2luSoJOuTrL/++uvHbJokSZI0no2elCTJR4Fd5th03KQbU1UnAScBrF27tia9f0mSJGlTbDQsV9WBvW1Jvplk16q6NsmuwHUTbZ0kSZI0ReNOwzgDOKItHwGcPub+JEmSpM3GuGH5ROCgJJcBB7XbJFmb5PUzhZJ8EjgNOCDJ1Ul+Y8x6JUmSpEW30WkY86mqG4ED5li/Hnj+yO3HjlOPJEmSNA2ewU+SJEnqMCxLkiRJHYZlSZIkqWOsOcva/G048eBpN0GSJGnZcmRZkiRJ6jAsS5IkSR2GZUmSJKnDsCxJkiR1GJYlSZKkDsOyJEmS1GFYliRJkjoMy5IkSVKHYVmSJEnqMCxLkiRJHYZlSZIkqcOwLEmSJHUYliVJkqQOw7IkSZLUYViWJEmSOgzLkiRJUodhWZIkSeowLEuSJEkdq6bdAEmSJC2uDScePO0mLFuOLEuSJEkdhmVJkiSpw7AsSZIkdRiWJUmSpA7DsiRJktRhWJYkSZI6DMuSJElSh2FZkiRJ6jAsS5IkSR2GZUmSJKnDsCxJkiR1GJYlSZKkDsOyJEmS1GFYliRJkjoMy5IkSVLHWGE5yY5JzkpyWbu+zxxl9k3y6SSXJLkoyTPHqVOSJElaKuOOLB8LrKuqvYF17fZstwDPraqHAE8E/i7JDmPWK0mSJC26ccPyIcApbfkU4NDZBarqy1V1WVu+BrgOWD1mvZIkSdKiGzcs71xV1wK06/vNVzjJI4GtgK90th+VZH2S9ddff/2YTZMkSZLGs2pjBZJ8FNhljk3HbUpFSXYF3gwcUVU/nqtMVZ0EnASwdu3a2pT9S5IkSZO20bBcVQf2tiX5ZpJdq+raFoav65S7N/BB4CVV9Zm73FpJkiRpCY07DeMM4Ii2fARw+uwCSbYC3gu8qapOG7M+SZIkacmMG5ZPBA5KchlwULtNkrVJXt/KPAN4HHBkkgvaZd8x65UkSZIW3UanYcynqm4EDphj/Xrg+W35LcBbxqlHkiRJmgbP4CdJkiR1GJYlSZKkDsOyJEmS1GFYliRJkjoMy5IkSVKHYVmSJEnqMCxLkiRJHYZlSZIkqcOwLEmSJHUYliVJkqQOw7IkSZLUYViWJEmSOgzLkiRJUodhWZIkSeowLEuSJEkdhmVJkiSpw7AsSZIkdRiWJUmSpA7DsiRJktRhWJYkSZI6DMuSJElSh2FZkiRJ6jAsS5IkSR2GZUmSJKnDsCxJkiR1GJYlSZKkDsOyJEmS1GFYliRJkjoMy5IkSVKHYVmSJEnqMCxLkiRJHYZlSZIkqcOwLEmSJHUYliVJkqQOw7IkSZLUYViWJEmSOgzLkiRJUodhWZIkSeoYKywn2THJWUkua9f3maPMnknOS3JBkkuSHD1OnZIkSdJSGXdk+VhgXVXtDaxrt2e7FviVqtoX2A84Nsn9x6xXkiRJWnTjhuVDgFPa8inAobMLVNV/VtVt7ebWE6hTkiRJWhLjBtedq+pagHZ9v7kKJdk9yUXAVcArquqaMeuVJEmSFt2qjRVI8lFglzk2HbfQSqrqKuBhbfrF+5K8q6q+OUddRwFHAeyxxx4L3b0kSZK0KDYalqvqwN62JN9MsmtVXZtkV+C6jezrmiSXAI8F3jXH9pOAkwDWrl1bG2ubJEmStJjGnYZxBnBEWz4COH12gSS7JdmmLd8H+FXgS2PWK0mSJC26ccPyicBBSS4DDmq3SbI2yetbmQcDn01yIfAJ4G+r6vNj1itJkiQtuo1Ow5hPVd0IHDDH+vXA89vyWcDDxqlHkiRJmgZ/xk2SJEnqMCxLkiRJHYZlSZIkqcOwLEmSJHUYliVJkqQOw7IkSZLUYViWJEmSOgzLkiRJUodhWZIkSeowLEuSJEkdhmVJkiSpw7AsSZIkdRiWJUmSpA7DsiRJktRhWJYkSZI6DMuSJElSh2FZkiRJ6jAsS5IkSR2GZUmSJKnDsCxJkiR1GJYlSZKkDsOyJEmS1GFYliRJkjpWTbsBkrQ52nDiwdNugiRpM+DIsiRJktRhWJYkSZI6DMuSJElSh2FZkiRJ6jAsS5IkSR2GZUmSJKnDsCxJkiR1GJYlSZKkDsOyJEmS1GFYliRJkjoMy5IkSVKHYVmSJEnqMCxLkiRJHYZlSZIkqWOssJxkxyRnJbmsXd9nnrL3TvL1JK8Zp05JkiRpqYw7snwssK6q9gbWtds9LwM+MWZ9kiRJ0pIZNywfApzSlk8BDp2rUJJHADsDZ45ZnyRJkrRkxg3LO1fVtQDt+n6zCyS5B/Aq4I/HrEuSJElaUqs2ViDJR4Fd5th03ALreCHwoaq6KsnG6joKOApgjz32WODuJUmSpMWx0bBcVQf2tiX5ZpJdq+raJLsC181R7NHAY5O8ENgW2CrJzVX1U/Obq+ok4CSAtWvX1kI7IUmSJC2GjYbljTgDOAI4sV2fPrtAVR0+s5zkSGDtXEFZkiRJ2tyMO2f5ROCgJJcBB7XbJFmb5PXjNk6SJEmaprFGlqvqRuCAOdavB54/x/qTgZPHqVOSJElaKp7BT5IkSeowLEuSJEkdhmVJkiSpw7AsSZIkdRiWJUmSpA7DsiRJktRhWJYkSZI6DMuSJElSh2FZkiRJ6jAsS5IkSR2GZUmSJKnDsCxJkiR1GJYlSZKkDsOyJEmS1GFYliRJkjoMy5IkSVKHYVmSJEnqMCxLkiRJHYZlSZIkqcOwLEmSJHWsmnYDNkcbTjx42k2QJEnSZsCRZUmSJKnDsCxJkiR1GJYlSZKkDsOyJEmS1GFYliRJkjoMy5IkSVKHYVmSJEnqMCxLkiRJHYZlSZIkqcOwLEmSJHUYliVJkqQOw7IkSZLUYViWJEmSOgzLkiRJUkeqatptmFOS64Erp92OCdoJuGHajZiQldQXWFn9WUl9gZXVn5XUF7A/m7OV1BdYWf1ZSX2BldWfPatq9VwbNtuwvNIkWV9Va6fdjklYSX2BldWfldQXWFn9WUl9AfuzOVtJfYGV1Z+V1BdYef3pcRqGJEmS1GFYliRJkjoMy0vnpGk3YIJWUl9gZfVnJfUFVlZ/VlJfwP5szlZSX2Bl9Wcl9QVWXn/m5JxlSZIkqcORZUmSJKnDsKw5JXl9kn3a8s3Tbs+0JFmT5OJpt2NGkhcl+UKSt067LQux2O1NckKSFy/Gvhdbkv2TfGDa7ZAkzc+wrDlV1fOr6tJpt2OpJFk17TYs0AuBJ1fV4RsruJn0acHt1eYjyZFJXjPmPjYk2WmBZXdJ8o4kX0lyaZIPJfm5ceof2fdS92W3JKcnuSzJFUlek2Trceqftf+l7s9xSS5JclGSC5LsN2bdi/4GN8kbk1y3FAMdi92fJLsn+XgbdLgkyR8sYl2L3ZefSfIfSS5sffnLxapr0gzLE5LkfUnOa0+Ao9q65yX5cpKzk/zLzD+4JKuTvDvJue3yq1Nu+72SfLA9gS9O8szW5rUjZV6V5Pwk65Ksbute1F7YLkryjrbuhCRvTvKx9mLxu0vYjzVJvpjklNamdyW5Z5Lj23G+OMlJSdLKn53k5Uk+AfxBkp2TvLcdhwuT/Erb9Rbt8bskyZlJtlmqPs3q3z8DDwTOSPKnSf5fks+16we1MkcmOS3J+4Ez27o/bv2/aCn/Oc1q73HtBezc1uZDRtr7viTvT/LVJMck+aNW5jNJdmzlfrfd98L2t3PPOerbK8mH29/hJ5P8/BL0ceY59/r2/HprkgOTnNOe/49sl596rGbt515zHZ/lKskWm1A2wHuBs6tqr6raB/hfwM6L1b5NcRf68h7gfVW1N7A3sA3wN4vUvE22if15NPBfgV+qqocBBwJXLVbbJuhk4InTbsSE3A78z6p6MPAo4H+kfeq7DN0GPKGqHg7sCzwxyaOm3KaFqSovE7gAO7brbYCLgQcAG4AdgS2BTwKvaWXeBjymLe8BfGHKbX8a8C8jt7cHzgbWttsFHN6Wjx/pxzXA1m15h3Z9AnBhOw47Mfxjvf8S9WNNa+uvtttvBF4889i0dW8G/ltbPht47ci2dwJ/2Ja3aMdhDcM/q33b+lOB357iY7WhHdd7A6vaugOBd7flI4GrR56Pv87wbeUwvDn+APC4KbT35TPHDdgB+DJwr9bey4HtgNXAd4CjW7lXjzwe9x3Z518Bvz/yfHtxW14H7N2W9wM+tkTPuduBX2jH97z2vAtwCPC+eR6r/YEPtOU5j88E2vfbwH8AFwCva8/rm4FXtLZ+FHhk+1u4AnjKyPPodODDwJeAvxjZ5/vafS8BjhpZfzPwUuCzwGNGHvtt2n5+t9PGJwD/vkL6csDsvrTH/9vAtsuwP78JvH/M5+BzgYsYXhfezJ3/Zn8XOLdtezdwz7b+6QyvoxfOHE/gISPH6yLa3/pG/jYvXoS/+an0Z6T+04GDlntfgHsC5wP7TfoxWozL1BuwUi7cERIvZHjBPxY4ZWT7i7gjZF7XnlQzl68D202x7T8HfJXhn/Rj27qzuSMs/4g7XuwfCFzQlj8MvIvhn/62I8fhpSP7fhNw6BL1Yw3wtZHbT2B48Xgaw4vE59uxPnakj48fKX89LfzP2udlI7f/FHjJFB+rDQwvcrszjMZd3Pr1xbb9SOBfR8r/bbvPzHPtcuB5U2jv+tbWmXZ8DXhwa+/oG7WvAQ9oy78D/F1bfjzDG87Pt+fqP488314MbAvcOuvvatHfhM7x/HgTd7yxfGBrR++x2p87wvKcx2fMtj0YeD+wZbv9WoYXxwKe1Na9l+ETiC2Bh3PH3/aRwLXAfbljAGDm/8HsgYH7ttsFPGPWY7+GIfQ9d552vgh49UruC/A52hvuZdafbRmej19ubXz8Jj4HH8IQ6HeaaR93DmS9N8Gf547/AzMDMf/IHX9bWwHbLOBvc6JheZr9GenT14B7L9e+MLwpvID2xnCSj89iXjaHOY3LXpL9GUaMHl1VtyQ5m+FJ+ODOXe7Ryt66NC2cX1V9OckjgCcD/zvJmRu7S7s+GHgc8BTgz5M8ZNb22eWXwlx1v5bhxeSqJCcAPzOy/fsL2OdtI8s/YnghmraXAR+vqqcmWcMQ/GeM9inA/66q1y1d0+YU4GlV9aU7rRzmP44e3x+P3P4x/OR/1MkMb7ouTHIkQ9AcdQ/gpqrad7LNXpCNtX++x2rGnMdnTAcAjwDObTOPtmF4o/6fDG90YXjhu62qfpjk8wwvxjPOqqobAZK8h2FEcj3woiRPbWV2Z5hqcCPD38a7Z7XhdOBvqmrcL3gul76Euf/fZTn2p6pubq8NjwV+DXhnkmOr6uTefWZ5AvCuqrqh7e9brb8zHprkrxg+TdkW+Ehbfw5wcpJTGaa1AHwaOC7JbsB7quqyBbZhkqbWnyTbMjyGf1hV312ufamqHwH7JtkBeG+Sh1bVZvMl+h7nLE/G9sC3W1D+eYZ5RfcEHp/kPhm+aPW0kfJnAsfM3EgyjRf4n0hyf+CWqnoLw0jkL80qcg/gsLb8W8CnktwD2L2qPg78CXf8QQEckmEi/30ZQs25i9yFUXu0eXYAzwY+1ZZvaP9sDpv7bsDwMf4LYJjXl+Tei9fMsW3PMEoOw0hTz0eA32l9J8kDktxvkdvWa8fvtzmdJPnFTbz/dsC1SbYEfurLgu3F46tJnt72nyQPH7PNk7KQx2rc4zOXMHy6tW+7PKiqTgB+WG2Ih5FwX1Wjb05gjjeeswYGHs4wYjrz5vMH7YVw1DnAk2b61XEJQ3BcKX1ZO7qi/R/ZmWEAZbn1h6r6UVWdXVV/wfC69bT5ys/Se3cZq3EAAAVNSURBVPMw42TgmKr6BeAvZ9pbVUcDL2EI/BckuW9VvY1hYOZW4CNJnrAJ7ZiUqfSn/d97N/DWqnpPr9wmmupjU1U3MQwcLIu55YblyfgwsCrJRQyjSJ9heHF8OcPH/x8FLmWYngHDR3VrM3zh6lLg6KVv8p38AvAfSS4AjmP4yGXU94GHJDmP4d3oSxk+SnlLG/H4HMNHjze18v8BfJDhOLysqq5Zgj7M+AJwRHssdgT+L/AvDKM072P+4P4HwK+1Pp3H8DHV5upvGD4FOIfhsZhTVZ3JMEf+061f72IInkvtZQwfJ1+U4RvqL9vE+/85w9/SWcAXO2UOB56X5EKG0LK5fEluIY/VuMdnLuuAw2beHCXZMcmem3D/g9p9tgEOZQhXcw0MzOd4hpHN185T5mPA1hn5MnCSX07y+GXYl3XAPZM8t7VzC+BVDFPwbp1VbrPvT5IHJdl7ZNW+wJWb0M51wDPawAlpX9gdMeeb4CR7VdVnq+p44AZg9yQPBK6oqn8AzgAetgntmJQl7097M/MGhmll/2eZ92V1G1GmPXcPpP//fPMyrfkfd4cLd8zjXcUwP+2p027TEvT5BNqcpynUvYZF+EKHFy/L9QI8kzu+dHMeQ4C6eWT7nf5eZ7YxjICfyvCm9ydfIgO2Bv6t7e80hpGh/UfvO7KvDQzz1QP8K8NH/r123r/V9xWGNzofZNaXhJZRX3ZnCAyXATcBr1uujw3DiP//YxjsuYjhY/edNvE5eAR3fCHsZO48L/YFDN9BOJth3uvJbf17GAY4Lgb+vrXzz9pz4wKGAaod56nz7Qzzun/I8IXniX1PY6n7wzDFptrxn/lOw5OXaV8exjC4dlG7//GTelwW++LprhdRkr9leOf0MwxTL/6gVvgBzzAn+Oaq+tsp1L2G4QtTD13quiVptgw/P/l24Der6rxpt0fSXWNYliRJkjr8NQxJ0qJrcyPXzbHpgGq/7LBcrKS+wPLsz3Js83xWUn9WUl9mOLIsSZIkdfhrGJIkSVKHYVmSJEnqMCxL0pQk+VGSC0Yua+7CPnZI8sLJt06SBM5ZlqSpSXJzVW278ZLz7mMNd+EnE5NsUT99VjdJ0iyOLEvSZqSdav2VSc5tZ/n8vbZ+2yTrkpyf5PNJZs5QeCKwVxuZfmWS/ZN8YGR/r0lyZFvekOT4JJ8Cnp5kryQfTnJekk+2M7+R5OlJLk5yYZJ/X9ojIEmbF386TpKmZ5t2mnmAr1bVU4HnAd+pql9OsjVwTpIzgasYzgL63SQ7AZ9JcgZwLPDQqtoXIMn+G6nzB1X1mFZ2HXB0VV2WZD+GUx8/geFUyL9RVV+fOT2tJN1dGZYlaXpunQm5I34deFiSw9rt7YG9GU7b+/IkjwN+DDwA2Pku1PlOGEaqgV8BTksys23rdn0OcHKSUxlObytJd1uGZUnavAT4/ar6yJ1WDlMpVgOPqKofJtkA/Mwc97+dO0+xm13m++36HsBNc4R1quroNtJ8MHBBkn2X68kEJGlczlmWpM3LR4AXJNkSIMnPJbkXwwjzdS0o/xqwZyv/PWC7kftfCeyTZOsk2wMHzFVJVX0X+GqSp7d6kuThbXmvqvpsVR0P3ADsPvluStLy4MiyJG1eXg+sAc7PMD/ieuBQ4K3A+5OsBy4AvghQVTcmOSfJxcC/VdUft+kTFwGXAZ+bp67Dgf+b5CXAlsA7gAuBVybZm2GUe11bJ0l3S/50nCRJktThNAxJkiSpw7AsSZIkdRiWJUmSpA7DsiRJktRhWJYkSZI6DMuSJElSh2FZkiRJ6jAsS5IkSR3/H4t4oiHqK8DDAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "# Helper method to print importances and visualize distribution\n", + "def visualize_importances(feature_names, importances, title=\"Average Feature Importances\", plot=True, axis_title=\"Features\"):\n", + " print(title)\n", + " for i in range(len(feature_names)):\n", + " print(feature_names[i], \": \", '%.3f'%(importances[i]))\n", + " x_pos = (np.arange(len(feature_names)))\n", + " if plot:\n", + " plt.figure(figsize=(12,6))\n", + " plt.bar(x_pos, importances, align='center')\n", + " plt.xticks(x_pos, feature_names, wrap=True)\n", + " plt.xlabel(axis_title)\n", + " plt.title(title)\n", + "visualize_importances(feature_names, np.mean(attr, axis=0))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "From the feature attribution information, we obtain some interesting insights regarding the importance of various features. We see that the strongest features appear to be age and being male, which are negatively correlated with survival. Embarking at Queenstown and the number of parents / children appear to be less important features generally." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "An important thing to note is that the average attributions over the test set don't necessarilly capture all the information regarding feature importances. We should also look at the distribution of attributions for each feature. It is possible that features have very different attributions for different examples in the dataset. \n", + "\n", + "For instance, we can visualize the distribution of attributions for sibsp, the number of siblings / spouses." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAXOElEQVR4nO3dfbRldX3f8fdHQDRCBJwBYUAGlFiRFUc7oSxNVkzAiGgEu0SxPowGF7HRGI1tBTWJWqnYVE3SNrowWAd8AHwqRLGCKEXbCA4WeRCUERCGQWYQefIBBL79Y/+GHC73zj33aWbuz/drrbPOPr/923t/f2ff8zn77rPvuakqJEl9ecTWLkCSNP8Md0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuW0GSDyf5i3la1xOS3J1ku/b4giSvnY91t/V9Kcmq+VrfDLb7niS3JvnRLJZ9sOYkr07yjfmvcMvodf9OJ8n1SQ7b2nUsZob7PGs/lD9PcleS25P83ySvS/Lgc11Vr6uq/zjmujb7A15VN1TVTlV1/zzU/s4kH5+w/udV1eq5rnuGdewDvAU4sKoeP0WftyW5rgXfuiRnbJq3NWoeqeuCJD9JsuOE9o8lec+Eti73b5ITklw4SfuSJPcmOWg+t6fJGe4L4w+ramdgX+Ak4K3AKfO9kSTbz/c6txH7Aj+uqg2TzWxHmq8EDquqnYCVwPlbsL5JJVkO/A5QwAvnYX2Ldf+eBjwzyX4T2o8BLq+qK7ZCTb96qsrbPN6A6xlCZ7TtYOAB4KD2+GPAe9r0EuALwO3AbcDXGd50T2vL/By4G/gPwHKG4DgWuAG4cKRt+7a+C4D3AhcDdwBnAbu1ec8G1k1WL3A4cC/wy7a974ys77Vt+hHAO4AfAhuAU4HHtnmb6ljVarsVePtmnqfHtuU3tvW9o63/sDbmB1odH5tk2f8G/M1m1j1a86uB/wP81/Z8XA0cOtL31cC1wF3AdcDLx1luiu3+ZVvmA8AXRtqPa8/rvW1M//grsH/PBf5yQtvFwBvb9BOBrwI/buv6BLDLZK8jRl4vk40T2Av4LMPP0nWbtjHy2lsD3AncAnxga2fEFsuirV1AbzcmCffWfgPwb9v0gz+s7YX6YWCHdvsdIJOta+QFdirwGODRU7z4bwIOan0+C3y8zZvyxd+m37mp78j80Rf/HwFrgf2BnYDPAadNqO0jra6nAfcAT5nieTqVIZh2bst+Hzh2qjonLPsKhjfCf89w1L7dZmp+NXAf8Ob2/L6UIRR3a8/PncCTW989gadOt9xm6loL/AnwLxlCdI+ReQ/u86l+Vjrbvy8Hrhl5/GSGN5el7fGTgOcAOwJLGd7I/maKuh/y3I2Ok+EN6RKGN9ZHttqvBZ7b5v8T8Mo2vRNwyNbOiC1187TMlrOeIVAm+iVDqOxbVb+sqq9X+0ncjHdW1U+r6udTzD+tqq6oqp8CfwG8ZNMHcnP0coYjn2ur6m7gBOCYCacP3lVVP6+q7wDfYQiBh2i1vBQ4oaruqqrrgfcznGqZVlV9HPhT4LnA/wY2JDl+M4tsYAiOX1bVGcD3gOe3eQ8AByV5dFXdXFVXjrncxDH9NsPppDOr6hLgB8C/GWc8k1jU+7f5PLBHkme2x68CvlRVGwGqam1VnVdV97S2DwC/O4uaf4vhDePdVXVvVV3L8AZ0TJv/S+BJSZZU1d1V9c1ZbGNRMty3nGUMR5sT/TXD0dK5Sa6dJqQ2uXEG83/IcOS5ZKwqN2+vtr7RdW8P7DHSNnp1y88YjpYmWsJwlDVxXcvGLaSqPlFVhwG7AK8D3p3kuVN0v2nCG+YPgb1aOL60LX9zki8m+RfTLTfFNlYB51bVre3xJ1vbbCz2/UtV/Qz4NPCqJGF443jwg9skuyc5PclNSe4EPj7LMewL7NUuXrg9ye3A20ZqPhb4DeDqJN9K8oJZbGNRMty3gCS/xRBcD7skrx25vqWq9gf+EPjzJIdumj3FKqc7st9nZPoJDEcvtwI/BX5tpK7tGH4lHne96xleTKPrvo/hXOZM3Npqmrium2a4HtpR9aeByxhOVUxmWQuY0W2tb8t/uaqew/Db09UMR33TLjcqyaOBlwC/m+RH7fLNNwNPS7LpyHay57bX/bvJaobn5TkMp9++MDLvva2e36yqX2c41ZaHrWHwkHEBo1dQ3QhcV1W7jNx2rqojAKrqmqp6GbA78D7gM0keM8vxLCqG+wJK8uvtSOF0hnOdl0/S5wVJntRC5E7g/naD4UW1/yw2/YokByb5NeDdwGdquJTu+8Cjkjw/yQ4MH56NXrJ3C7B89LLNCT4FvDnJfkl2Av4TcEZV3TeT4lotZwInJtk5yb7AnzMcvU2rXbv+/LbsI5I8D3gqcNEUi+wOvDHJDkmOBp4CnJNkjyQvbC/2exg+aLx/uuUmWf9RbbkDgRXt9hSGD8df1fpMti+73L8jvs5wocDJwOlVde/IvJ0Znu/bkyxj+PxkKpcCRyTZLcnjgTeNzLsYuDPJW5M8Osl2SQ5qB1QkeUWSpVX1QKsFHrqPu2W4L4x/THIXw1HF2xnOJ75mir4HAF9h+EH/J+Dvq+qCNu+9wDvar5v/bgbbP43hQ6gfAY8C3ghQVXcwfOD3DwxHyT8F1o0s9+l2/+Mk355kvR9t676Q4aqEXzCc+56NP23bv5bhN5pPtvWP406GX71vYHjB/meGD6un+mOlixie51uBE4EXV9WPGX7+38JwxHobwznfPxljuYlWAf+jhmvSf7TpxnBVz8vbOetTgAPbvvyfbbme9y/tlNapDL8NnDph9ruAZzB8SP1Fhg9vp3Iaw/n96xmuwhn9m4b7GX7jXdFqvpVh/I9tXQ4HrkxyN/C3wDFV9YvZjmkx2XRVhqQRSV7NcBXJb2/tWqTZ8MhdkjpkuEtShzwtI0kd8shdkjq0TXwx0ZIlS2r58uVbuwxJWlQuueSSW6tq6WTztolwX758OWvWrNnaZUjSopLkh1PN87SMJHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1aJv4C1VJ+lWy/PgvPjh9/UmT/s/1OfPIXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDk0b7kkeleTiJN9JcmWSd7X2/ZJclOSaJGckeWRr37E9XtvmL1/YIUiSJhrnyP0e4Per6mnACuDwJIcA7wM+WFUHAD8Bjm39jwV+UlVPAj7Y+kmStqBpw70Gd7eHO7RbAb8PfKa1rwaOatNHtse0+YcmybxVLEma1ljn3JNsl+RSYANwHvAD4Paquq91WQcsa9PLgBsB2vw7gMdNss7jkqxJsmbjxo1zG4Uk6SHGCvequr+qVgB7AwcDT5msW7uf7Ci9HtZQdXJVrayqlUuXLh23XknSGGZ0tUxV3Q5cABwC7JJk0xeP7Q2sb9PrgH0A2vzHArfNR7GSpPGMc7XM0iS7tOlHA4cBVwFfA17cuq0CzmrTZ7fHtPlfraqHHblLkhbOOF/5uyewOsl2DG8GZ1bVF5J8Fzg9yXuA/wec0vqfApyWZC3DEfsxC1C3JGkzpg33qroMePok7dcynH+f2P4L4Oh5qU6SNCv+haokdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHZo23JPsk+RrSa5KcmWSP2vt70xyU5JL2+2IkWVOSLI2yfeSPHchByBJerjtx+hzH/CWqvp2kp2BS5Kc1+Z9sKr+y2jnJAcCxwBPBfYCvpLkN6rq/vksXJI0tWmP3Kvq5qr6dpu+C7gKWLaZRY4ETq+qe6rqOmAtcPB8FCtJGs+MzrknWQ48HbioNb0hyWVJPppk19a2DLhxZLF1TPJmkOS4JGuSrNm4ceOMC5ckTW3scE+yE/BZ4E1VdSfwIeCJwArgZuD9m7pOsng9rKHq5KpaWVUrly5dOuPCJUlTGyvck+zAEOyfqKrPAVTVLVV1f1U9AHyEfz71sg7YZ2TxvYH181eyJGk641wtE+AU4Kqq+sBI+54j3V4EXNGmzwaOSbJjkv2AA4CL569kSdJ0xrla5lnAK4HLk1za2t4GvCzJCoZTLtcDfwxQVVcmORP4LsOVNq/3ShlJ2rKmDfeq+gaTn0c/ZzPLnAicOIe6JElz4F+oSlKHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOjRtuCfZJ8nXklyV5Mokf9bad0tyXpJr2v2urT1J/i7J2iSXJXnGQg9CkvRQ4xy53we8paqeAhwCvD7JgcDxwPlVdQBwfnsM8DzggHY7DvjQvFctSdqsacO9qm6uqm+36buAq4BlwJHA6tZtNXBUmz4SOLUG3wR2SbLnvFcuSZrSjM65J1kOPB24CNijqm6G4Q0A2L11WwbcOLLYutY2cV3HJVmTZM3GjRtnXrkkaUpjh3uSnYDPAm+qqjs313WStnpYQ9XJVbWyqlYuXbp03DIkSWMYK9yT7MAQ7J+oqs+15ls2nW5p9xta+zpgn5HF9wbWz0+5kqRxjHO1TIBTgKuq6gMjs84GVrXpVcBZI+2valfNHALcsen0jSRpy9h+jD7PAl4JXJ7k0tb2NuAk4MwkxwI3AEe3eecARwBrgZ8Br5nXiiVJ05o23KvqG0x+Hh3g0En6F/D6OdYlSZoD/0JVkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUoWnDPclHk2xIcsVI2zuT3JTk0nY7YmTeCUnWJvlekucuVOGSpKmNc+T+MeDwSdo/WFUr2u0cgCQHAscAT23L/H2S7earWEnSeKYN96q6ELhtzPUdCZxeVfdU1XXAWuDgOdQnSZqFuZxzf0OSy9ppm11b2zLgxpE+61rbwyQ5LsmaJGs2btw4hzIkSRPNNtw/BDwRWAHcDLy/tWeSvjXZCqrq5KpaWVUrly5dOssyJEmTmVW4V9UtVXV/VT0AfIR/PvWyDthnpOvewPq5lShJmqlZhXuSPUcevgjYdCXN2cAxSXZMsh9wAHDx3EqUJM3U9tN1SPIp4NnAkiTrgL8Cnp1kBcMpl+uBPwaoqiuTnAl8F7gPeH1V3b8wpUuSpjJtuFfVyyZpPmUz/U8ETpxLUZKkufEvVCWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nq0LThnuSjSTYkuWKkbbck5yW5pt3v2tqT5O+SrE1yWZJnLGTxkqTJjXPk/jHg8AltxwPnV9UBwPntMcDzgAPa7TjgQ/NTpiRpJqYN96q6ELhtQvORwOo2vRo4aqT91Bp8E9glyZ7zVawkaTyzPee+R1XdDNDud2/ty4AbR/qta20Pk+S4JGuSrNm4ceMsy5AkTWa+P1DNJG01WceqOrmqVlbVyqVLl85zGZL0q2224X7LptMt7X5Da18H7DPSb29g/ezLkyTNxmzD/WxgVZteBZw10v6qdtXMIcAdm07fSJK2nO2n65DkU8CzgSVJ1gF/BZwEnJnkWOAG4OjW/RzgCGAt8DPgNQtQsyRpGtOGe1W9bIpZh07St4DXz7UoSdLc+BeqktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDm0/l4WTXA/cBdwP3FdVK5PsBpwBLAeuB15SVT+ZW5mSpJmYjyP336uqFVW1sj0+Hji/qg4Azm+PJUlb0EKcljkSWN2mVwNHLcA2JEmbMddwL+DcJJckOa617VFVNwO0+90nWzDJcUnWJFmzcePGOZYhSRo1p3PuwLOqan2S3YHzklw97oJVdTJwMsDKlStrjnVIkkbM6ci9qta3+w3A54GDgVuS7AnQ7jfMtUhJ0szMOtyTPCbJzpumgT8ArgDOBla1bquAs+ZapCRpZuZyWmYP4PNJNq3nk1X1v5J8CzgzybHADcDRcy9TkjQTsw73qroWeNok7T8GDp1LUZKkufEvVCWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nq0PZbuwBJ2lYtP/6LD05ff9Lzx543WZ8tbcHCPcnhwN8C2wH/UFUnLdS2JGkq44TwVP0XswUJ9yTbAf8deA6wDvhWkrOr6rsLsb2ezPQHcVvkGLbc+rfEcz2XbUy17FyOeudrnAsV4tvKm8NCHbkfDKytqmsBkpwOHAnMe7j3ECTj+FUZ50zNV/DMZvltzTihsiXeABbaOKE/VZ/eA31Uqmr+V5q8GDi8ql7bHr8S+FdV9YaRPscBx7WHTwa+N++FbF1LgFu3dhHzzDEtHj2OyzE93L5VtXSyGQt15J5J2h7yLlJVJwMnL9D2t7oka6pq5dauYz45psWjx3E5pplZqEsh1wH7jDzeG1i/QNuSJE2wUOH+LeCAJPsleSRwDHD2Am1LkjTBgpyWqar7krwB+DLDpZAfraorF2Jb27AeTzk5psWjx3E5phlYkA9UJUlbl18/IEkdMtwlqUOG+zxJ8tdJrk5yWZLPJ9llin7XJ7k8yaVJ1mzpOmdiBmM6PMn3kqxNcvyWrnMmkhyd5MokDySZ8hK0Rbafxh3TotlPAEl2S3Jekmva/a5T9Lu/7adLk2yTF25M99wn2THJGW3+RUmWz3Wbhvv8OQ84qKp+E/g+cMJm+v5eVa1YBNfsTjumka+aeB5wIPCyJAdu0Spn5grgXwMXjtF3seynace0CPcTwPHA+VV1AHB+ezyZn7f9tKKqXrjlyhvPmM/9scBPqupJwAeB9811u4b7PKmqc6vqvvbwmwzX9i9qY47pwa+aqKp7gU1fNbFNqqqrqqqrv4Yec0yLaj81RwKr2/Rq4KitWMtcjPPcj471M8ChSSb7Y9CxGe4L44+AL00xr4Bzk1zSvoJhsZhqTMuAG0cer2tti91i3U9TWYz7aY+quhmg3e8+Rb9HJVmT5JtJtsU3gHGe+wf7tAOqO4DHzWWjfp/7DCT5CvD4SWa9varOan3eDtwHfGKK1TyrqtYn2R04L8nVVTXOKYIFMQ9jmvarJra0ccY0hkW3n6ZbxSRtW/066M2NawareULbV/sDX01yeVX9YH4qnBfjPPfzvn8M9xmoqsM2Nz/JKuAFwKE1xR8QVNX6dr8hyecZfmXbaqExD2Pa5r5qYroxjbmORbWfxrDN7SfY/LiS3JJkz6q6OcmewIYp1rFpX12b5ALg6cC2FO7jPPeb+qxLsj3wWOC2uWzU0zLzpP1zkrcCL6yqn03R5zFJdt40DfwBw4dh26RxxkSHXzWx2PbTmBbjfjobWNWmVwEP+w0lya5JdmzTS4BnsQBfLT5H4zz3o2N9MfDVqQ4Qx1ZV3ubhBqxlOGd2abt9uLXvBZzTpvcHvtNuVzL8Sr3Va5/LmNrjIxiupvnBIhjTixiOku4BbgG+3MF+mnZMi20/tXofx3CVzDXtfrfWvpLhv7sBPBO4vO2ry4Fjt3bdU4zlYc898G6GAyeARwGfbq+5i4H957pNv35AkjrkaRlJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjr0/wGW9omx+5ZJMQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plt.hist(attr[:,1], 100);\n", + "plt.title(\"Distribution of Sibsp Attribution Values\");" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We note that a vast majority of the examples have an attribution value of 0 for sibsp, which likely corresponds to having a value of 0 for the feature (IntegratedGradients would provide an attribution of 0 when the feature value matches the baseline of 0). More significantly, we see that although the average seems smaller in magnitude in the plot above, there are a small number of examples with extremely negative attributions for this feature." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To better understand this, we can bucket the examples by the value of the sibsp feature and plot the average attribution for the feature. In the plot below, the size of the dot is proportional to the number of examples with that value." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEGCAYAAABLgMOSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deZxcVZ338c83ewhJICRASIAQCDCgY8QWxWhkSdhGlihLcPRBB40LKujjgsvj4DaDPuPgzCgOYZGoCAxLBAVkk7CICJ0YshBDWALERAgkLEnI1v2bP+5pKDpV1dW3u7qq6O/79apX3Xvuuef+OvDqX99z7j1HEYGZmVln9al1AGZm1picQMzMLBcnEDMzy8UJxMzMcnECMTOzXPrVOoCeNHLkyBg3blytwzAzayhz5859LiJGtS/vVQlk3LhxNDc31zoMM7OGIunJYuXuwjIzs1ycQMzMLBcnEDMzy6VXjYF01iubW/jNgpX8/I/LeealTWzc0sLg/n0Zu+Ng/unde3HkAbsyoJ9zsJn1TupNc2E1NTVFJYPoz6/bxH/csYyrm1cgwYbNLdvUGTKwL30kPvzOPfn0Yfuw/UDnYjN7Y5I0NyKa2pf7t147j61ex/SZ9/PC+s1saS2dXNdvypLKJfc+wc2LVnHFxw9h1+GDeipMM7Oaq3n/i6SjJS2V9Kikc4ocHyjpqnT8T5LGFRz7aipfKumorsayYu0G3n/BfTy3blPZ5FFo09ZWnlrzCide8AfWrt/c1RDMzBpGTROIpL7AT4BjgAOA0yQd0K7aGcDaiNgHOB/4fjr3AGA6cCBwNHBBai+XrS2tnHbR/azbuJXO9uq1tAZr1m3io5c9mPfyZmYNp9Z3IAcDj0bE4xGxGbgSOKFdnROAWWn7GuAISUrlV0bEpoh4Ang0tZfL7//yLGvWbaYl55jQ5pZg6d9e5qGnX8gbgplZQ6l1AhkDPF2wvyKVFa0TEVuBF4GdKjwXSTMkNUtqXr16dclA/vuux1hfZLC8MzZtbeGiex7vUhtmZo2i1glERcra3wKUqlPJuUTEzIhoioimUaO2mcoFgCeeW8/ilS91FGuHWgNue/gZj4WYWa9Q6wSyAti9YH8ssLJUHUn9gOHAmgrPrcj9jz9PHxXLR53Xv28f5j21tlvaMjOrZ7VOIA8CEyTtJWkA2aD4De3q3ACcnrZPAn4f2csrNwDT01NaewETgAfyBPHChi1sbula91Wb1ghe2rilW9oyM6tnNX0PJCK2SvoMcAvQF7g0IhZL+jbQHBE3AJcAv5D0KNmdx/R07mJJ/wM8DGwFzoyIXFmgNaJI51d+ra3d15aZWb2q+YuEEXETcFO7sm8WbG8ETi5x7veA73U1hh2260//fn1o2dL13/x9JIYP7t/ldszM6l2tu7DqQtOeI7qtrS0trbxl9x26rT0zs3rlBALst+tQ9h61fZfbETB531GMGjqw60GZmdU5J5Dkk+/dmyEDc7/IDsDgAX2ZMXl8N0VkZlbfnECSow7clSED+hV9uaQSffuIsTsOpmnPHbs1LjOzeuUEkgzo14dfffwdbJfjLqSPYPig/vzijHegbnqfxMys3jmBFNhn56FcNeMQhg/uT58K80D/vmLU9gO57tPvYpdhns7dzHoPJ5B23jRmODed9R6OffNoBvbrw6D+xf+JthvQl0H9+zDtrWO4+ezJjBs5pIcjNTOrrZq/B1KPxuwwmB9/8CDWrt/MVQ8+xS//9BRr1m9m05ZWBvbvw67DBvHRSeOYdtBYr0RoZr2Wl7Q1M7OySi1p6y4sMzPLxQnEzMxycQIxM7NcnEDMzCwXJxAzM8vFCcTMzHJxAjEzs1ycQMzMLBcnEDMzy6UmCUTSCEm3SVqWvreZA13SREl/lLRY0gJJpxYcu0zSE5Lmp8/Env0JzMysVncg5wB3RMQE4I60394G4P9ExIHA0cCPJBWuFfuliJiYPvOrH7KZmRWqVQI5AZiVtmcBJ7avEBGPRMSytL0SeBYY1WMRmplZWbVKILtExCqA9L1zucqSDgYGAI8VFH8vdW2dL6nkIuSSZkhqltS8evXq7ojdzMyoYgKRdLukRUU+J3SyndHAL4CPRkRrKv4qsD/wdmAE8JVS50fEzIhoioimUaN8A2Nm1l2qtphFREwpdUzSM5JGR8SqlCCeLVFvGHAj8I2IuL+g7VVpc5OknwFf7MbQzcysArXqwroBOD1tnw5c376CpAHAbODnEXF1u2Oj07fIxk8WVTVaMzPbRq0SyHnAVEnLgKlpH0lNki5OdU4BJgMfKfK47uWSFgILgZHAd3s2fDMz84qEZmZWllckNDOzbuUEYmZmuTiBmJlZLk4gZmaWixOImZnl4gRiZma5VO1NdGtsC1a8wA9vfYSW1lY+e/gE3jF+p1qHZGZ1xgnEtvH0mg1Mn3k/Gza3AND85Atcf+Yk9tt1aI0jM7N64i4s28YDT6x53X5ra3Dvo8/VKBozq1dOILaN0cMHvW6/X19tU2Zm5gRi2zhk7504+W1j6d9X9O8rpv7dLhx94K61DsvM6oznwrKSXtywhdYIdhwyoNahmFkNlZoLy4PoVtLw7frXOgQzq2PuwjIzs1ycQMzMLBcnEDMzy8UJxMzMcnECMTOzXCp6CkvSGGDPwvoRcXe1gjIzs/rXYQKR9H3gVOBhoCUVB9ClBCJpBHAVMA5YDpwSEWuL1GsBFqbdpyLi+FS+F3AlMAKYB3w4IjZ3JSYzM6tcJV1YJwL7RcSxEXFc+hzfDdc+B7gjIiYAd6T9Yl6JiInpU3jd7wPnp/PXAmd0Q0xmZlahShLI40A13ig7AZiVtmeRJaqKSBJwOHBNnvPNzKzrKhkD2QDMl3QHsKmtMCI+18Vr7xIRq1JbqyTtXKLeIEnNwFbgvIj4NbAT8EJEbE11VgBjip0saQYwA2CPPfboYshmZtamkgRyQ/p0mqTbgWKz8H29E83sERErJY0Hfi9pIfBSkXpFJ/WKiJnATMjmwurEdc3MrIwOE0hEzJI0ANg3FS2NiC2VNB4RU0odk/SMpNHp7mM08GyJNlam78clzQHeClwL7CCpX7oLGQusrCQmMzPrHh2OgUg6FFgG/AS4AHhE0uRuuPYNwOlp+3Tg+iLX3lHSwLQ9EpgEPBzZFMJ3AieVO9/MzKqnkkH0HwJHRsR7I2IycBRwfjdc+zxgqqRlwNS0j6QmSRenOn8HNEt6iCxhnBcRD6djXwG+IOlRsjGRS7ohJjMzq1AlYyD9I2Jp205EPCKpy09lRcTzwBFFypuBj6Xt+4A3lzj/ceDgrsZhZmb5VJJAmiVdAvwi7f8jMLd6IZmZWSOoJIF8CjgT+BwgsjfQL6hmUGZmVv8qeQprE/Dv6WNmZgaUSSCS/iciTknvXWzz/kRE/H1VIzMzs7pW7g7krPT9vp4IxMzMGkvJx3jbphkBPh0RTxZ+gE/3THhmZlavKnkPZGqRsmO6OxAzM2ss5cZAPkV2p7G3pAUFh4YCf6h2YGZmVt/KjYH8CrgZ+Fdev1bHyxGxpqpRmZlZ3SuZQCLiReBFSV9pd2h7SdtHxFPVDc3MzOpZJS8S3kj2GK+AQcBewFLgwCrGZWZmda6SFwlfNxeVpIOAT1QtIjMzawiVPIX1OhExD3h7FWIxM7MG0uEdiKQvFOz2AQ4CVlctIjMzawiVjIEMLdjeSjYmcm11wjEzs0ZRyRjItwAkDct24+WqR2VmZnWvkiVtm9KEiguAhZIekvS26odmZmb1rJIurEvJ5sO6B0DSu4GfAZ6N18ysF6vkKayX25IHQETcC3SpG0vSCEm3SVqWvncsUucwSfMLPhslnZiOXSbpiYJjE7sSj5mZdV7JBCLpoPTOxwOSLpR0qKT3SroAmNPF654D3BERE4A7eP1UKQBExJ0RMTEiJgKHAxuAWwuqfKnteETM72I8ZmbWSeW6sH7Ybv+fC7a3WWCqk04ADk3bs8gSUvspUwqdBNwcERu6eF0zM+sm5ebCOqyK192lbb2RiFglaecO6k9n2yV1vyfpm6Q7mLT07jYkzQBmAOyxxx5di9rMzF5Vbjr3D0XEL9u9SPiqiCi7Rrqk24Fdixz6emcClDQaeDNwS0HxV4G/AQOAmWR3L98uEefMVIempqau3jmZmVlSrgtrSPoeWqZOSRExpdQxSc9IGp3uPkYDz5Zp6hRgdkRsKWi7bbXETZJ+BnwxT4xmZpZfuS6sCyX1BV6KiPO7+bo3AKcD56Xv68vUPY3sjuNVBclHwInAom6Oz8zMOlD2Md6IaAGOr8J1zwOmSlpGtmTuefDqS4sXt1WSNA7YHbir3fmXp5cbFwIjge9WIUYzMyujkhcJ75P0Y+AqYH1bYZqVN5eIeB44okh5M/Cxgv3lwJgi9Q7Pe20zM+selSSQd6XvwkHqIHs3w8zMeqlKEsgZEfF4YYGk8VWKx8zMGkQlU5lcU6Ts6u4OxMzMGku590D2J1v3fLik9xccGka2NrqZmfVi5bqw9gPeB+wAHFdQ/jLw8WoGZWZm9a/ceyDXA9dLmhwRdxcekzSp6pGZmVldq2QM5EdFyv6ruwMxM7PGUm4M5BCyR3hHtZsPaxjQt9qBmZlZfSs3BjIA2D7VKZwP6yWy6dXNzKwXKzcGchdwl6TLIuLJtnJJg8gG1Zf1QHxmZlanOhwDiYgnJfWVdIyknwPLgVOrHpmZmdW1sm+iS5oMfBD4B+ABYBIw3isDmplZuUH0FcBTwE/J1h9/WdITTh5mZgblu7CuJZsJ91TgOElD6Ppa6GZm9gZRMoFExFnAOLK1yA8DHiF7pPcUSdv3THhmZlavOlpQKiLi9xHxcbJk8kGyFQCXVz80MzOrZ5VM5w5AWpP8N8BvJA2uXkhmZtYIKpnKZBsR8Up3B2JmZo0lVwIxMzOrOIGkp7C6jaSTJS2W1CqpqUy9oyUtlfSopHMKyveS9CdJyyRdJWlAd8ZnZmbldZhAJL1L0sPAkrT/FkkXdMO1FwHvB+4uVUFSX+AnwDHAAcBpkg5Ih78PnB8RE4C1wBndEJOZmVWokjuQ84GjgOcBIuIhYHJXLxwRSyJiaQfVDgYejYjHI2IzcCVwgiQBh/PacruzyJ4OMzOzHlJRF1ZEPN2uqKUKsRQzBii89opUthPwQkRsbVe+DUkzJDVLal69enVVgzUz600qeYz3aUnvAiKNM3yO1J3VEUm3A7sWOfT1tOJhh00UKYsy5dsWRswEZgI0NTX5TXozs25SSQL5JPAfZH/hrwBuBc6spPGImJI/NEjX271gfyywEngO2EFSv3QX0lZuZmY9pMMEEhHPAf/YA7EU8yAwQdJewF+B6cAHIyIk3Um2sNWVwOlAJXc0ZiXd99hz/HbBKt4ydjinNO1ONtRmZqV0mEAk/WeR4heB5gq7oUq1O41sbfVRwI2S5kfEUZJ2Ay6OiGMjYqukzwC3kC2je2lELE5NfAW4UtJ3gT8Dl+SNxWzBihf4p8seZOOWVmbP68vLG7fysfeMr3VYZnWtki6sQcD+wNVp/wPAYuAMSYdFxNl5LhwRs4HZRcpXAscW7N8E3FSk3uNkT2mZddncJ9cSaYTslS0tzFm62gnErAOVJJB9gMPbnniS9FOycZCpwMIqxmbWY94+bgRtPVaD+/dlyt/tXNuAzBpAJQlkDDCErNuKtL1bRLRI2lS1yMx60JvGDOeXZ7yDmxf9jb8fO5zj37JbrUMyq3uVJJAfAPMlzSF7fHYy8C9papPbqxibWY9qGjeCpnEjah2GWcOo5CmsSyTdRDbeIOBraZwC4EvVDM7MzOpXpZMpbgRWAWuAfSR1eSoTMzNrbJU8xvsx4Cyyl/XmA+8E/kg2F5WZmfVSldyBnAW8HXgyIg4D3gp4Uikzs16ukgSyMSI2AkgaGBF/AfarblhmZlbvKnkKa4WkHYBfA7dJWovnnTIz6/UqeQprWto8N80/NRz4XVWjMjOzulc2gUjqAyyIiDcBRMRdPRKVmZnVvbJjIBHRCjwkaY8eisfMzBpEJWMgo4HFkh4A1rcVRsTxVYvKzMzqXiUJ5FtVj8LMzBpOJYPod0naE5gQEbdL2o5sbQ4zM+vFOnwPRNLHgWuAC1PRGLJHes3MrBer5EXCM4FJwEsAEbEM8GIJZma9XCUJZFNEbG7bkdQPiOqFZGZmjaCSBHKXpK8BgyVNJVva9jfVDcvMzOpdJQnkHLLJExcCnyBbn/wbXbmopJMlLZbUKqmpRJ3dJd0paUmqe1bBsXMl/VXS/PQ5tlgbZmZWPZU8xnsC8POIuKgbr7sIeD+vDcwXsxX4vxExT9JQYK6k2yLi4XT8/Ij4t26MyczMOqGSO5DjgUck/ULSP6QxkC6JiCURsbSDOqsiYl7afhlYQvYEmJmZ1YEOE0hEfBTYh2zs44PAY5IurnZghSSNI1uH5E8FxZ+RtEDSpZJ2LHPuDEnNkppXr/YyJmZm3aWiJW0jYgtwM3AlMJesW6ssSbdLWlTk0+G57drZHrgWODsiXkrFPwX2BiaSLbX7wzKxz4yIpohoGjVqVGcubWZmZVSypO3RwHTgMGAOcDFwSkfnRcSUrgYnqT9Z8rg8Iq4raPuZgjoXAb/t6rXMeqNXNrewbtNWRg0dWOtQrAFVMp7xEbI7j09ExKbqhvMaSQIuAZZExL+3OzY6Ilal3Wlkg/Jm1gmPPruOE3/yBzZtbWHG5PF86aj9ax2SNZhKxkCmR8Sv25KHpEmSftKVi0qaJmkFcAhwo6RbUvlukm5K1SYBHwYOL/K47g8kLZS0gOzO6PNdicesN7pp4So2bN7Klpbg5/c9WetwrAFV9ESVpIlkA+inAE8A15U/o7yImA3MLlK+Ejg2bd8LqMT5H+7K9c0M3jl+Jwb060MfiUkTRtY6HGtAJROIpH3Jxj5OA54HrgIUEYf1UGxmVkUH7zWCGz/3Hla9sJF3jh9R63CsAZW7A/kLcA9wXEQ8CiDJXUVmbyB7j9qevUdtX+swrEGVGwP5APA34E5JF0k6ghJdSmZm1vuUTCARMTsiTgX2J3t89/PALpJ+KunIHorPzMzqVCVPYa2PiMsj4n3AWGA+2QSLZmbWi1X0JnqbiFgTERdGxOHVCsjMzBpDpxKImZlZGycQMzPLxQnEzMxycQIxM7NcnEDMzCwXJxAzM8vFCcTMzHJxAjEzs1ycQMzMLBcnEDMzy8UJxMzMcnECMTOzXGqSQCSdLGmxpFZJTWXqLU9rn8+X1FxQPkLSbZKWpe8deyZyMzNrU6s7kEXA+4G7K6h7WERMjIjCRHMOcEdETADuwNPLm5n1uJokkIhYEhFLu9DECcCstD0LOLHrUZmZWWfU+xhIALdKmitpRkH5LhGxCiB971yqAUkzJDVLal69enWVwzUz6z36VathSbcDuxY59PWIuL7CZiZFxEpJOwO3SfpLRFTS7fWqiJgJzARoamqKzpxrZmalVS2BRMSUbmhjZfp+VtJs4GCycZNnJI2OiFWSRgPPdvVaZmbWOXXbhSVpiKShbdvAkWSD7wA3AKen7dOBSu9ozMysm9TqMd5pklYAhwA3Srolle8m6aZUbRfgXkkPAQ8AN0bE79Kx84CpkpYBU9O+mZn1IEX0nmGBpqamaG5u7riimZm9StLcdq9SAHXchWVmZvXNCcTMzHJxAjEzs1ycQMzMLBcnEDMzy8UJxMzMcnECMTOzXJxAzMwsFycQMzPLxQnEzMxycQIxM7NcnEDMzCwXJxAzM8vFCcTMzHJxAjEzs1ycQMzMLBcnEDMzy8UJxMzMcnECMTOzXGqSQCSdLGmxpFZJ26yzm+rsJ2l+weclSWenY+dK+mvBsWN79icwM7N+NbruIuD9wIWlKkTEUmAigKS+wF+B2QVVzo+If6tmkGZmVlpNEkhELAGQVOkpRwCPRcSTVQvKzMw6pVHGQKYDV7Qr+4ykBZIulbRjqRMlzZDULKl59erV1Y3SzKwXqVoCkXS7pEVFPid0sp0BwPHA1QXFPwX2JuviWgX8sNT5ETEzIpoiomnUqFE5fhIzs8a1dv1mHnhiDZu2tnR721XrwoqIKd3U1DHAvIh4pqDtV7clXQT8tpuuZWb2hvHihi0c/sM5bNrayoG7DePqT76rW9tvhC6s02jXfSVpdMHuNLJBeTMzK7D8+fVs3NLKhs0t/PmpF7q9/Vo9xjtN0grgEOBGSbek8t0k3VRQbztgKnBduyZ+IGmhpAXAYcDneyh0M7OGceBuwzhk/E4M6t+Hs6fs2+3tKyK6vdF61dTUFM3NzbUOw8ysoUiaGxHbvLPXCF1YZmZWh5xAzMwsFycQMzPLxQnEzMxycQIxM7NcnEDMzCwXJxAzM8vFCcTMzHLpVS8SSloN1HJK+JHAczW8fh6Ouec0YtyNGDM0Zty1jHnPiNhmNtpelUBqTVJzsbc565lj7jmNGHcjxgyNGXc9xuwuLDMzy8UJxMzMcnEC6Vkzax1ADo655zRi3I0YMzRm3HUXs8dAzMwsF9+BmJlZLk4gZmaWixNID5B0qaRnJTXM0ruSdpd0p6QlkhZLOqvWMXVE0iBJD0h6KMX8rVrHVClJfSX9WdJvax1LpSQtTyuDzpfUECu1SdpB0jWS/pL+3z6k1jF1RNJ+6d+47fOSpLNrHRd4DKRHSJoMrAN+HhFvqnU8lUjrzo+OiHmShgJzgRMj4uEah1aSJAFDImKdpP7AvcBZEXF/jUPrkKQvAE3AsIh4X63jqYSk5UBTRDTMC3mSZgH3RMTFkgYA20VE9y8WXiWS+gJ/Bd4REbV8KRrwHUiPiIi7gTW1jqMzImJVRMxL2y8DS4AxtY2qvMisS7v906fu/0KSNBb4B+DiWsfyRiZpGDAZuAQgIjY3UvJIjgAeq4fkAU4gVgFJ44C3An+qbSQdS11B84Fngdsiou5jBn4EfBlorXUgnRTArZLmSppR62AqMB5YDfwsdRdeLGlIrYPqpOnAFbUOoo0TiJUlaXvgWuDsiHip1vF0JCJaImIiMBY4WFJddxlKeh/wbETMrXUsOUyKiIOAY4AzU1dtPesHHAT8NCLeCqwHzqltSJVLXW7HA1fXOpY2TiBWUhpHuBa4PCKuq3U8nZG6JuYAR9c4lI5MAo5P4wlXAodL+mVtQ6pMRKxM388Cs4GDaxtRh1YAKwruSq8hSyiN4hhgXkQ8U+tA2jiBWFFpQPoSYElE/Hut46mEpFGSdkjbg4EpwF9qG1V5EfHViBgbEePIuid+HxEfqnFYHZI0JD1cQeoGOhKo66cMI+JvwNOS9ktFRwB1+1BIEadRR91XkN3SWZVJugI4FBgpaQXwzxFxSW2j6tAk4MPAwjSmAPC1iLiphjF1ZDQwKz2p0gf4n4homMdiG8wuwOzs7wz6Ab+KiN/VNqSKfBa4PHUHPQ58tMbxVETSdsBU4BO1jqWQH+M1M7Nc3IVlZma5OIGYmVkuTiBmZpaLE4iZmeXiBGJmZrk4gViPkjRNUkjav9axdETSdpIuTzPOLpJ0b3ozH0n3pe9Du3sG3dTmiwWzr96es52Jko7tztgK2v5Iejy9sGykpNWSBpY57zJJJ1UjJut5TiDW004jmyV3enc0lt75qJazgGci4s1pFuUzgC0AEfGuKl4XshljJ6bPlJxtTAQ6lUCUqeT3wnXA1PR+QpuTgBsiYlNnrmmNywnEekz6630S2S/i6QXlVxX+pZz+Sv1Amhjx/0t6UNICSZ9Ixw9Na5X8CliYyn6dJvVbXDixn6QzJD0iaY6kiyT9OJWPknRtavtBSZOKhDyabOpsACJiadsvR0nrCuoNkzRb0sOS/ltSnxT7ZenOZaGkz6fz5kj6kaT70rGKp/8oFbOkg1N7f07f+6UX5b4NnJruYk6VdK6kLxa0t0jSuPRZIukCYB6wu6QjJf1R0jxJV7fdeRX8W7wE3A0cV1D86kR/kr6ZYlwkaWaa2aD9z7Nc0si03SRpTtoeomwNnQfTz3RCpf9G1sMiwh9/euQDfAi4JG3fBxyUtqcBs9L2AOBpYDAwA/hGKh8INAN7kb3Vvx7Yq6DtEel7MNmUGjsBuwHLgRFkU7vfA/w41fsV8O60vQfZlC3t451INqvvH4HvAhMKjq1L34cCG8lmeu0L3Eb2l/jbyGYDbqu/Q/qeA1yUticDi4pc91DgRWB++ny9XMzAMKBf2p4CXJu2P9L286b9c4EvFuwvAsalTyvwzlQ+kiw5DEn7XwG+WSTOk4HZaXs3YCXQt/C/R9r+BXBc2r4MOCltLwdGpu0mYE7a/hfgQ23/bsAjbbH4U18fT2ViPek0sqnLIZs48DSyv3hvBv4z9Z0fDdwdEa9IOhL4+4I+8+HABGAz8EBEPFHQ9uckTUvbu6d6uwJ3RcQaAElXA/umOlOAAwr+MB4maWhka58AEBHzJY0nm+dpCvCgpEMiYkm7n+uBiHg8XeMK4N3AHcB4Sf8F3AjcWlD/itT+3ZKGSdohtl2X4p7YdmGpojGnf5dZkiaQTbHen857Ml5beOudwAHAH9K1BpAl0fZ+C1ygbJ2NU4BrIqIlHTtM0peB7cgS+GLgNxXGciTZBJNtd0uDSAmzcz+SVZsTiPUISTsBhwNvkhRkf62HpC9HxMbUfXEUcCqvTRgn4LMRcUu7tg4luwMp3J8CHBIRG1Jbg9L5pfRJ9V8pF3dkC1RdB1wnqZVsTKH9L7L28wFFRKyV9Jb0M51J9gv2n0rVLxdDRzGnJHVnRExTtnbLnBLnb+X13daDCrbXF2yL7O7ptHLBpCT/O7I7yOlAWzfdIOACstUKn5Z0brtrFYun8LiAD0TE0nLXt9rzGIj1lJPIlvTdMyLGRcTuwBNkf61DdkfyUeA9QFvCuAX4lLJp5ZG0r4ovADQcWJuSx/5kf0EDPAC8V9KOkvoBHyg451bgM207kia2b1TSJEk7pu0BZH+VF1sJ7mBJeykbfD4VuDf17feJiGuB/8frpw0/NbX5buDFiHixSJvFlIp5OK+N1XykoP7LwNCC/eVtcUg6iKw7sJj7gUmS9kl1t5O0b4m6VwBfIJtcse0Opi0ZPJfGTko9dbWcrKsPXv/f5hbgs23jJpLeWuJ8qzEnEOspp5GtGY8L0MYAAAEqSURBVFHoWuCDaftWsjGB2yNicyq7mGy67XmSFgEXUvyu+XdAP0kLgO+QfpFFxF/J+tP/BNye2mr7Zf05oEnZ4PzDwCeLtLs3cJekhcCfycZgri1S74/AeWRjCk+kn3MMMEfZTMaXAV8tqL9W2WPA/032QEGlSsX8A+BfJf2B7M6uzZ1kXV7zJZ2aYh+RYvoU2djCNiJiNVkiuiL9m94PlHrs+lay8Y+rIrJBi9QddxHZAw6/Bh4sce63gP+QdA/QUlD+HbJuuAXpv/t3SpxvNebZeO0NTdL2EbEu3YHMBi6NiPaJrCfjmUM2kN1cqxjMuovvQOyN7tz0F3fb3cGvaxyP2RuG70DMzCwX34GYmVkuTiBmZpaLE4iZmeXiBGJmZrk4gZiZWS7/C3EWt6NljJPsAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "bin_means, bin_edges, _ = stats.binned_statistic(test_features[:,1], attr[:,1], statistic='mean', bins=6)\n", + "bin_count, _, _ = stats.binned_statistic(test_features[:,1], attr[:,1], statistic='count', bins=6)\n", + "\n", + "bin_width = (bin_edges[1] - bin_edges[0])\n", + "bin_centers = bin_edges[1:] - bin_width/2\n", + "plt.scatter(bin_centers, bin_means, s=bin_count)\n", + "plt.xlabel(\"Average Sibsp Feature Value\");\n", + "plt.ylabel(\"Average Attribution\");\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We see that the larger magnitude attributions correspond to the examples with larger Sibsp feature values, suggesting that the feature has a larger impact on prediction for these examples. Since there are substantially fewer of these examples (compared to those with a feature value of 0), the average attribution does not completely capture this effect." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that we have a better understanding of the importance of different input features, the next question we can ask regarding the function of the neural network is how the different neurons in each layer work together to reach the prediction. For instance, in our first hidden layer output containing 12 units, are all the units used for prediction? Do some units learn features positively correlated with survival while others learn features negatively correlated with survival?" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This leads us to the second type of attributions available in Captum, **Layer Attributions**. Layer attributions allow us to understand the importance of all the neurons in the output of a particular layer. For this example, we will be using Layer Conductance, one of the Layer Attribution methods in Captum, which is an extension of Integrated Gradients applied to hidden neurons. More information regarding conductance can be found in the original paper here: https://arxiv.org/abs/1805.12233. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To use Layer Conductance, we create a LayerConductance object passing in the model as well as the module (layer) whose output we would like to understand. In this case, we choose net.sigmoid1, the output of the first hidden layer." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "cond = LayerConductance(net, net.sigmoid1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can now obtain the conductance values for all the test examples by calling attribute on the LayerConductance object. LayerConductance also requires a target index for networks with mutliple outputs, defining the index of the output for which gradients are computed. Similar to feature attributions, we provide target = 1, corresponding to survival. LayerConductance also utilizes a baseline, but we simply use the default zero baseline as in integrated gradients." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "cond_vals = cond.attribute(test_input_tensor,target=1)\n", + "cond_vals = cond_vals.detach().numpy()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can begin by visualizing the average conductance for each neuron." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Neuron Importances\n", + "0 : -0.135\n", + "1 : -0.038\n", + "2 : 0.033\n", + "3 : 0.013\n", + "4 : -0.016\n", + "5 : -0.130\n", + "6 : 0.009\n", + "7 : 0.001\n", + "8 : -0.169\n", + "9 : -0.000\n", + "10 : -0.309\n", + "11 : 0.017\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtEAAAGDCAYAAADtZ0xmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3de7wdZX3v8c9XIqhU5RYgAgFUKqJHo0awtVIPF8FiDfV4gXo0tVKOrdbaaiteTrVWe+DYVm2rbRGEqHhFLVRbBaKo9QgSNAiIGK4SCRe5CF6Kor/zx3p2XWz2TvJk7WTtvf28X695rZl5npn5zV5J9jeznjWTqkKSJEnSxrvPuAuQJEmS5hpDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLmnWSnJvktiTbjLuWmdDO5z+T7DG07pAk14yxrI2SZK8klWTBuGsBaLU8fNx1SJIhWtKskmQv4ClAAc/cTMcYRyD8AfC/N/dBZkvYnWnz9bwkzV2GaEmzzQuB84BTgeUTK5M8KckNSbYaWvdbSb7e5u+T5LgkVya5JclHkuzQ2iaupr44ybeBz7b1H237/F6SLyR51NC+d0zyr0nuSHJBkjcn+Y+h9n2TnJ3k1iSXJ3nuBs7r74Cjp7uKmuQhST6W5OYkVyd5+VDbqUnePLT81CRrh5avSfLq9rP4QZIFSR7ZroDfnuTSJM+ctL93JvlUkjuTnJ/kYRuof3jbdyX59yTfT/KlJLsmeXv79OCbSR43qbbXJPlGaz8lyf2G2n8vyRXt53hmkocMtVWSlyZZA6xJ8oXWdFE79vOSbJ/kk+3ndlub331oH+cm+ctW551Jzkqy01D7ryX5f+3ndF2S32nrt0ny10m+neTGJP+U5P6tbad2nNtb3V9M4u9T6ReMf+klzTYvBE5r02FJdgGoqvMYXM09aKjvbwMfaPMvB44Efh14CHAb8M5J+/514JHAYW3534F9gJ2Br7ZjTnhnO96uDML8cKDfFji7HXtn4GjgXcMhfArfAd4NvHFyQwtg/wpcBOwGHAy8Islhk/uux9HAEcB2QNr+zmr1/SFwWpJHTOr/F8D2wBXAWzqO9Vzg9cBOwF3Alxn8/HYCTgf+dlL/5zP4mT8M+OW2LUkOAv5P298i4FrgQ5O2PRI4ANivqg5s6x5bVb9UVR9m8HvsFGBPYDHwI+AfJu3jt4EXMfhZbA28qh1/MYM/A38PLASWAKvbNie0WpcAD2fwvvx5a3slsLZtswvwWgafnEj6RVJVTk5OTrNiAn4N+AmwU1v+JvDHQ+1vBt7T5h/IIOTu2ZYvAw4e6ruo7WsBsBeDkPPQ9Rx7u9bnwcBWbdtHTDr2f7T55wFfnLT9PwNvmGbf5wLHMAhd3wMeBRwCXNPaDwC+PWmb1wCntPlTgTcPtT0VWDu0fA3wu0PLTwFuAO4ztO6DwBuH9nfSUNtvAN+cpvaJn92CoW3fPdT+h8BlQ8v/Dbh9Um0vmXSsK9v8ycD/HWr7pfZz36stF3DQpHoKePh63sclwG2TfvavH1r+A+DTQz/jT0yxj7Q/Ww8bWvcrwNVt/k3AGeurw8nJaf5PXomWNJssB86qqu+25Q8wdAW4LT8rgy8cPgv4alVd29r2BD7RPmK/nUGo/imDK4UTrpuYSbJVkuPb8I87GIQ9GFxNXcggfF831bbtWAdMHKsd7/kMrlpPq6puZnCV9E2TmvYEHjJpf6+dVPuGDNf3EOC6qvrZ0LprGVxNnXDD0PwPGQTYjXXj0PyPplievK/h2q5t9U3UOfH+UVXfB26ZVOfwtveS5AFJ/jnJte19/AKwXYaG/TD9ue4BXDnFbhcCDwAuHHo/Pt3WA7yVwdX7s5JcleS49dUoaX7yixqSZoU23vS5wFZJJkLPNgwC0WOr6qKq+kaSa4Gnc8+hHDAIW79bVV+aYt97tdnhj9x/G1hGuyLM4Ar0bQyuQt4M3A3sDnyr9d9jaNvrgM9X1aGbcKpvBa4CvjJpf1dX1T7TbPMDBqFuwlRhffjcrgf2SHKfoSC9mJ+fy5Y2/LNbzKA+2uueEw1tmMyODIa+TNjQMIlXAo8ADqiqG5IsAb7G4H3ckOuA/adY/10G/xl4VFV9Z3JjVd3ZjvvKNoTnc0kuqKqVG3FMSfOEV6IlzRZHMrhyvB+Dj+SXMBi//EUG46QnfIDB+OcDgY8Orf8n4C1J9gRIsjDJsvUc74EMxvPewiCg/tVEQ1X9FPg48MZ2pXPfSTV8EvjlJC9Ict82PTHJIzd0klV1O/A3wJ8Nrf4KcEf7cuD921XyRyd5YmtfDfxGkh2S7Aq8YgOHOZ9B8P6zVttTgd/k3uONt5SXJtk9gy96vhb4cFv/AeBFSZa0Txf+Cji/qq5Zz75uBB46tPxABoH39rb/N3TUdRpwSJLnZvBlzB2TLGn/8Xg38LYkOwMk2W1ijHqSZyR5eJIAdzD4c/vTjuNKmgcM0ZJmi+UMxgB/u6pumJgYDH94fn5+i7MPMhgT/NmhYR8A7wDOZPAR+50M7vBxwHqO914GQwm+A3yj9R/2MgZXp28A3teOexf815XIpwFHMbiaegODL6Jt7H2t38FQ6Gqh/TcZ/MfhagZXQk9qx6cd/yIGV8zP4uchdEpV9WMGtwd8etvXu4AXVtU3N7K+mfYBBnVf1aY3tzpXMrjt38eAdQy+eHjUBvb1RmBFG2bxXODtwP0ZnOd5DIZdbJSq+jaDMdqvBG5l8J+Vx7bmVzMYsnFeGyZyDoMr3jD4Muo5wPcZfKnyXVV17sYeV9L8kCq/UCxJG5LkBGDXqlq+wc76Lxk8UOaYqjpn3LVI0kzySrQkTSGD+0A/JgP7Ay8GPjHuuiRJs8OMhOgkh2fwsIErpvqWcrtp/Ydb+/kTX/LJ4AEIP0qyuk3/NBP1SNIMeCCDcdE/AD7CYBzzGWOtSJI0a4w8nKPdRuhbwKEMbj5/AXB0VX1jqM8fAI+pqpckOQr4rap6XgvTn6yqR49UhCRJkrQFzcSV6P2BK6rqqvZllg8xuG3UsGXAijZ/OnBw+1azJEmSNOfMRIjejXveDH8t97xR/j36VNXdDJ7YtWNr2zvJ15J8PslTZqAeSZIkabOaiYetTHVFefIYken6rAMWV9UtSZ4A/EuSR1XVHfc6SHIscCzAtttu+4R99913xLIlSZKk6V144YXfraqFU7XNRIheyz2fRrU7P38a1eQ+a9u9Xh8M3FqDAdkT9129MMmVwC8DqyYfpKpOBE4EWLp0aa1ada8ukiRJ0oxpT8md0kwM57gA2CfJ3km2ZnCj/DMn9TmTwYMUAJ7N4CEJ1Z4otlUr8qEMbmB/1QzUJEmSJG02I1+Jrqq7k7wM+AywFfCeqro0yZuAVVV1JnAy8L4kVzB4KtTEE6kOBN6U5G4GT+96SVXdOmpNkiRJ0uY0J59Y6HAOSZIkbW5JLqyqpVO1+cRCSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqdPIj/2WZoO9jvvUuEvYKNccf8S4S5AkSTPAK9GSJElSJ0O0JEmS1MkQLUmSJHUyREuSJEmdDNGSJElSJ0O0JEmS1MkQLUmSJHUyREuSJEmdDNGSJElSJ0O0JEmS1MkQLUmSJHUyREuSJEmdDNGSJElSpwXjLkCSJEnjs9dxnxp3CRt0zfFHjLuEe/FKtCRJktTJEC1JkiR1MkRLkiRJnRwTLc1Cjk+TJGl280q0JEmS1MkQLUmSJHUyREuSJEmdHBMtSb/AHH8vSZvGK9GSJElSJ0O0JEmS1MkQLUmSJHWakRCd5PAklye5IslxU7Rvk+TDrf38JHsNtb2mrb88yWEzUY8kSZK0OY0copNsBbwTeDqwH3B0kv0mdXsxcFtVPRx4G3BC23Y/4CjgUcDhwLva/iRJkqRZaybuzrE/cEVVXQWQ5EPAMuAbQ32WAW9s86cD/5Akbf2Hquou4OokV7T9fXkG6pIk/QKZC3caAe82Is0XMxGidwOuG1peCxwwXZ+qujvJ94Ad2/rzJm2721QHSXIscCzA4sWLZ6DsfvPtH+i5cD4bey7z7ZfSfDuf+fRnbT6dS2/f2W4+nQvMvz9r88l8e29+Ud/HUc3EmOhMsa42ss/GbDtYWXViVS2tqqULFy7sLFGSJEmaOTMRotcCewwt7w5cP12fJAuABwO3buS2kiRJ0qwyEyH6AmCfJHsn2ZrBFwXPnNTnTGB5m3828Nmqqrb+qHb3jr2BfYCvzEBNkiRJ0mYz8pjoNsb5ZcBngK2A91TVpUneBKyqqjOBk4H3tS8O3sogaNP6fYTBlxDvBl5aVT8dtSZJkiRpc5qJLxZSVf8G/NukdX8+NP+fwHOm2fYtwFtmog5JkiRpS/CJhZIkSVInQ7QkSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVInQ7QkSZLUyRAtSZIkdTJES5IkSZ1m5GErkrQ+1xx/xLhLkCRpRnklWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6LRh3ARqfa44/YtwlSJIkzUleiZYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkTiOF6CQ7JDk7yZr2uv00/Za3PmuSLB9af26Sy5OsbtPOo9QjSZIkbQmjXok+DlhZVfsAK9vyPSTZAXgDcACwP/CGSWH7+VW1pE03jViPJEmStNmNGqKXASva/ArgyCn6HAacXVW3VtVtwNnA4SMeV5IkSRqbUUP0LlW1DqC9TjUcYzfguqHltW3dhFPaUI7/nSQj1iNJkiRtdht87HeSc4Bdp2h63UYeY6pgXO31+VX1nSQPBD4GvAB47zR1HAscC7B48eKNPLQkSZI08zYYoqvqkOnaktyYZFFVrUuyCJhqTPNa4KlDy7sD57Z9f6e93pnkAwzGTE8ZoqvqROBEgKVLl9ZUfSRJkqQtYdThHGcCE3fbWA6cMUWfzwBPS7J9+0Lh04DPJFmQZCeAJPcFngFcMmI9kiRJ0mY3aog+Hjg0yRrg0LZMkqVJTgKoqluBvwQuaNOb2rptGITprwOrge8A7x6xHkmSJGmz2+BwjvWpqluAg6dYvwo4Zmj5PcB7JvX5AfCEUY4vSZIkjYNPLJQkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkTiOF6CQ7JDk7yZr2uv00/T6d5PYkn5y0fu8k57ftP5xk61HqkSRJkraEUa9EHwesrKp9gJVteSpvBV4wxfoTgLe17W8DXjxiPZIkSdJmN2qIXgasaPMrgCOn6lRVK4E7h9clCXAQcPqGtpckSZJmk1FD9C5VtQ6gve7cse2OwO1VdXdbXgvsNmI9kiRJ0ma3YEMdkpwD7DpF0+tGPHamWFfrqeNY4FiAxYsXj3hoSZIkadNtMERX1SHTtSW5McmiqlqXZBFwU8exvwtsl2RBuxq9O3D9euo4ETgRYOnSpdOGbUmSJGlzG3U4x5nA8ja/HDhjYzesqgI+Bzx7U7aXJEmSxmXUEH08cGiSNcChbZkkS5OcNNEpyReBjwIHJ1mb5LDW9GrgT5JcwWCM9Mkj1iNJkiRtdhsczrE+VXULcPAU61cBxwwtP2Wa7a8C9h+lBkmSJGlL84mFkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUqcF4y5AkuaSa44/YtwlSJJmAa9ES5IkSZ0M0ZIkSVInQ7QkSZLUyTHRHRwLKUmSJPBKtCRJktTNEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktRppBCdZIckZydZ0163n6bfp5PcnuSTk9afmuTqJKvbtGSUeiRJkqQtYdQr0ccBK6tqH2BlW57KW4EXTNP2p1W1pE2rR6xHkiRJ2uxGDdHLgBVtfgVw5FSdqmolcOeIx5IkSZJmhVFD9C5VtQ6gve68Cft4S5KvJ3lbkm2m65Tk2CSrkqy6+eabN7VeSZIkaWQbDNFJzklyyRTTshk4/muAfYEnAjsAr56uY1WdWFVLq2rpwoULZ+DQkiRJ0qZZsKEOVXXIdG1JbkyyqKrWJVkE3NRz8Imr2MBdSU4BXtWzvSRJkjQOow7nOBNY3uaXA2f0bNyCN0nCYDz1JSPWI0mSJG12o4bo44FDk6wBDm3LJFma5KSJTkm+CHwUODjJ2iSHtabTklwMXAzsBLx5xHokSZKkzW6DwznWp6puAQ6eYv0q4Jih5adMs/1BoxxfkiRJGgefWChJkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUqcF4y5AkiTd2zXHHzHuEiSth1eiJUmSpE6GaEmSJKmTIVqSJEnqZIiWJEmSOhmiJUmSpE6GaEmSJKmTIVqSJEnqZIiWJEmSOhmiJUmSpE6GaEmSJKmTIVqSJEnqZIiWJEmSOhmiJUmSpE6GaEmSJKmTIVqSJEnqZIiWJEmSOhmiJUmSpE6GaEmSJKmTIVqSJEnqZIiWJEmSOhmiJUmSpE6GaEmSJKnTSCE6yQ5Jzk6ypr1uP0WfJUm+nOTSJF9P8ryhtr2TnN+2/3CSrUepR5IkSdoSRr0SfRywsqr2AVa25cl+CLywqh4FHA68Pcl2re0E4G1t+9uAF49YjyRJkrTZjRqilwEr2vwK4MjJHarqW1W1ps1fD9wELEwS4CDg9PVtL0mSJM02o4boXapqHUB73Xl9nZPsD2wNXAnsCNxeVXe35rXAbiPWI0mSJG12CzbUIck5wK5TNL2u50BJFgHvA5ZX1c/alejJaj3bHwscC7B48eKeQ0uSJEkzaoMhuqoOma4tyY1JFlXVuhaSb5qm34OATwGvr6rz2urvAtslWdCuRu8OXL+eOk4ETgRYunTptGFbkiRJ2txGHc5xJrC8zS8Hzpjcod1x4xPAe6vqoxPrq6qAzwHPXt/2kiRJ0mwzaog+Hjg0yRrg0LZMkqVJTmp9ngscCPxOktVtWtLaXg38SZIrGIyRPnnEeiRJkqTNboPDOdanqm4BDp5i/SrgmDb/fuD902x/FbD/KDVIkiRJW5pPLJQkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqdNIITrJDknOTrKmvW4/RZ8lSb6c5NIkX0/yvKG2U5NcnWR1m5aMUo8kSZK0JYx6Jfo4YGVV7QOsbMuT/RB4YVU9CjgceHuS7Yba/7SqlrRp9Yj1SJIkSZvdqCF6GbCiza8Ajpzcoaq+VVVr2vz1wE3AwhGPK0mSJI3NqCF6l6paB9Bed15f5yT7A1sDVw6tfksb5vG2JNusZ9tjk6xKsurmm28esWxJkiRp020wRCc5J8klU0zLeg6UZBHwPuBFVfWztvo1wL7AE4EdgFdPt31VnVhVS6tq6cKFXsiWJEnS+CzYUIeqOmS6tiQ3JllUVetaSL5pmn4PAj4FvL6qzhva97o2e1eSU4BXdVUvSZIkjcGowznOBJa3+eXAGZM7JNka+ATw3qr66KS2Re01DMZTXzJiPZIkSdJmN2qIPh44NMka4NC2TJKlSU5qfZ4LHAj8zhS3sjstycXAxcBOwJtHrEeSJEna7DY4nGN9quoW4OAp1q8Cjmnz7wfeP832B41yfEmSJGkcfGKhJEmS1MkQLUmSJHUyREuSJEmdDNGSJElSJ0O0JEmS1MkQLUmSJHUyREuSJEmdDNGSJElSJ0O0JEmS1MkQLUmSJHUyREuSJEmdDNGSJElSJ0O0JEmS1MkQLUmSJHUyREuSJEmdDNGSJElSJ0O0JEmS1MkQLUmSJHUyREuSJEmdDNGSJElSJ0O0JEmS1MkQLUmSJHUyREuSJEmdDNGSJElSJ0O0JEmS1MkQLUmSJHUyREuSJEmdDNGSJElSJ0O0JEmS1MkQLUmSJHUyREuSJEmdDNGSJElSJ0O0JEmS1MkQLUmSJHUaOUQn2SHJ2UnWtNftp+izZ5ILk6xOcmmSlwy1PSHJxUmuSPJ3STJqTZIkSdLmNBNXoo8DVlbVPsDKtjzZOuBXq2oJcABwXJKHtLZ/BI4F9mnT4TNQkyRJkrTZzESIXgasaPMrgCMnd6iqH1fVXW1xm4njJlkEPKiqvlxVBbx3qu0lSZKk2WQmQvQuVbUOoL3uPFWnJHsk+TpwHXBCVV0P7AasHeq2tq2TJEmSZq0FG9MpyTnArlM0vW5jD1RV1wGPacM4/iXJ6cBU459rmhqOZTDsg8WLF2/sYSVJkqQZt1EhuqoOma4tyY1JFlXVujY846YN7Ov6JJcCTwG+BOw+1Lw7cP00250InAiwdOnSKYO2JEmStCXMxHCOM4HlbX45cMbkDkl2T3L/Nr898GTg8jb8484kT2p35XjhVNtLkiRJs8lMhOjjgUOTrAEObcskWZrkpNbnkcD5SS4CPg/8dVVd3Np+HzgJuAK4Evj3GahJkiRJ2mw2ajjH+lTVLcDBU6xfBRzT5s8GHjPN9quAR49ahyRJkrSl+MRCSZIkqZMhWpIkSeo08nAOSZKkXyTXHH/EuEvQLOCVaEmSJKmTIVqSJEnqZIiWJEmSOhmiJUmSpE6GaEmSJKmTIVqSJEnqZIiWJEmSOhmiJUmSpE6GaEmSJKmTIVqSJEnqZIiWJEmSOhmiJUmSpE6GaEmSJKmTIVqSJEnqlKoadw3dktwMXDvuOmbITsB3x13EDJlP5wKez2w2n84F5tf5zKdzAc9nNptP5wLz63zm07nsWVULp2qYkyF6PkmyqqqWjruOmTCfzgU8n9lsPp0LzK/zmU/nAp7PbDafzgXm1/nMp3NZH4dzSJIkSZ0M0ZIkSVInQ/T4nTjuAmbQfDoX8Hxms/l0LjC/zmc+nQt4PrPZfDoXmF/nM5/OZVqOiZYkSZI6eSVakiRJ6mSIHpMkhye5PMkVSY4bdz2jSPKeJDcluWTctcyEJHsk+VySy5JcmuSPxl3TpkpyvyRfSXJRO5e/GHdNMyHJVkm+luST465lFEmuSXJxktVJVo27nlEl2S7J6Um+2f7+/Mq4a9pUSR7R3peJ6Y4krxh3XZsqyR+3fwMuSfLBJPcbd02jSPJH7VwunYvvy1S/N5PskOTsJGva6/bjrHFjTXMuz2nvzc+SzNu7dBiixyDJVsA7gacD+wFHJ9lvvFWN5FTg8HEXMYPuBl5ZVY8EngS8dA6/P3cBB1XVY4ElwOFJnjTmmmbCHwGXjbuIGfLfq2rJPLkd1DuAT1fVvsBjmcPvUVVd3t6XJcATgB8CnxhzWZskyW7Ay4GlVfVoYCvgqPFWtemSPBr4PWB/Bn/OnpFkn/FW1e1U7v178zhgZVXtA6xsy3PBqdz7XC4BngV8YYtXswUZosdjf+CKqrqqqn4MfAhYNuaaNllVfQG4ddx1zJSqWldVX23zdzIIAruNt6pNUwPfb4v3bdOc/iJEkt2BI4CTxl2Lfi7Jg4ADgZMBqurHVXX7eKuaMQcDV1bVXH7I1wLg/kkWAA8Arh9zPaN4JHBeVf2wqu4GPg/81phr6jLN781lwIo2vwI4cosWtYmmOpequqyqLh9TSVuMIXo8dgOuG1peyxwNafNdkr2AxwHnj7eSTdeGPqwGbgLOrqo5ey7N24E/A3427kJmQAFnJbkwybHjLmZEDwVuBk5pQ21OSrLtuIuaIUcBHxx3EZuqqr4D/DXwbWAd8L2qOmu8VY3kEuDAJDsmeQDwG8AeY65pJuxSVetgcDEH2HnM9WgDDNHjkSnWzemrg/NRkl8CPga8oqruGHc9m6qqfto+kt4d2L99FDonJXkGcFNVXTjuWmbIk6vq8QyGdr00yYHjLmgEC4DHA/9YVY8DfsDc+Th6Wkm2Bp4JfHTctWyqNrZ2GbA38BBg2yT/c7xVbbqqugw4ATgb+DRwEYNheNIWZYgej7Xc83/NuzO3P1qbd5Lcl0GAPq2qPj7uemZC+2j9XOb2+PUnA89Mcg2DYVAHJXn/eEvadFV1fXu9icF42/3HW9FI1gJrhz7pOJ1BqJ7rng58tapuHHchIzgEuLqqbq6qnwAfB351zDWNpKpOrqrHV9WBDIYSrBl3TTPgxiSLANrrTWOuRxtgiB6PC4B9kuzdrnIcBZw55prUJAmDcZ2XVdXfjrueUSRZmGS7Nn9/Br9MvzneqjZdVb2mqnavqr0Y/L35bFXNyStqSbZN8sCJeeBpDD6mnpOq6gbguiSPaKsOBr4xxswvCTQAAAM3SURBVJJmytHM4aEczbeBJyV5QPv37WDm8Jc+AZLs3F4XM/gC21x/j2CQA5a3+eXAGWOsRRthwbgL+EVUVXcneRnwGQbfkn5PVV065rI2WZIPAk8FdkqyFnhDVZ083qpG8mTgBcDFbSwxwGur6t/GWNOmWgSsaHeEuQ/wkaqa07eFm0d2AT4xyDQsAD5QVZ8eb0kj+0PgtHZx4CrgRWOuZyRtvO2hwP8ady2jqKrzk5wOfJXBsIevMfefKPexJDsCPwFeWlW3jbugHlP93gSOBz6S5MUM/uPznPFVuPGmOZdbgb8HFgKfSrK6qg4bX5Wbh08slCRJkjo5nEOSJEnqZIiWJEmSOhmiJUmSpE6GaEmSJKmTIVqSJEnqZIiWpFkqSSX5m6HlVyV54xhLkiQ1hmhJmr3uAp6VZKeZ3GkG/PdfkkbgP6KSNHvdzeChGH88uaE9jfJjSS5o05Pb+jcmedVQv0uS7NWmy5K8i8FDN/ZIcnSSi1ufE4a2+X6StyS5KMl5SXZp65/T+l6U5Aub++QlaTYzREvS7PZO4PlJHjxp/TuAt1XVE4H/AZy0Eft6BPDeqnocgye9nQAcBCwBnpjkyNZvW+C8qnos8AXg99r6PwcOa+ufOcI5SdKc52O/JWkWq6o7krwXeDnwo6GmQ4D92mPDAR6U5IEb2N21VXVem38icG5V3QyQ5DTgQOBfgB8DE4+Hv5DBo68BvgScmuQjwMc3/awkae4zREvS7Pd2BkMwThladx/gV6pqOFiT5G7u+Snj/YbmfzDcdT3H+0lVVZv/Ke13RVW9JMkBwBHA6iRLquqWrjORpHnC4RySNMtV1a3AR4AXD60+C3jZxEKSJW32GuDxbd3jgb2n2e35wK8n2SnJVsDRwOfXV0eSh1XV+VX158B3gT36z0aS5gdDtCTNDX8DDN+l4+XA0iRfT/IN4CVt/ceAHZKsBn4f+NZUO6uqdcBrgM8BFwFfraozNlDDWye+iMhgrPRFm3w2kjTH5eef2EmSJEnaGF6JliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6vT/AWDuw0FJ2hJrAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "visualize_importances(range(12),np.mean(cond_vals, axis=0),title=\"Average Neuron Importances\", axis_title=\"Neurons\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can also look at the distribution of each neuron's attributions. Below we look at the distributions for neurons 7 and 9, and we can confirm that their attribution distributions are very close to 0, suggesting they are not learning substantial features." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAVx0lEQVR4nO3de7SkVX3m8e8TWkAkkVuDyMWGhMSgo0lsL5CYMZJRREbILHQwRDuIsoya6MQkNLIyzMRoMJkVLwuNYYHCROUS4gyMkCCDoMlE0YYQ5SJ2BxHaRmgEBDGjgr/5o95mikOdPnU9fc7m+1nrrHprv/vd7951+jy1a79V1akqJElt+bFt3QFJ0vQZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcpREleUeSM6fY3neTHNhtn53kj6fY9oeT/OG02tPyYbhrQUluTXJnkif1lb0+yVXbsFtD6/q6oQvRv0vy1K3UvSrJ/03yQJL7k1yTZG2SHbbUqap3V9XrhzjvVUkWrFdVO1fVLcOPaN7z/WaSf5jT9hur6p2Ttq3lx3DXsFYAb531SZKsmHJ7/xZ4N3AUsBvwdeDcBQ57S1X9OLA38HbgWODSJJly36Y6Vqmf4a5h/Rnwe0l2GbQzydOTXJ7kniQ3J3lV375HzWDnzjCTVJI3J1kPrO/KDk3ypSTf6W4PndPeO5P8n26G/ekke8zT738P/HVV3VBVPwDeCfxykp9caMBV9WBVXQW8AjgEeHl3/v+S5GPd9o5JPpbk20nu6/q6V5J3AS8ETu9eMZy+lbFWkp/qO/Ue3WP5QJLPJnlaV29VV/eRJ4Utj22SnwU+DBzSne++bv+jlnmSvKF7FXNPkov7X8V0bb8xyfok9yb54LSf0LR4DHcNax1wFfB7c3d0yzWXA58A9gReDXwoyTNGaP9o4PnAwUl2Ay4BPgDsDvw5cEmS3fvq/zpwfHe+7Qf1a0v3up/++wDPHLZjVXUbvfG/cMDuNcCTgf26vr4R+NeqOgX4e3qvAnauqrf0HfPIWOc55XH0noT2AK4DPj5EH2/qzv357nyPeRJO8mLgT4BX0XtV8g3gvDnVjgSeCzy7q/fShc6tpclw1yj+M/DbSVbOKT8SuLWqPlpVD1XVtcDfAMeM0PafVNU9VfWv9GbI66vqr7r2zgW+Sm8WvsVHq+prXf0LgJ+bp91LgVcleVaSJ3ZjKGCnEfoGsIness5cP6QX6j9VVQ9X1TVVdf8CbfWPdZBLqupzVfV94BR6s/H9RuzvIMcBH6mqa7u2T+7aXtVX57Squq97QruS+R9XLXGGu4ZWVdcDnwLWztn1NOD53bLEfd2SwHHAU0Zo/va+7afSm1X2+wawT9/9b/Vtfw/YeZ4+XwGcSu/J5hvArcADwMYR+kZ37nsGlP8VcBlwXpJNSf40yRMWaOv2YfdX1Xe78857EXgEj3pcu7a/zRiPq5Y+w12jOhV4A48OhNuBz1bVLn0/O1fVb3X7H+TRM+VBod//9aSb6D1h9Nsf+OY4Ha6qD1bVQVW1J72QXwFcP+zx3az5OfSWWea2/cOq+q9VdTBwKL1XMa/dsnu+Li1wykdm6Ul2pveKYRO9xxHmfywXavdRj2u3nLY7Yz6uWtoMd42kqjYA5wO/01f8KeCnk7wmyRO6n+d2F/mgt278H5Ls1F04PGGB01zatffrSVYk+Y/01qc/NWp/uwuez0zP/sAZwPur6t4hjt2pe7fNRcAXu37NrfMrSf5Nku2A++kt0zzc7b4TOHDUPgNHJPmlJNvTW3u/uqpur6rN9IL4N5Jsl+R1QP+F4TuBfbvjBvkEcHySn+ve2vnuru1bx+ijljjDXeP4I+CR97xX1QPAS+i9ZXATvZf27wG2vDf8vcAP6IXPOSxwgbCqvk1vBvx2essGfwAcWVV3j9HXHemF2nfpBfTngYU+1HN6kge6/r6P3mz/8Kr60YC6TwEupBfsNwGfBT7W7Xs/cEz3zpMPjNDnT9B7hXQPvVcMx/XtewPw+/Qel2cA/9i37zPADcC3kjzmseqWqP6wG88d9J4Yjh2hX1pG4n/WIUntceYuSQ0y3CWpQYa7JDXIcJekBi2JLy7aY489atWqVdu6G5K0rFxzzTV3V9XcT4wDSyTcV61axbp167Z1NyRpWUky95Pcj3BZRpIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGrQkPqEqSY8nq9Ze8sj2rae9fCbncOYuSQ1aMNyTfCTJXUmu7yv7syRfTfLlJP8jyS59+05OsiHJzUleOquOS5LmN8zM/Wzg8DlllwPPrKpnAV8DTgZIcjC9/5PxGd0xH+r+42BJ0iJaMNyr6nP0/qPe/rJPV9VD3d0vAPt220cB51XV96vq68AG4HlT7K8kaQjTWHN/HfC33fY+wO19+zZ2ZY+R5MQk65Ks27x58xS6IUnaYqJwT3IK8BDw8S1FA6rVoGOr6oyqWl1Vq1euHPhd85KkMY39Vsgka4AjgcOqakuAbwT266u2L7Bp/O5JksYx1sw9yeHAScArqup7fbsuBo5NskOSA4CDgC9O3k1J0igWnLknORd4EbBHko3AqfTeHbMDcHkSgC9U1Rur6oYkFwA30luueXNVPTyrzkuSBlsw3Kvq1QOKz9pK/XcB75qkU5KkyfgJVUlqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMWDPckH0lyV5Lr+8p2S3J5kvXd7a5deZJ8IMmGJF9O8guz7LwkabBhZu5nA4fPKVsLXFFVBwFXdPcBXgYc1P2cCPzFdLopSRrFguFeVZ8D7plTfBRwTrd9DnB0X/l/r54vALsk2XtanZUkDWfcNfe9quoOgO52z658H+D2vnobu7LHSHJiknVJ1m3evHnMbkiSBpn2BdUMKKtBFavqjKpaXVWrV65cOeVuSNLj27jhfueW5Zbu9q6ufCOwX1+9fYFN43dPkjSOccP9YmBNt70GuKiv/LXdu2ZeAHxny/KNJGnxrFioQpJzgRcBeyTZCJwKnAZckOQE4DbglV31S4EjgA3A94DjZ9BnSdICFgz3qnr1PLsOG1C3gDdP2ilJ0mT8hKokNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNWiicE/yn5LckOT6JOcm2THJAUmuTrI+yflJtp9WZyVJwxk73JPsA/wOsLqqnglsBxwLvAd4b1UdBNwLnDCNjkqShjfpsswK4IlJVgA7AXcALwYu7PafAxw94TkkSSMaO9yr6pvAfwNuoxfq3wGuAe6rqoe6ahuBfQYdn+TEJOuSrNu8efO43ZAkDTDJssyuwFHAAcBTgScBLxtQtQYdX1VnVNXqqlq9cuXKcbshSRpgkmWZXwW+XlWbq+qHwCeBQ4FdumUagH2BTRP2UZI0oknC/TbgBUl2ShLgMOBG4ErgmK7OGuCiybooSRrVJGvuV9O7cHot8JWurTOAk4DfTbIB2B04awr9lCSNYMXCVeZXVacCp84pvgV43iTtSpIm4ydUJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBE4V7kl2SXJjkq0luSnJIkt2SXJ5kfXe767Q6K0kazqQz9/cDf1dVTweeDdwErAWuqKqDgCu6+5KkRTR2uCf5CeCXgbMAquoHVXUfcBRwTlftHODoSTspSRrNJDP3A4HNwEeT/FOSM5M8Cdirqu4A6G73HHRwkhOTrEuybvPmzRN0Q5I01yThvgL4BeAvqurngQcZYQmmqs6oqtVVtXrlypUTdEOSNNck4b4R2FhVV3f3L6QX9ncm2Rugu71rsi5KkkY1drhX1beA25P8TFd0GHAjcDGwpitbA1w0UQ8lSSNbMeHxvw18PMn2wC3A8fSeMC5IcgJwG/DKCc8hSRrRROFeVdcBqwfsOmySdiVJk/ETqpLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUErtnUHJOnxYNXaSxb1fM7cJalBE4d7ku2S/FOST3X3D0hydZL1Sc5Psv3k3ZQkjWIaM/e3Ajf13X8P8N6qOgi4FzhhCueQJI1gonBPsi/wcuDM7n6AFwMXdlXOAY6e5BySpNFNOnN/H/AHwI+6+7sD91XVQ939jcA+gw5McmKSdUnWbd68ecJuSJL6jR3uSY4E7qqqa/qLB1StQcdX1RlVtbqqVq9cuXLcbkiSBpjkrZC/CLwiyRHAjsBP0JvJ75JkRTd73xfYNHk3JUmjGHvmXlUnV9W+VbUKOBb4TFUdB1wJHNNVWwNcNHEvJUkjmcX73E8CfjfJBnpr8GfN4BySpK2YyidUq+oq4Kpu+xbgedNoV5I0Hj+hKkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktSgFdu6A1p8q9Ze8sj2rae9fBv2RNKsOHOXpAYZ7pLUIMNdkho0drgn2S/JlUluSnJDkrd25bsluTzJ+u521+l1V5I0jEkuqD4EvL2qrk3y48A1SS4HfhO4oqpOS7IWWAucNHlXNQteXJXaNPbMvaruqKpru+0HgJuAfYCjgHO6aucAR0/aSUnSaKbyVsgkq4CfB64G9qqqO6D3BJBkz3mOORE4EWD//fefRjc0Rc7opeVt4guqSXYG/gZ4W1XdP+xxVXVGVa2uqtUrV66ctBuSpD4ThXuSJ9AL9o9X1Se74juT7N3t3xu4a7IuSpJGNcm7ZQKcBdxUVX/et+tiYE23vQa4aPzuSZLGMcma+y8CrwG+kuS6ruwdwGnABUlOAG4DXjlZFyVJoxo73KvqH4DMs/uwcduVJE3OT6hKUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDpvKVv2pD/9f8SlrenLlLUoMMd0lqkOEuSQ1yzV2SpmipXLsy3LUg/z9VaflxWUaSGuTMXWqEr7DUz5m7JDXImbukZcVXKMMx3KUGLXYAzuJ8SyHEl0IfxuWyjCQ1yJl7o2Y145jvPbzLbVajpWM5z46XMmfuktQgZ+5aUoaZxTnTm62tfcJymMd7MV/dTXKu1v8dGe6PA0vl49D9ptWn5fpkMGqflsoYluK/pWkYZ1xL/bGY2bJMksOT3JxkQ5K1szqPJOmxZjJzT7Id8EHg3wEbgS8lubiqbpz2uZbKjGYxLcWLmsP0aTFnOvOda5wlh2H6PcmrhlF/n9Oa9W/Lmeeov59J+vp4zAiY3cz9ecCGqrqlqn4AnAccNaNzSZLmSFVNv9HkGODwqnp9d/81wPOr6i19dU4ETuzu/gxw89Q7Mrw9gLu34fkXQ+tjdHzLX+tjnMX4nlZVKwftmNUF1Qwoe9SzSFWdAZwxo/OPJMm6qlq9rfsxS62P0fEtf62PcbHHN6tlmY3Afn339wU2zehckqQ5ZhXuXwIOSnJAku2BY4GLZ3QuSdIcM1mWqaqHkrwFuAzYDvhIVd0wi3NNyZJYHpqx1sfo+Ja/1se4qOObyQVVSdK25XfLSFKDDHdJalBz4Z5ktySXJ1nf3e46T701XZ31Sdb0lT8nyVe6r034QJJ05a9MckOSHyVZPaetk7v6Nyd56TId38B2kzw5yf9K8s/d+I+f5fi2xRi7fS9Kcl03xs+2Nr5u/3OTPNx9DqWZ8SU5LsmXu59/TPLsGY5tq1+rkmSHJOd3+69Osqpv38CcmK/N9N6QcnU33vPTe3PK8KqqqR/gT4G13fZa4D0D6uwG3NLd7tpt79rt+yJwCL336v8t8LKu/GfpfdjqKmB1X1sHA/8M7AAcAPwLsN0yHN/AdoF39G2vBO4Btl+mv8P5xrgLcCOwf3d/z5bG193fDvgMcClwTEvjAw7tO/ZlwNUzGtd23d/3gcD29P7uD55T503Ah7vtY4Hzu+2BObG1NoELgGO77Q8DvzVSf2f5S94WP/Q+6bp3t703cPOAOq8G/rLv/l92ZXsDX52vXld2FY8O95OBk/vuXwYcstzGN1+73fg+1P2hHQBsAH5sOf4OtzLGNwF/vNz/jW6tXeBtwJuBs5l9uC/6+Prq7wp8c0bjOgS4rO/+o/72u7JH/v7pvRvx7u5vZ2BOzNdmd8zdwIpB5x7mp7llGWCvqroDoLvdc0CdfYDb++5v7Mr26bbnlm/NfG3NyqzGN1+7p9N71bIJ+Arw1qr60XSGMq/FHuNPA7smuSrJNUleO7WRDLao40uyD/Br9GZ/i2Gxf3/9TqA325+FYf7WH6lTVQ8B3wF238qx85XvDtzXtTHfubZqWX6fe5L/DTxlwK5Thm1iQFltpXyctsa2xMb3UuA64MXATwKXJ/n7qrp/yL4M7uDSGuMK4DnAYcATgc8n+UJVfW3Ivjy2c0trfO8DTqqqh7vl64ktsfFt6dOv0Av3XxqyD6Mapm+jjmvQBHuix2GLZRnuVfWr8+1LcmeSvavqjiR7A3cNqLYReFHf/X3pLbds7Lb7yxf62oSpf9XCNhrffO0eD5xWvdeGG5J8HXg6vXXRsS2xMW4E7q6qB4EHk3wOeDYwdrgvsfGtBs7rgn0P4IgkD1XV/xx9ZD1LbHwkeRZwJr31+W+PMaRhDPO3vqXOxiQrgCfTu061tWMHld8N7JJkRTd7HzlXWlyWuRjYcuV9DXDRgDqXAS9Jsmt3xf0l9Naz7gAeSPKC7gr9a+c5fu75ju2ukh8AHMSEwTfE+WYxvvnavY3ejJYke9G7qHzLdIf0GIs9xouAFyZZkWQn4PnATdMeVJ9FHV9VHVBVq6pqFXAh8KZJgn0Iizq+JPsDnwReM8mrrSEM87Uq/X08BvhMNzGaLycGttkdc2XXBsz/OM5vFhcetuUPvbWqK4D13e1uXflq4My+eq+jd3FwA3B8X/lq4Hp6V7BP5/9/ivfX6D37fh+4k0dfBDmlq38z3ZX9ZTi++dp9KvBpeuvt1wO/sYx/hwPb7fb9Pr13zFwPvK218fUdezazv6C62P9GzwTupbd8eB2wboZjO4LeK7p/AU7pyv4IeEW3vSPw192Yvggc2HfswJwY1GZXfmDXxoauzR1G6atfPyBJDWpxWUaSHvcMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktSg/wfP05cSp33PDQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAUcklEQVR4nO3df9RlVX3f8fenjGB10vBjBjIO6EBCYyBt1Iy/E5tK2yBaYXVhiqE6NegsIyQm0cYhrFQba4LaRuIy0TUVZFTkR9EuWGKbzELQpK3EgVAER5wJEhhnhFFEiKYq+u0f98z0zsN95nnur+fHnvdrrWfdc8/ZZ5+975n53H33uT9SVUiS2vL3FrsBkqTJM9wlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtDSvI7ST44wfr+NslJ3fLlSf7jBOv+QJLfnVR9Wj4Md80pyb1JHkjy5L51r01y8yI2a16SnNuF576/7ySpJD87S/mbk/zfJI8meSTJrUk2JTliX5mq+v2qeu08jn1zkjnLVdXKqrpnuJ4NPN6/TfIXM+p+fVW9fdy6tfwY7pqvFcAbp32QJCsmWV9VXdGF58qqWgm8AbgHuO0gu11QVT8CrAHeBJwDfCpJJtm2SfdV6me4a77eDbw5yZGDNiZ5epKtSR5KcneSX+rbdsAIduYIsxtJn59kB7CjW/eCJJ9P8q3u9gUz6nt7kv/ZjbD/LMmqefZjA/DhmsdHs6vq21V1M/By4PnAS7vjvy3JR7vlJyb5aJJvJHm4a+txSd4B/Dzwvu4Vw/sO0tdK8hN9h17VPZaPJvlMkqd15dZ1Zfc/Kex7bJP8FPAB4Pnd8R7uth8wzZPkdUl2dufp+iRPmXEeXp9kR5JvJvnjST+haeEY7pqvbcDNwJtnbuima7YCHwOOBV4J/EmSU4eo/yzgucApSY4GbgDeCxwD/CFwQ5Jj+sr/MvCa7niHD2rXgHY+DXgR8OEh2kVV3Uev/z8/YPMG4EeBE7q2vh74u6q6CPhzeq8CVlbVBX377O/rLIc8F3g7sAq4HbhiHm3c3h37f3fHe9yTcJIXA38A/BK9VyV/A1w1o9jLgGcDP9OV+8W5jq2lyXDXMP498GtJVs9Y/zLg3qr6UFU9VlW3AR8Hzh6i7j+oqoeq6u/ojZB3VNVHuvquBL4E/Mu+8h+qqi935a8BnjGPY7wa+POq+soQ7dpnN3D0gPXfpxfqP1FVP6iqW6vqkTnq6u/rIDdU1Wer6rvARfRG4yeM0OaZzgUuq6rburov7Ope11fm4qp6uHtCu4n5Pa5aggx3zVtV3Ql8Etg0Y9PTgOd20xIPd1MC5wI/NkT19/ctP4XeqLLf3wBr++5/rW/5O8DKeRzj1cCWIdrUby3w0ID1HwH+FLgqye4k70ryhDnqun++26vqb7vjPmX24vN2wOPa1f0Nxn9ctQQZ7hrWW4HXcWAg3A98pqqO7PtbWVW/2m3/NvCkvvKDQr9/Dnw3vSeMfk8Fvjpqo5O8kF64XTvCvicAP0tvmuUAVfX9qvoPVXUK8AJ6r2JevW/zLFXONd+/f5SeZCW9Vwy76T2OMPtjOVe9Bzyu3XTaMYzxuGrpMtw1lKraCVwN/Hrf6k8C/zDJq5I8oft7dneRD3rzxv8qyZO6C4fnzXGYT3X1/XKSFUn+Nb356U+O0fQNwMer6tH57tC1958A1wF/2bVrZpl/muQfJTkMeITeNM0Pus0PACeN0NYzkvxcksPpzb3fUlX3V9VeekH8b5IcluRXgB/v2+8B4Phuv0E+BrwmyTO6t3b+flf3vSO0UUuc4a5R/B6w/z3vXWD+C3pvGdxN76X9O4F97w1/D/A9euGzhTkuEFbVN+iNgN9Eb9rgt4GXVdXXR2lskifSuzg43ymZ9yV5tGvvJfSuH5xeVT8cUPbH6L0aeATYDnwG+Gi37Y+As7t3nrx3iCZ/jN4rpIfovWI4t2/b64B/R+9xORX4X33bPg3cBXwtyeMeq6q6Efjdrj976D0xnDNEu7SMxB/rkKT2OHKXpAYZ7pLUIMNdkhpkuEtSg5bEFxetWrWq1q1bt9jNkKRl5dZbb/16Vc38xDiwRMJ93bp1bNu2bbGbIUnLSpKZn+Tez2kZSWqQ4S5JDZoz3JNcluTBJHf2rXt3ki8luSPJf+v/ju8kF3bfF313Er8uVJIWwXxG7pcDp89YtxX46ar6x8CX6X11KElOofdx5lO7ff6k+84NSdICmjPcq+qzzPiq06r6s6p6rLv7OeD4bvlM4Kqq+m73ndk7gedMsL2SpHmYxJz7rwD/vVtey4HfVb2LA78adr8kG5NsS7Jt7969E2iGJGmfscI9yUXAY/z/b/kb9HuLA7+ZrKo2V9X6qlq/evXAt2lKkkY08vvck2yg97Wsp/X92PAu+n5ogN50ze7RmydJGsVII/ckpwNvAV5eVd/p23Q9cE6SI5KcCJxM70cOJEkLaM6Re5IrgV8AViXZRe9HBC6k90MMW5MAfK6qXl9VdyW5Bvgivema86vqB4Nrnox1m27Yv3zvxS+d5qEkadmYM9yr6pUDVl96kPLvAN4xTqMkSePxE6qS1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaNGe4J7ksyYNJ7uxbd3SSrUl2dLdHdeuT5L1Jdia5I8mzptl4SdJg8xm5Xw6cPmPdJuDGqjoZuLG7D/AS4OTubyPw/sk0U5I0jDnDvao+Czw0Y/WZwJZueQtwVt/6D1fP54Ajk6yZVGMlSfMz6pz7cVW1B6C7PbZbvxa4v6/crm7d4yTZmGRbkm179+4dsRmSpEEmfUE1A9bVoIJVtbmq1lfV+tWrV0+4GZJ0aBs13B/YN93S3T7Yrd8FnNBX7nhg9+jNkySNYtRwvx7Y0C1vAK7rW//q7l0zzwO+tW/6RpK0cFbMVSDJlcAvAKuS7ALeClwMXJPkPOA+4BVd8U8BZwA7ge8Ar5lCmyVJc5gz3KvqlbNsOm1A2QLOH7dRkqTx+AlVSWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDRor3JP8ZpK7ktyZ5MokT0xyYpJbkuxIcnWSwyfVWEnS/Iwc7knWAr8OrK+qnwYOA84B3gm8p6pOBr4JnDeJhkqS5m/caZkVwN9PsgJ4ErAHeDFwbbd9C3DWmMeQJA1p5HCvqq8C/wm4j16ofwu4FXi4qh7riu0C1g7aP8nGJNuSbNu7d++ozZAkDTDOtMxRwJnAicBTgCcDLxlQtAbtX1Wbq2p9Va1fvXr1qM2QJA0wzrTMPwO+UlV7q+r7wCeAFwBHdtM0AMcDu8dsoyRpSOOE+33A85I8KUmA04AvAjcBZ3dlNgDXjddESdKwxplzv4XehdPbgC90dW0G3gL8VpKdwDHApRNopyRpCCvmLjK7qnor8NYZq+8BnjNOvZKk8fgJVUlqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0aK9yTHJnk2iRfSrI9yfOTHJ1ka5Id3e1Rk2qsJGl+xh25/xHwP6rq6cDPANuBTcCNVXUycGN3X5K0gEYO9yT/AHgRcClAVX2vqh4GzgS2dMW2AGeN20hJ0nDGGbmfBOwFPpTkr5J8MMmTgeOqag9Ad3vsBNopSRrCOOG+AngW8P6qeibwbYaYgkmyMcm2JNv27t07RjMkSTONE+67gF1VdUt3/1p6Yf9AkjUA3e2Dg3auqs1Vtb6q1q9evXqMZkiSZho53Kvqa8D9SX6yW3Ua8EXgemBDt24DcN1YLZQkDW3FmPv/GnBFksOBe4DX0HvCuCbJecB9wCvGPIYkaUhjhXtV3Q6sH7DptHHqlSSNx0+oSlKDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJatDY4Z7ksCR/leST3f0Tk9ySZEeSq5McPn4zJUnDmMTI/Y3A9r777wTeU1UnA98EzpvAMSRJQxgr3JMcD7wU+GB3P8CLgWu7IluAs8Y5hiRpeOOO3C8Bfhv4YXf/GODhqnqsu78LWDvmMSRJQxo53JO8DHiwqm7tXz2gaM2y/8Yk25Js27t376jNkCQNMM7I/YXAy5PcC1xFbzrmEuDIJCu6MscDuwftXFWbq2p9Va1fvXr1GM2QJM00crhX1YVVdXxVrQPOAT5dVecCNwFnd8U2ANeN3UpJ0lCm8T73twC/lWQnvTn4S6dwDEnSQayYu8jcqupm4OZu+R7gOZOoV5I0Gj+hKkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDZrIJ1SXinWbbjjg/r0Xv3SRWiJJi8uRuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDVo5HBPckKSm5JsT3JXkjd2649OsjXJju72qMk1V5I0H+OM3B8D3lRVPwU8Dzg/ySnAJuDGqjoZuLG7L0laQCOHe1XtqarbuuVHge3AWuBMYEtXbAtw1riNlCQNZyI/kJ1kHfBM4BbguKraA70ngCTHzrLPRmAjwFOf+tRJNONx+n8w2x/LlnQoGTvck6wEPg78RlU9kmRe+1XVZmAzwPr162vcdkjSXA6lAd9Y75ZJ8gR6wX5FVX2iW/1AkjXd9jXAg+M1UZI0rHHeLRPgUmB7Vf1h36brgQ3d8gbgutGbJ0kaxTjTMi8EXgV8Icnt3brfAS4GrklyHnAf8IrxmihJGtbI4V5VfwHMNsF+2qj1SpLG5ydUJalBE3krpCQttkPpnTDz4chdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNeiQeSukb5OSdChx5C5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIadMi8FVIalW+jXTiTeqw9Z47cJalJh+TI3Wd1aXnx/+zwDslw7+c/msnwcdRS1f9v81DitIwkNeiQH7lLWlzDjqy96Do/jtwlqUGO3CUtuPmM1idVZlitjOgduUtSgxy5j2Eaz/DzqXO2MvNZP9N8yi1k38YpP+6+y33ENs6/HbXHcF8mxrnotBQtdHAPa9ov9/tNow9LJcSXSjvmstT/v4xiatMySU5PcneSnUk2Tes4kqTHm8rIPclhwB8D/xzYBXw+yfVV9cVpHG9SRpnWmKvMfKZNhq1z2DKTNO1XENN4W9wkH6NpT33Mp63TqHMUw74SWe6j45ntX+pvw5zWyP05wM6quqeqvgdcBZw5pWNJkmZIVU2+0uRs4PSqem13/1XAc6vqgr4yG4GN3d2fBO4e4VCrgK+P2dylyH4tL632C9rtWyv9elpVrR60YVoXVDNg3QHPIlW1Gdg81kGSbVW1fpw6liL7tby02i9ot2+t9qvftKZldgEn9N0/Htg9pWNJkmaYVrh/Hjg5yYlJDgfOAa6f0rEkSTNMZVqmqh5LcgHwp8BhwGVVddcUDjXWtM4SZr+Wl1b7Be32rdV+7TeVC6qSpMXld8tIUoMMd0lq0KKG+1xfUZDkiCRXd9tvSbKub9uF3fq7k/ziXHV2F3dvSbKjq/PwRvp1eZKvJLm9+3vGtPo1xb5dluTBJHfOqOvoJFu7c7Y1yVGN9OttSb7ad87OWC79SnJCkpuSbE9yV5I39pVftudrjn4t2PmaqKpalD96F1r/GjgJOBz4P8ApM8q8AfhAt3wOcHW3fEpX/gjgxK6eww5WJ3ANcE63/AHgVxvp1+XA2cv1nHXbXgQ8C7hzRl3vAjZ1y5uAdzbSr7cBb16O5wtYAzyrK/MjwJf7/i0u2/M1R78W5HxN+m8xR+7z+YqCM4Et3fK1wGlJ0q2/qqq+W1VfAXZ29Q2ss9vnxV0ddHWetdz7NaX2H8w0+kZVfRZ4aMDx+utabufsYP1aKBPvV1XtqarbAKrqUWA7sHZAXcvqfM3Rr2VpMcN9LXB/3/1dPP7B3F+mqh4DvgUcc5B9Z1t/DPBwV8dsx5qUhezXPu9IckeS9yQ5YhKdmMU0+nYwx1XVnq6uPcCxI7f84Ba6XwAXdOfssilOX0y1X91UxzOBW7pVTZyvAf2ChTlfE7WY4T7nVxQcpMyk1k/DQvYL4ELg6cCzgaOBt8yvmSOZRt+WgoXu1/uBHweeAewB/vNcDRzR1PqVZCXwceA3quqRkVs4moXu10Kdr4lazHCfz1cU7C+TZAXwo/Re5s6272zrvw4c2dUx27EmZSH7Rfdysqrqu8CH6KYEpmQafTuYB5Ks6epaAzw4cssPbkH7VVUPVNUPquqHwH9heudsKv1K8gR6AXhFVX2ir8yyPl+z9WsBz9dkLdZkP71Px95D76LGvosip84ocz4HXhS5pls+lQMvitxD76LIrHUC/5UDL6i+oZF+reluA1wCXLyczlnffut4/IXHd3PgBbp3NdKvNX3Lv0lvDnhZ9Kv7d/Zh4JIBx1u252uOfi3I+Zr447SoB4cz6F2V/mvgom7d7wEv75afSC+UdwJ/CZzUt+9F3X53Ay85WJ3d+pO6OnZ2dR7RSL8+DXwBuBP4KLByGZ6zK+m93P0+vZHVed36Y4AbgR3d7dGN9Osj3Tm7g953Lq1ZLv0Cfo7eNMYdwO3d3xnL/XzN0a8FO1+T/PPrBySpQX5CVZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBv0/H3t6tVcHw7wAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plt.hist(cond_vals[:,9], 100);\n", + "plt.title(\"Neuron 9 Distribution\")\n", + "plt.figure()\n", + "plt.hist(cond_vals[:,7], 100);\n", + "plt.title(\"Neuron 7 Distribution\");\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, we can look at the distributions of neurons 0 and 10, which appear to be learning strong features negatively correlated with survival." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEICAYAAABGaK+TAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAUPUlEQVR4nO3df7TkdX3f8edLfkgUCb8umxWElbhFMadgc4NGa5u4kmAgwklQMYasFrvHmljTaM0mamo0bZZ6qkmOSXr2iLohRvkRKQRSK1kl0daiiwKRgC7SBTbg7uWnCCaKefeP+S5ch7k7c3/M3P3sPh/nzPn+nO/3PZ977+t+5jPfmUlVIUlqz5OWuwBJ0sIY4JLUKANckhplgEtSowxwSWqUAS5JjTLApTkk+e9J3rlExzo2ybeS7NctX5Pk9Utx7O54/zPJ2qU6ntpggOsxSbYl2ZHkqbPWvT7JNctY1siSrElyS5JHknwmyXG72Xdbkm8neSjJA0n+T5I3JHnsb6Kq3lBV7xnhvNuSvHR3+1TVHVV1cFV9b36PauD53pXkT/uO/7Kq2rTYY6stBrj67Q+8edwnSbL/Eh/vSOATwDuBw4EtwEVD7vazVfU04DhgA/DrwAVLWVdX25I+VmkXA1z93gu8NcmhgzYmeXaSq5Pcl+SrSV45a9v3DQskeW2Sz81ariS/nGQrsLVb98IkX0zyYDd9Yd/x3pPkf3c95U91QT3IzwE3VdUlVfUPwLuAk5I8e9gDrqoHq+oK4FXA2iQ/0p3/I0l+p5s/MsmVXW/9viSfTfKkJBcCxwJ/0Q2RvC3Jqu6xnpfkDuDTs9bNDvMfTvKF7rFfnuTw7lw/kWR7X7tvS/LSJKcBvwm8qjvfDf1t39X1jiS3J9mZ5E+S/GC3bVcda5PckeSeJG8f1kbaMxng6rcFuAZ4a/+GbmjlauDPgKOAVwN/lOS58zj+WcDzgRO7wLoK+APgCOB9wFVJjpi1/y8Ar+vOd+CgujrPBW7YtVBVDwNf79aPpKq+AGwHXjxg81u6bVPACnohWlV1LnAHvd78wVX1X2fd518DzwF+eo5T/hLwb4CnA4/Sa4dhNX4S+C/ARd35Thqw22u7208CxwMHAx/o2+dfAicAa4DfSvKcYefWnscA1yC/BbwpyVTf+jOAbVX14ap6tKq+BPw5cPY8jv27VXVfVX0bOB3YWlUXdsf7GHAL8LOz9v9wVX2t2/9i4OQ5jnsw8GDfugeBp82jNoC76A3B9PsusBI4rqq+W1WfreEfJPSuqnq4q32QC6vqK90/m3cCr9z1IucivQZ4X1XdVlXfAn4DOKev9//bVfXtqrqB3j++Qf8ItIczwPUEVfUV4Epgfd+m44Dnd8MIDyR5gF5Y/NA8Dn/nrPmnA7f3bb8dOHrW8jdmzT9CL6gH+RZwSN+6Q4CH5lEb3bnvG7D+vcCtwKeS3Jakv20GuXMe228HDgDmGiKaj/52vZ3eaxsrZq0btV21BzPANZf/BPxbvj9M7wT+uqoOnXU7uKr+Xbf9YeAps/YfFOyze6130funMNuxwN8voN6bmNWL7IZ7frhbP5IkP0bv8X6uf1tVPVRVb6mq4+k9Q/i1JGt2bZ7jkMN66M+YNX8svV7+PfS1Y9crn/1saNhx+9v1WHpDNDuG3E+NMcA1UFXdSu8qjn8/a/WVwD9Lcm6SA7rbj80aP70e+LkkT0nyLOC8Iaf5y+54v5Bk/ySvAk7szjNflwE/kuTnkxxEbxjoxqq6ZdgdkxyS5Azg48CfVtXfDtjnjCTPShLgm8D3uhv0gvH4BdT8i0lOTPIU4N3Apd1lhl8DDkpyepIDgHcAT551vx3AqtmXPPb5GPAfkjwzycE8Pmb+6AJq1B7MANfuvBt47JrwqnoI+CngHHq9vG8A5/N4uLwf+A69gNkEfHR3B6+qe+mNq78FuBd4G3BGVd0z30Kragb4eeA/A/fTe6H0nCF3+4skD9F7ZvF2ei+ivm6OfVcDf0VvqObzwB9V1TXdtt8F3tENK831IusgFwIfodeOB9H9s6yqB4E3Ah+k92zkYXovoO5ySTe9N8mXBhz3Q92x/wb4f8A/AG+aR11qRPxCB0lqkz1wSWqUAS5JjTLAJalRBrgkNWqiH7Jz5JFH1qpVqyZ5Sklq3nXXXXdPVfW/M3qyAb5q1Sq2bNkyyVNKUvOS9L9jGXAIRZKaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGjXRd2JK0r5i1fqrHpvftuH0sZzDHrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0aGuBJTkhy/azbN5P8apLDk1ydZGs3PWwSBUuSeoYGeFV9tapOrqqTgR8FHgEuA9YDm6tqNbC5W5YkTch8h1DWAF+vqtuBM4FN3fpNwFlLWZgkaffmG+DnAB/r5ldU1d0A3fSoQXdIsi7JliRbZmZmFl6pJOn7jBzgSQ4EXg5cMp8TVNXGqpququmpqan51idJmsN8euAvA75UVTu65R1JVgJ0051LXZwkaW7zCfBX8/jwCcAVwNpufi1w+VIVJUkabqQAT/IU4FTgE7NWbwBOTbK127Zh6cuTJM1lpM8Dr6pHgCP61t1L76oUSdIy8J2YktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqNG/VLjQ5NcmuSWJDcn+fEkhye5OsnWbnrYuIuVJD1u1B747wOfrKpnAycBNwPrgc1VtRrY3C1LkiZkaIAnOQT4V8AFAFX1nap6ADgT2NTttgk4a1xFSpKeaJQe+PHADPDhJF9O8sEkTwVWVNXdAN30qEF3TrIuyZYkW2ZmZpascEna140S4PsD/wL446p6HvAw8xguqaqNVTVdVdNTU1MLLFOS1G+UAN8ObK+qa7vlS+kF+o4kKwG66c7xlChJGmRogFfVN4A7k5zQrVoD/B1wBbC2W7cWuHwsFUqSBtp/xP3eBHw0yYHAbcDr6IX/xUnOA+4AXjGeEiVJg4wU4FV1PTA9YNOapS1HkjQq34kpSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNWqk78RMsg14CPge8GhVTSc5HLgIWAVsA15ZVfePp0xJUr/59MB/sqpOrqpdX268HthcVauBzd2yJGlCFjOEciawqZvfBJy1+HIkSaMaNcAL+FSS65Ks69atqKq7AbrpUYPumGRdki1JtszMzCy+YkkSMOIYOPCiqroryVHA1UluGfUEVbUR2AgwPT1dC6hRkjTASD3wqrqrm+4ELgNOAXYkWQnQTXeOq0hJ0hMNDfAkT03ytF3zwE8BXwGuANZ2u60FLh9XkZKkJxplCGUFcFmSXfv/WVV9MskXgYuTnAfcAbxifGVKkvoNDfCqug04acD6e4E14yhKkjSc78SUpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGjVygCfZL8mXk1zZLT8zybVJtia5KMmB4ytTktRvPj3wNwM3z1o+H3h/Va0G7gfOW8rCJEm7N1KAJzkGOB34YLcc4CXApd0um4CzxlGgJGmwUXvgvwe8DfinbvkI4IGqerRb3g4cPeiOSdYl2ZJky8zMzKKKlSQ9bmiAJzkD2FlV181ePWDXGnT/qtpYVdNVNT01NbXAMiVJ/fYfYZ8XAS9P8jPAQcAh9HrkhybZv+uFHwPcNb4yJUn9hvbAq+o3quqYqloFnAN8uqpeA3wGOLvbbS1w+diqlCQ9wWKuA/914NeS3EpvTPyCpSlJkjSKUYZQHlNV1wDXdPO3AacsfUnSZKxaf9Vj89s2nL6MlUgL4zsxJalRBrgkNWpeQyiSpLnNHpabBHvgktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGDQ3wJAcl+UKSG5LclOS3u/XPTHJtkq1JLkpy4PjLlSTtMkoP/B+Bl1TVScDJwGlJXgCcD7y/qlYD9wPnja9MSVK/oQFePd/qFg/obgW8BLi0W78JOGssFUqSBhrpOzGT7AdcBzwL+EPg68ADVfVot8t24Og57rsOWAdw7LHHLrZeaSxmf5fhtg2nL2Ml0uhGehGzqr5XVScDxwCnAM8ZtNsc991YVdNVNT01NbXwSiVJ32deV6FU1QPANcALgEOT7OrBHwPctbSlSZJ2Z5SrUKaSHNrN/wDwUuBm4DPA2d1ua4HLx1WkJOmJRhkDXwls6sbBnwRcXFVXJvk74ONJfgf4MnDBGOuUJPUZGuBVdSPwvAHrb6M3Hi5JWga+E1OSGmWAS1KjRroOXNpbzL7eW2qdPXBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqNG+Vb6ZyT5TJKbk9yU5M3d+sOTXJ1kazc9bPzlSpJ2GaUH/ijwlqp6DvAC4JeTnAisBzZX1Wpgc7csSZqQoQFeVXdX1Ze6+YeAm4GjgTOBTd1um4CzxlWkJOmJ5jUGnmQV8DzgWmBFVd0NvZAHjprjPuuSbEmyZWZmZnHVSpIeM3KAJzkY+HPgV6vqm6Per6o2VtV0VU1PTU0tpEZJ0gAjBXiSA+iF90er6hPd6h1JVnbbVwI7x1OiJGmQ/YftkCTABcDNVfW+WZuuANYCG7rp5WOpUNpDrFp/1WPz2zacvoyVSD1DAxx4EXAu8LdJru/W/Sa94L44yXnAHcArxlOiJGmQoQFeVZ8DMsfmNUtbjiS1ZfYzs0nznZiS1CgDXJIaZYBLUqMMcElqlAEuSY0a5TJCqWmLuUpgOa8wkIaxBy5JjTLAJalRDqFIfRw2USvsgUtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElq1NAAT/KhJDuTfGXWusOTXJ1kazc9bLxlSpL6jdID/whwWt+69cDmqloNbO6WJUkTNDTAq+pvgPv6Vp8JbOrmNwFnLXFdkqQhFjoGvqKq7gbopkfNtWOSdUm2JNkyMzOzwNNJkvqN/UXMqtpYVdNVNT01NTXu00nSPmOhAb4jyUqAbrpz6UqSJI1ioV/ocAWwFtjQTS9fsoqkJeCXMmhfMMplhB8DPg+ckGR7kvPoBfepSbYCp3bLkqQJGtoDr6pXz7FpzRLXIjWvv+e/bcPpy1SJ9gW+E1OSGmWAS1KjDHBJapQBLkmNMsAlqVELvQ5cUsdrzvcNs3/Oe8rVRfbAJalRBrgkNcohFGkBJj1ssic+fd+X7SnDZvbAJalRBrgkNcohFDXH4QSpxx64JDXKHriatqf3xuf7Ytee+BhaN9fvyCi/O3vKi5VzsQcuSY0ywCWpUQ6haK+xpz/dnaTFDBvM95h7ij29vnGwBy5JjTLAJalRixpCSXIa8PvAfsAHq2psX268Lz490uP2xeGRUR7zYvZZTJvOd4hm1HON+2qQcbTFclpwDzzJfsAfAi8DTgReneTEpSpMkrR7ixlCOQW4tapuq6rvAB8HzlyasiRJw6SqFnbH5GzgtKp6fbd8LvD8qvqVvv3WAeu6xROAry683CYcCdyz3EUsM9ugx3awDWBp2uC4qprqX7mYMfAMWPeE/wZVtRHYuIjzNCXJlqqaXu46lpNt0GM72AYw3jZYzBDKduAZs5aPAe5aXDmSpFEtJsC/CKxO8swkBwLnAFcsTVmSpGEWPIRSVY8m+RXgf9G7jPBDVXXTklXWrn1muGg3bIMe28E2gDG2wYJfxJQkLS/fiSlJjTLAJalRBvgiJTk8ydVJtnbTw3az7yFJ/j7JByZZ47iN0gZJjktyXZLrk9yU5A3LUes4jdgOJyf5fNcGNyZ51XLUOi6j/j0k+WSSB5JcOekaxyXJaUm+muTWJOsHbH9ykou67dcmWbXYcxrgi7ce2FxVq4HN3fJc3gP89USqmqxR2uBu4IVVdTLwfGB9kqdPsMZJGKUdHgF+qaqeC5wG/F6SQydY47iN+vfwXuDciVU1ZiN+tMh5wP1V9Szg/cD5iz2vAb54ZwKbuvlNwFmDdkryo8AK4FMTqmuShrZBVX2nqv6xW3wye+fv3ijt8LWq2trN3wXsBJ7wDruGjfT3UFWbgYcmVdQEjPLRIrPb5lJgTZJBb4gc2d74RzRpK6rqboBuelT/DkmeBPw34D9OuLZJGdoGAEmekeRG4E7g/C7A9iYjtcMuSU4BDgS+PoHaJmVebbAXOZre7/Uu27t1A/epqkeBB4EjFnNSv5FnBEn+CvihAZvePuIh3gj8ZVXduch/uMtmCdqAqroT+Ofd0Mn/SHJpVe1YqhonYSnaoTvOSuBCYG1V/dNS1DYpS9UGe5lRPlpkpI8fmQ8DfARV9dK5tiXZkWRlVd3d/VHuHLDbjwMvTvJG4GDgwCTfqqrdjZfvUZagDWYf664kNwEvpvdUshlL0Q5JDgGuAt5RVf93TKWOzVL+LuxFRvlokV37bE+yP/CDwH2LOalDKIt3BbC2m18LXN6/Q1W9pqqOrapVwFuBP2kpvEcwtA2SHJPkB7r5w4AXsfd9MuUo7XAgcBm934FLJljbpAxtg73UKB8tMrttzgY+XYt9J2VVeVvEjd4Y1mZgazc9vFs/Te9bivr3fy3wgeWue9JtAJwK3Ajc0E3XLXfdy9QOvwh8F7h+1u3k5a59km3QLX8WmAG+Ta9n+tPLXfsSPPafAb5G7zWNt3fr3g28vJs/CLgEuBX4AnD8Ys/pW+klqVEOoUhSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1Kj/D1Psbh+vDtCUAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAEICAYAAACpqsStAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAWoklEQVR4nO3dfbRddX3n8fenxMAoIkQCBjAG1lAU24quW6U6q1VQimAlM0ULVSZaaKpjHV3q0qidap22gjOtbVdnpmZ8ig9FEGuJUqtpADvOKDUqPiBiIkWJiUnkQUVbFf3OH2dfPFzPzT333nPuvT/yfq1119kPv7339/zOzSf7/PY+56aqkCS16WcWuwBJ0twZ4pLUMENckhpmiEtSwwxxSWqYIS5JDTPEpSEluSvJCSPa16uSvLmbXpOkkiwb0b5Xd7UeNIr9aWkzxA9gSW5JsifJA/qWXZTk2kUsayhJlie5onsOleSJU9YnySVJbut+3pAk0+zriUl+3AXfXUl2Jrk8yS/2t6uqQ6vq5hnqemKSnTPVX1V/XFUXDfFUZ9T1wZP79v21rtYfjWL/WtoMcS0DXjTug4zqLHOKjwHPBr4xYN16YC3wKOAXgKcBv7Offe2qqkOBBwKnAl8C/k+S00daMWPrCx2gDHH9N+BlSQ4ftDLJw5NsSXJ7kpuSPLNv3bVJLuqbf06Sj/XNV5IXJNkObO+WPT7JJ5N8q3t8/JT9/dck/zfJd5J8JMmRg+qqqh9U1Z9V1ceAQWec64A/qaqdVfV14E+A58zUGdWzs6p+H3gzcMmU5/Nvu+mzknyxq/PrSV7WvaP5EHBM31n9MUle271reFeSbwPP6Za9a8rhfyvJriS7k7y077hvT/KHffP3nO0neSewGvhAd7yXTx2e6WrY3L2GO5L8dt++Xtu963hH91xuSDIxUz9p6TDEtQ24FnjZ1BVdKG0B/ho4Cjgf+J9JHjmL/a8FHgecnGQFcBXwF8CDgT8Frkry4L72vwk8tzve8kF1DemRwGf75j/bLZuNvwEe0z/c1OctwO9U1QOBnwOurqrvAk+lO6vvfnZ17c8BrgAOB949zfGeBJwInAFs6B8imU5VXQB8Dfi17nhvGNDsUmAncAxwLvDHU95hPB14T1fbZuAvZzqulg5DXAC/D7wwycopy58G3FJVb6uqu6vq08D76AXBsF5fVbdX1b8AZwPbq+qd3f4upTds8Wt97d9WVV/u2l8OnDLH53Qo8K2++W8Bh043Lj6NXUDohdtUP6T3H9NhVXVH1zf78/Gq+tuq+nH33Ab5g6r6blV9Hngbvf805yXJQ4F/B7yiqv61qq6n9w7jgr5mH6uqv+vG0N9JbwhKjTDERVV9AfggsGHKqocBj0ty5+QP8CzgIbPY/a1908cAX52y/qvAsX3z/ePb36MXxnNxF3BY3/xhwF01u298OxYo4M4B634dOAv4apKPJvmlGfZ16wzrp7b5Kr3+mq9jgNur6jtT9r2/Pj/Ecft2GOKa9Brgt7n3P+5bgY9W1eF9P4dW1fO79d8F7t/XflC494fmLnr/MfRbDXx9fqUPdAP3PqN8VLdsNv498OlumORequqTVXUOvWGfv6X3rgHu/XzvtckQx3to3/Rqev0FM/fz/va9C1iR5IFT9j2OPtciMMQFQFXtAC4D/nPf4g8CP5vkgiT3635+MckjuvXXA/8hyf27C34XznCYv+v295tJliX5DeDk7jizluTgJId0s8uTHNI3XPIO4CVJjk1yDPBS4O1D7DPdNq8BLgJeNaDN8iTPSvKgqvoh8G1+cnF1D/DgJA+aw1P6L11fPpLedYHLuuXXA2clWZHkIcCLp2y3Bxh4/3pV3Qr8P+D1Xf/8Ar3XabpxeTXGEFe/1wH3XMTr3oKfAZxH74zuG/Tu1ji4a/JG4Af0QmQTMwRDVd1Gb5z9pcBtwMuBp1XVN+dY703Av9B79/DhbnryTP9NwAeAzwNfoHdB9U372dcxSe6iNwzzSeDngSdW1UemaX8BcEt3t8nz6N3qSFV9id6FxJu7IajZDIl8FNgBbAX+e9+x30nvwuwtwEf4SbhPej3we93xBl0IPh9YQ+81fD/wmqraMou6tITFPwohSe3yTFySGmaIS1LDZgzxJCclub7v59tJXtxdZNmSZHv3eMRCFCxJ+olZjYmn961oX6f3CbwX0Lv/9OIkG4AjquoV4ylTkjTIbEP8DHpXtp+Q5CZ6V+93J1kFXFtVJ+1v+yOPPLLWrFkzr4Il6UDzqU996ptVNfUT1UDvG+xm4zx6t08BHF1VuwG6ID9q0AZJ1tP7RjlWr17Ntm3bZnlISTqwJZn6Sed7DH1hM8lyel+U897ZHLyqNlbVRFVNrFw58D8SSdIczebulKfS+wjynm5+TzeMQve4d9TFSZL2bzYhfj4/GUqB3ldWruum1wFXjqooSdJwhgrxJPcHnkLv+5UnXQw8Jb0v/H9KNy9JWkBDXdisqu/R+xL//mW3ASP/01WSpOH5iU1JapghLkkNM8QlqWGGuCQ1zL+jpwPKmg1X3TN9y8VnL2Il0mh4Ji5JDTPEJalhhrgkNcwQl6SGGeKS1DBDXJIaZohLUsMMcUlqmCEuSQ0zxCWpYYa4JDXMEJekhhniktQwQ1ySGmaIS1LDDHFJapghLkkNGyrEkxye5IokX0pyY5JfSrIiyZYk27vHI8ZdrCTp3oY9E/9z4O+r6uHAo4AbgQ3A1qo6EdjazUuSFtCMIZ7kMOCXgbcAVNUPqupO4BxgU9dsE7B2XEVKkgYb5kz8BGAf8LYkn0ny5iQPAI6uqt0A3eNRgzZOsj7JtiTb9u3bN7LCJUnDhfgy4DHA/6qqRwPfZRZDJ1W1saomqmpi5cqVcyxTkjTIMCG+E9hZVdd181fQC/U9SVYBdI97x1OiJGk6M4Z4VX0DuDXJSd2i04EvApuBdd2ydcCVY6lQkjStZUO2eyHw7iTLgZuB59L7D+DyJBcCXwOeMZ4SJUnTGSrEq+p6YGLAqtNHW44kaTb8xKYkNcwQl6SGGeKS1DBDXJIaZohLUsMMcUlqmCEuSQ0zxCWpYYa4JDXMEJekhhniktQwQ1ySGmaIS1LDDHFJapghLkkNM8QlqWGGuCQ1zBCXpIYZ4pLUMENckhpmiEtSwwxxSWrYsmEaJbkF+A7wI+DuqppIsgK4DFgD3AI8s6ruGE+ZkqRBZnMm/qSqOqWqJrr5DcDWqjoR2NrNS5IW0HyGU84BNnXTm4C18y9HkjQbw4Z4AR9J8qkk67tlR1fVboDu8ahBGyZZn2Rbkm379u2bf8WSpHsMNSYOPKGqdiU5CtiS5EvDHqCqNgIbASYmJmoONUqSpjHUmXhV7eoe9wLvBx4L7EmyCqB73DuuIiVJg80Y4kkekOSBk9PAGcAXgM3Auq7ZOuDKcRUpSRpsmOGUo4H3J5ls/9dV9fdJPglcnuRC4GvAM8ZXpiRpkBlDvKpuBh41YPltwOnjKEqSNBw/sSlJDTPEJalhhrgkNcwQl6SGGeKS1DBDXJIaZohLUsMMcUlqmCEuSQ0zxCWpYYa4JDXMEJekhhniktQwQ1ySGmaIS1LDDHFJapghLkkNM8QlqWGGuCQ1zBCXpIYZ4pLUMENckho2dIgnOSjJZ5J8sJs/Psl1SbYnuSzJ8vGVKUkaZDZn4i8CbuybvwR4Y1WdCNwBXDjKwiRJMxsqxJMcB5wNvLmbD3AacEXXZBOwdhwFSpKmN+yZ+J8BLwd+3M0/GLizqu7u5ncCxw7aMMn6JNuSbNu3b9+8ipUk3duMIZ7kacDeqvpU/+IBTWvQ9lW1saomqmpi5cqVcyxTkjTIsiHaPAF4epKzgEOAw+idmR+eZFl3Nn4csGt8ZUqSBpnxTLyqXllVx1XVGuA84OqqehZwDXBu12wdcOXYqpQkDTSf+8RfAbwkyQ56Y+RvGU1JkqRhDTOcco+quha4tpu+GXjs6EuSJA1rViGu9q3ZcNU907dcfPYiViJpFPzYvSQ1zBCXpIY5nHIAc2hFap9n4pLUMENckhpmiEtSwwxxSWqYIS5JDTPEJalhhrgkNcwQl6SGGeKS1DBDXJIaZohLUsMMcUlqmCEuSQ0zxCWpYYa4JDXM7xM/APR/b7ik+xbPxCWpYYa4JDVsxhBPckiSf0ry2SQ3JPmDbvnxSa5Lsj3JZUmWj79cSVK/Yc7Evw+cVlWPAk4BzkxyKnAJ8MaqOhG4A7hwfGVKkgaZMcSr565u9n7dTwGnAVd0yzcBa8dSoSRpWkONiSc5KMn1wF5gC/AV4M6qurtrshM4dppt1yfZlmTbvn37RlGzJKkzVIhX1Y+q6hTgOOCxwCMGNZtm241VNVFVEytXrpx7pZKknzKru1Oq6k7gWuBU4PAkk/eZHwfsGm1pkqSZDHN3ysokh3fT/wZ4MnAjcA1wbtdsHXDluIqUJA02zCc2VwGbkhxEL/Qvr6oPJvki8J4kfwh8BnjLGOuUJA0wY4hX1eeARw9YfjO98XFJ0iLxE5uS1DBDXJIaZohLUsMMcUlqmCEuSQ0zxCWpYYa4JDXMEJekhhniktQwQ1ySGmaIS1LDDHFJapghLkkNM8QlqWGGuCQ1zBCXpIYZ4pLUMENckhpmiEtSwwxxSWqYIS5JDTPEJalhM4Z4kocmuSbJjUluSPKibvmKJFuSbO8ejxh/uZKkfsOcid8NvLSqHgGcCrwgycnABmBrVZ0IbO3mJUkLaMYQr6rdVfXpbvo7wI3AscA5wKau2SZg7biKlCQNtmw2jZOsAR4NXAccXVW7oRf0SY6aZpv1wHqA1atXz6dWaU7WbLhqsUuQxmboC5tJDgXeB7y4qr497HZVtbGqJqpqYuXKlXOpUZI0jaFCPMn96AX4u6vqb7rFe5Ks6tavAvaOp0RJ0nSGuTslwFuAG6vqT/tWbQbWddPrgCtHX54kaX+GGRN/AnAB8Pkk13fLXgVcDFye5ELga8AzxlOiJGk6M4Z4VX0MyDSrTx9tOZKk2ZjV3Sk6MPTfzXHLxWcvYiWSZuLH7iWpYYa4JDXMEJekhhniktQwQ1ySGmaIS1LDDHFJapghLkkNM8QlqWGGuCQ1zBCXpIYZ4pLUMENckhpmiEtSwwxxSWqYIS5JDTPEJalhhrgkNcw/zybg3n+SbZg2/tk2aWnwTFySGmaIS1LDZhxOSfJW4GnA3qr6uW7ZCuAyYA1wC/DMqrpjfGXe9zg0IWkUhjkTfztw5pRlG4CtVXUisLWblyQtsBlDvKr+Ebh9yuJzgE3d9CZg7YjrkiQNYa53pxxdVbsBqmp3kqOma5hkPbAeYPXq1XM8nIbhEI104Bn7hc2q2lhVE1U1sXLlynEfTpIOKHMN8T1JVgF0j3tHV5IkaVhzHU7ZDKwDLu4erxxZRZqVYT6kM479T13u8I20OGY8E09yKfBx4KQkO5NcSC+8n5JkO/CUbl6StMBmPBOvqvOnWXX6iGuRJM2Sn9iUpIYZ4pLUMENckhrmV9E2oqUP8rRUq9Q6z8QlqWGGuCQ1zOGUBs32r/AshIU+3kw1OIyjA4Vn4pLUMENckhp2QA+nLMW330uxJklLl2fiktQwQ1ySGnZAD6cshGGGR5bCnR0LzWEjaTQ8E5ekhhniktQwQ1ySGnafHRMf15jrdPsdx7j2fWGs/L7wHKSlzDNxSWqYIS5JDWtmOGWYYYxhbuHzNr/7lvm8btNt6y2Paoln4pLUMENckho2r+GUJGcCfw4cBLy5qi4eSVUzGMdb6IXgkM1g47jjZ1TbOrSiuVqo36M5n4knOQj4H8BTgZOB85OcPKrCJEkzm89wymOBHVV1c1X9AHgPcM5oypIkDSNVNbcNk3OBM6vqom7+AuBxVfW7U9qtB9Z3sycBN8293LE6EvjmYhcxA2ucv6VeH1jjKCz1+mB2NT6sqlYOWjGfMfEMWPZT/yNU1UZg4zyOsyCSbKuqicWuY3+scf6Wen1gjaOw1OuD0dU4n+GUncBD++aPA3bNrxxJ0mzMJ8Q/CZyY5Pgky4HzgM2jKUuSNIw5D6dU1d1Jfhf4ML1bDN9aVTeMrLKFt+SHfLDGUVjq9YE1jsJSrw9GVOOcL2xKkhafn9iUpIYZ4pLUsAMqxJOsSLIlyfbu8YgBbZ6U5Pq+n39NsrZb9/Yk/9y37pTFqLFr96O+Ojb3LT8+yXXd9pd1F50XvMYkpyT5eJIbknwuyW/0rRtLPyY5M8lNSXYk2TBg/cFdn+zo+mhN37pXdstvSvKro6hnjjW+JMkXuz7bmuRhfesGvuYLXN9zkuzrq+OivnXrut+J7UnWjaO+IWt8Y199X05yZ9+6hejDtybZm+QL06xPkr/o6v9cksf0rZt9H1bVAfMDvAHY0E1vAC6Zof0K4Hbg/t3824Fzl0KNwF3TLL8cOK+b/ivg+YtRI/CzwInd9DHAbuDwcfUjvYvrXwFOAJYDnwVOntLmPwF/1U2fB1zWTZ/ctT8YOL7bz0Fj6LdhanxS3+/b8ydr3N9rvsD1PQf4ywHbrgBu7h6P6KaPWIwap7R/Ib2bLhakD7tj/DLwGOAL06w/C/gQvc/anApcN58+PKDOxOl9LcCmbnoTsHaG9ucCH6qq7421qnubbY33SBLgNOCKuWw/CzPWWFVfrqrt3fQuYC8w8BNnIzLM10D0130FcHrXZ+cA76mq71fVPwM7uv0teI1VdU3f79sn6H3+YqHM56s0fhXYUlW3V9UdwBbgzCVQ4/nApWOoY1pV9Y/0Tv6mcw7wjur5BHB4klXMsQ8PtBA/uqp2A3SPR83Q/jx++hfgj7q3QG9McvAi1nhIkm1JPjE53AM8GLizqu7u5ncCxy5ijQAkeSy9s6av9C0edT8eC9zaNz/oud/Tpuujb9Hrs2G2HYXZHudCemdskwa95otR3693r90VSSY/8Lfk+rAbijoeuLpv8bj7cBjTPYc59WEzf9lnWEn+AXjIgFWvnuV+VgE/T+8++EmvBL5BL5A2Aq8AXrdINa6uql1JTgCuTvJ54NsD2s3pHtIR9+M7gXVV9eNu8Uj6ceqhBiyb+tynazPUV0iMwNDHSfJsYAL4lb7FP/WaV9VXBm0/xvo+AFxaVd9P8jx672xOG3LbUZjNcc4DrqiqH/UtG3cfDmOkv4f3uRCvqidPty7JniSrqmp3Fy5797OrZwLvr6of9u17dzf5/SRvA162WDV2QxRU1c1JrgUeDbyP3luzZd2Z5py/CmEUNSY5DLgK+L3ubePkvkfSj1MM8zUQk212JlkGPIje296F+gqJoY6T5Mn0/rP8lar6/uTyaV7zUQbQjPVV1W19s/8buKRv2ydO2fbaEdY2aTav1XnAC/oXLEAfDmO65zCnPjzQhlM2A5NXfNcBV+6n7U+NpXWBNTn2vBYYePV53DUmOWJyCCLJkcATgC9W7+rINfTG8qfdfoFqXA68n97Y33unrBtHPw7zNRD9dZ8LXN312WbgvPTuXjkeOBH4pxHUNOsakzwaeBPw9Kra27d84Gu+CPWt6pt9OnBjN/1h4IyuziOAM7j3u9gFq7Gr8yR6Fwc/3rdsIfpwGJuB/9jdpXIq8K3uxGZufTjuK7VL6Yfe+OdWYHv3uKJbPkHvLxNNtlsDfB34mSnbXw18nl7ovAs4dDFqBB7f1fHZ7vHCvu1PoBdAO4D3AgcvUo3PBn4IXN/3c8o4+5HeVf8v0zuzenW37HX0AhHgkK5PdnR9dELftq/utrsJeOoYfwdnqvEfgD19fbZ5ptd8get7PXBDV8c1wMP7tv2trm93AM9drD7s5l8LXDxlu4Xqw0vp3Y31Q3pn1xcCzwOe160PvT+o85Wujon59KEfu5ekhh1owymSdJ9iiEtSwwxxSWqYIS5JDTPEJalhhrgkNcwQl6SG/X9YujghTsXb1QAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plt.hist(cond_vals[:,0], 100);\n", + "plt.title(\"Neuron 0 Distribution\")\n", + "plt.figure()\n", + "plt.hist(cond_vals[:,10], 100);\n", + "plt.title(\"Neuron 10 Distribution\");" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We have identified that some of the neurons are not learning important features, while others are. Can we now understand what each of these important neurons are looking at in the input? For instance, are they identifying different features in the input or similar ones?" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To answer these questions, we can apply the third type of attributions available in Captum, **Neuron Attributions**. This allows us to understand what parts of the input contribute to activating a particular input neuron. For this example, we will apply Neuron Conductance, which divides the neuron's total conductance value into the contribution from each individual input feature." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To use Neuron Conductance, we create a NeuronConductance object, analogously to Conductance, passing in the model as well as the module (layer) whose output we would like to understand, in this case, net.sigmoid1, as before." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "neuron_cond = NeuronConductance(net, net.sigmoid1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can now obtain the neuron conductance values for all the test examples by calling attribute on the NeuronConductance object. Neuron Conductance requires the neuron index in the target layer for which attributions are requested as well as the target index for networks with mutliple outputs, similar to layer conductance. As before, we provide target = 1, corresponding to survival, and compute neuron conductance for neurons 0 and 10, the significant neurons identified above. The neuron index can be provided either as a tuple or as just an integer if the layer output is 1-dimensional." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "neuron_cond_vals_10 = neuron_cond.attribute(test_input_tensor, neuron_index=10, target=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "neuron_cond_vals_0 = neuron_cond.attribute(test_input_tensor, neuron_index=0, target=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Feature Importances for Neuron 0\n", + "age : -0.128\n", + "sibsp : -0.004\n", + "parch : 0.003\n", + "fare : 0.003\n", + "female : 0.000\n", + "male : 0.001\n", + "embark_C : 0.006\n", + "embark_Q : -0.000\n", + "embark_S : -0.004\n", + "class_1 : 0.002\n", + "class_2 : 0.007\n", + "class_3 : -0.019\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtEAAAGECAYAAADwYnzeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deZxkVX3//9dbQERBFkFEQCcSXHDDOAFNXIis6teAu8bo8BMk6NcYf8Z8xaCAoAajfs1ijBIk4C64AWoEHMWFaGRAdpdxGQRBGTYVcMXP9497Woq2uqfPdPf0zPB6Ph716Lq3zr33c2rpetepU1WpKiRJkiTN3J0WugBJkiRpXWOIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJ67wkr09ybZIfLXQtku4YDNHSHUySs5PckGTjha5lLrT+/CLJTSOnR89yn4uSVJIN56rOGRzzqCTvW1PHm06SA5N8eaHrmKkkOwJ/C+xSVfeao31WkouT3Glk3euTnDgX+59vSf4iyeVJbk7yiSRbLXRN0vrGEC3dgSRZBDwWKODP5+kYayx4jnhpVW06cvrKAtTwOxmsk/9fF+j2m637AtdV1TW9G66iv/cGnrPaVc1NDauzvwcD7wKeD2wL3AK8Yy6PIckQLd3RvAD4KnAisGRiZZJHJflRkg1G1j01yUXt/J2SHJbku0muS3LyxMjWyKjtQUl+AHyurT+l7fMnSb7Yntgn9n2PJKcn+WmSc9sI35dHLn9gkrOSXJ/kW0metTqdnW4/SZ6c5OuthiuSHDWy6Rfb3xsnRrYnjxRPHq1uI+JvSHIOQ2i5X5LNk7w7ydVJftj6uQEz0Pb9kiTLk/wsyTFJdkrylVbzyUnu3NrukeTKJH/fpjSsSPK8kX1tnuQ9SVa20cnXTIT8Nup8TpK3Jbke+DDwTuDRre83rur6GrkuliT5Qavh8JHLN2i1fbf15bw2eryq2+hJSS5r2/wwySvHXE97AWcB9271ntjW/3mSS5Pc2G6bB41ssyLJq9r9++ZpQuw/Aq+b6vL2uPnvdowLk+wx6Rh7jSz/7v4zzWNmVTW/MslFGR5TH05ylynqfh5welV9sapuAl4LPC3JZlO0l7QaDNHSHcsLgPe3075JtgWoqq8CNwNPGGn7F8AH2vmXAQcAj2cYnbsB+LdJ+3488CBg37b8X8DOwD2B89sxJ/xbO969GML8aKC/G0Mo+kDb9rnAOzISwmdiBvu5uV0fWwBPBl6c5IB22ePa3y06R7afDxwCbAZcDpwE/Ab4Q+ARwD7AwR3d2A94JPAo4P8AxzEEpB2Bh7Q+TbgXsDWwPcP1eVySB7TL/hXYHLgfw+30AuD/G9l2d+B7DNfTXwKHAl9pfd+itZnu+prwGOABwJ7AESMh8BWt1icBdwdeCNwyg9vo3cBfVdVmrb+fm3wFVdVngScCV7V6D0xyf+CDwMuBbYBPA6dPvOhontv6sUVV/WbyfpuPAT8FDpx8QZLtgU8Brwe2Al4JfDTJNlPsa5zfPWZmWPOzGO4TfwA8bFxdzYOBCycWquq7wK+A+3fUJmkVDNHSHUSSxzC87X1yVZ0HfJchKE/4IC2UtRGrJ7V1AH8FHF5VV1bVL4GjgGdMGqE7qqpurqqfA1TVCVX1s5H2D28johsATweOrKpbquoyhrA54X8BK6rqP6vqN1V1PvBR4BnTdO9f2ujdjUnOn8l+qursqrq4qn5bVRe1vj5+Rlfm1E6sqktbKNuKIdy9vF0v1wBvo296wJuq6qdVdSlwCXBmVX2vqn7C8CLlEZPav7aqfllVX2AIeM9q1/ezgVe322MF8FaGwD/hqqr613Y9/XxcITO8vl5XVT+vqgsZQtzD2/qDgddU1bdqcGFVXceqb+tfA7skuXtV3dAun4lnA5+qqrOq6tfAW4BNgD8ZafMvVXXFVP2d6DbDKO4R+f3PEPwl8Omq+nS7Ts4CljE8bmZq9DEz05qvqqrrgdOBXafY76bATyat+wnDiztJc8QQLd1xLGEIYde25Q8wMgLclp/WwsLTgPOr6vJ22X2Bj08EVeAbwK0M8y0nXDFxpr19f2x7+/6nwIp20dYMo2wbjrafdP6+wO4jofhGhtHX6T4w9rKq2qKd/mgm+0mye5LPtykOP2EYfd16mmPMxOR+bARcPXL8dzGMuM7Uj0fO/3zM8qYjyzdU1c0jy5czvGuwNXDntjx62fZT1D3WDK+v0W/GuGWkvh0ZXrRNtqrb+ukMofTyJF/IzD8wem9G+ltVv2XoY1ef27afBn7A8A7D5NqfOan2xwDbzbDGyTXMpOaprt/JbmIY8R91d+BnHbVJWoV18QMkkjol2YThreANcttXgG0MbJHk4W1k8LIklzOMno5O5YDhyfyFVXXOmH0vamdrZPVfAPsDezEE6M0ZpoAEWMkwxWEH4Nut/Y6TjvWFqtp7tTo78/18AHg78MSq+kWSf+K2UFhj2t8M3HVkeVyoH93uCuCXwNbTTBeYS1smudtIkL4Pw+j1tQwjuvcFLhu57Icj207u77j+T3d9rcoVwE6tnsnrp7yNqupcYP8kGwEvBU7m9veVqVwFPHRiIUnadtP1eTqvAT7E7z8m3ltVL5pim977y0xqnqlLue1dAJLcj+Hx/u0pt5DUzZFo6Y7hAIaR410Y3gLelWEu5pcY5rlO+ADD/OfHAaeMrH8n8IYk9wVIsk2S/ac53mYMAfI6hiDxxokLqupWhrmmRyW5a5IHTqrhk8D9kzw/yUbt9MejH7KaoVXtZzPg+hYId+P2U1tWAr9lmEM84QLgcUnuk2Rz4NXTHbyqrgbOBN6a5O4ZPpy5U5LZThmZzuuS3DnJYxmmSpzSru+TGW6/zdpt+Apguq/T+zGww6T5uNNdX6tyPHBMkp0zeFiSezDNbdT68bwkm7fpDT9luA/PxMnAk5Ps2QL43zLcH/+7o+bfqaqzgYu5/Ts37wOekmTf9s7LXTJ8wHOHdvkFwHNanxYz/XSkua75/a22x7Z550cDH6sqR6KlOWSIlu4YlgD/WVU/qKofTZwYRhafNzK3+YPAHsDnRqZ9APwzcBpwZpKfMXzDx+7THO89DG9N/5Bh9POrky5/KcPo9I+A97bj/hKgPdHvwzB3+KrW5k0MI2kzNoP9vAQ4uvXnCIYQM7HtLcAbgHPaW/WPanNePwxcBJzHEABX5QUMUykuYxiJ/wh9b/f3+FE7xlUMIerQqvpmu+yvGUZGvwd8meHF0gnT7OtzDKOZP0oycT+Y8vqagf/b2p/JEIbfDWwyg9vo+cCKNiXoUIZ5yKtUVd9qbf+VYST+KcBTqupXHTVP9hqGee4Tx7iC4d2Wv2d40XUF8Hfc9rz6WobR9xuA13H7Uex5rbnNoT+U4X5wDcMLoJf07kfS9FLV846WJM29JG8C7lVVS1bZWL8nw1erva+qdlhVW0nS3HAkWtIal+G7gR/W3trfDTgI+PhC1yVJ0kz5wUJJC2Ezhikc92Z4u/mtwKkLWpEkSR2cziFJkiR1cjqHJEmS1MkQLUmSJHVaJ+dEb7311rVo0aKFLkOSJEnrsfPOO+/aqtpm3GXrZIhetGgRy5YtW+gyJEmStB5rv+Q7ltM5JEmSpE6GaEmSJKmTIVqSJEnqZIiWJEmSOhmiJUmSpE6GaEmSJKmTIVqSJEnqZIiWJEmSOhmiJUmSpE6GaEmSJKmTIVqSJEnqZIiWJEmSOhmiJUmSpE4bLnQBkiRJ65JFh31qoUtYpRXHPnmhS1jvORItSZIkdXIkWpLuwBxRk6TV40i0JEmS1MkQLUmSJHUyREuSJEmdDNGSJElSJ0O0JEmS1MkQLUmSJHXyK+60XlgXvqYLZv5VXetCf/zaMUnSHZkj0ZIkSVKnORmJTrIf8M/ABsDxVXXspMs3Bt4DPBK4Dnh2Va1ol70aOAi4FXhZVZ0xFzVJ0nzwXQJp9fjY0fpm1iPRSTYA/g14IrAL8Nwku0xqdhBwQ1X9IfA24E1t212A5wAPBvYD3tH2J0mSJK215mI6x27Ad6rqe1X1K+BDwP6T2uwPnNTOfwTYM0na+g9V1S+r6vvAd9r+JEmSpLVWqmp2O0ieAexXVQe35ecDu1fVS0faXNLaXNmWvwvsDhwFfLWq3tfWvxv4r6r6yJjjHAIcAnCf+9znkZdffvms6l4d68JbUeCH1yTdMa0L/9PA/9Fa+3hfm1qS86pq8bjL5mJOdMasm5zMp2ozk22HlVXHAccBLF68eHbJX5K03jHQSVqT5mI6x5XAjiPLOwBXTdUmyYbA5sD1M9xWkiRJWqvMRYg+F9g5yR8kuTPDBwVPm9TmNGBJO/8M4HM1zCM5DXhOko2T/AGwM/C1OahJkiRJmjezns5RVb9J8lLgDIavuDuhqi5NcjSwrKpOA94NvDfJdxhGoJ/Ttr00ycnAZcBvgP9dVbfOtiZJkiRpPs3J90RX1aeBT09ad8TI+V8Az5xi2zcAb5iLOiRJkqQ1wV8slCRJkjoZoiVJkqROhmhJkiSp05zMiZYkSXPL772W1m6G6Dsw/0FLkiStHqdzSJIkSZ0M0ZIkSVInQ7QkSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVInQ7QkSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVInQ7QkSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVInQ7QkSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVInQ7QkSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVInQ7QkSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVInQ7QkSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVKnDWezcZKtgA8Di4AVwLOq6oYx7ZYAr2mLr6+qk5LcFTgF2Am4FTi9qg6bTT2SJEnqs+LYJy90Ceuk2Y5EHwYsraqdgaVt+XZa0D4S2B3YDTgyyZbt4rdU1QOBRwB/muSJs6xHkiRJmnezDdH7Aye18ycBB4xpsy9wVlVd30apzwL2q6pbqurzAFX1K+B8YIdZ1iNJkiTNu9mG6G2r6mqA9veeY9psD1wxsnxlW/c7SbYAnsIwmj1WkkOSLEuybOXKlbMsW5IkSVp9q5wTneSzwL3GXHT4DI+RMetqZP8bAh8E/qWqvjfVTqrqOOA4gMWLF9dU7SRJkqT5tsoQXVV7TXVZkh8n2a6qrk6yHXDNmGZXAnuMLO8AnD2yfBywvKr+aUYVS5IkSQtsttM5TgOWtPNLgFPHtDkD2CfJlu0Dhfu0dSR5PbA58PJZ1iFJkiStMbMN0ccCeydZDuzdlkmyOMnxAFV1PXAMcG47HV1V1yfZgWFKyC7A+UkuSHLwLOuRJEmS5t2svie6qq4D9hyzfhlw8MjyCcAJk9pcyfj50pIkSdJazV8slCRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSp06xDdJKtkpyVZHn7u+UU7Za0NsuTLBlz+WlJLpltPZIkSdJ8m4uR6MOApVW1M7C0Ld9Okq2AI4Hdgd2AI0fDdpKnATfNQS2SJEnSvJuLEL0/cFI7fxJwwJg2+wJnVdX1VXUDcBawH0CSTYFXAK+fg1okSZKkeTcXIXrbqroaoP2955g22wNXjCxf2dYBHAO8FbhluoMkOSTJsiTLVq5cOfuqJUmSpNW04UwaJfkscK8xFx0+w+NkzLpKsivwh1X1/ydZNN0Oquo44DiAxYsX1wyPK0mSJM25GYXoqtprqsuS/DjJdlV1dZLtgGvGNLsS2GNkeQfgbODRwCOTrGi13DPJ2VW1B5IkSdJaai6mc5wGTHzbxhLg1DFtzgD2SbJl+0DhPsAZVfXvVXXvqloEPAb4tgFakiRJa7u5CNHHAnsnWQ7s3ZZJsjjJ8QBVdT3D3Odz2+notk6SJEla58xoOsd0quo6YM8x65cBB48snwCcMM1+VgAPmW09kiRJ0nzzFwslSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSeo0qxCdZKskZyVZ3v5uOUW7Ja3N8iRLRtbfOclxSb6d5JtJnj6beiRJkqQ1YbYj0YcBS6tqZ2BpW76dJFsBRwK7A7sBR46E7cOBa6rq/sAuwBdmWY8kSZI072YbovcHTmrnTwIOGNNmX+Csqrq+qm4AzgL2a5e9EPgHgKr6bVVdO8t6JEmSpHk32xC9bVVdDdD+3nNMm+2BK0aWrwS2T7JFWz4myflJTkmy7VQHSnJIkmVJlq1cuXKWZUuSJEmrb5UhOslnk1wy5rT/DI+RMesK2BDYATinqv4I+Arwlql2UlXHVdXiqlq8zTbbzPDQkiRJ0tzbcFUNqmqvqS5L8uMk21XV1Um2A64Z0+xKYI+R5R2As4HrgFuAj7f1pwAHzaxsSZIkaeHMdjrHacDEt20sAU4d0+YMYJ8kW7YPFO4DnFFVBZzObQF7T+CyWdYjSZIkzbvZhuhjgb2TLAf2bsskWZzkeICquh44Bji3nY5u6wBeBRyV5CLg+cDfzrIeSZIkad6tcjrHdKrqOoYR5MnrlwEHjyyfAJwwpt3lwONmU4MkSZK0pvmLhZIkSVInQ7QkSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVInQ7QkSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVInQ7QkSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVInQ7QkSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVInQ7QkSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVInQ7QkSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVInQ7QkSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVInQ7QkSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVInQ7QkSZLUyRAtSZIkdZp1iE6yVZKzkixvf7ecot2S1mZ5kiUj65+b5OIkFyX5TJKtZ1uTJEmSNJ/mYiT6MGBpVe0MLG3Lt5NkK+BIYHdgN+DIJFsm2RD4Z+DPquphwEXAS+egJkmSJGnezEWI3h84qZ0/CThgTJt9gbOq6vqqugE4C9gPSDvdLUmAuwNXzUFNkiRJ0ryZixC9bVVdDdD+3nNMm+2BK0aWrwS2r6pfAy8GLmYIz7sA7x53kCSHJFmWZNnKlSvnoGxJkiRp9cwoRCf5bJJLxpz2n+FxMmZdJdmIIUQ/Arg3w3SOV4/bQVUdV1WLq2rxNttsM8PDSpIkSXNvw5k0qqq9prosyY+TbFdVVyfZDrhmTLMrgT1GlncAzgZ2bfv/btvXyYyZUy1JkiStTeZiOsdpwMS3bSwBTh3T5gxgn/Zhwi2Bfdq6HwK7JJkYWt4b+MYc1CRJkiTNmxmNRK/CscDJSQ4CfgA8EyDJYuDQqjq4qq5Pcgxwbtvm6Kq6vrV7HfDFJL8GLgcOnIOaJEmSpHkz6xBdVdcBe45Zvww4eGT5BOCEMe3eCbxztnVIkiRJa4q/WChJkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUqdZhegkWyU5K8ny9nfLKdp9JsmNST45af0fJPmftv2Hk9x5NvVIkiRJa8JsR6IPA5ZW1c7A0rY8zpuB549Z/ybgbW37G4CDZlmPJEmSNO9mG6L3B05q508CDhjXqKqWAj8bXZckwBOAj6xqe0mSJGltMtsQvW1VXQ3Q/t6zY9t7ADdW1W/a8pXA9lM1TnJIkmVJlq1cuXK1C5YkSZJma8NVNUjyWeBeYy46fJbHzph1NVXjqjoOOA5g8eLFU7aTJEmS5tsqQ3RV7TXVZUl+nGS7qro6yXbANR3HvhbYIsmGbTR6B+Cqju0lSZKkBTHb6RynAUva+SXAqTPdsKoK+DzwjNXZXpIkSVoosw3RxwJ7J1kO7N2WSbI4yfETjZJ8CTgF2DPJlUn2bRe9CnhFku8wzJF+9yzrkSRJkubdKqdzTKeqrgP2HLN+GXDwyPJjp9j+e8Bus6lBkiRJWtP8xUJJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjrNKkQn2SrJWUmWt79bTtHuM0luTPLJSevfn+RbSS5JckKSjWZTjyRJkrQmzHYk+jBgaVXtDCxty+O8GXj+mPXvBx4IPBTYBDh4lvVIkiRJ8262IXp/4KR2/iTggHGNqmop8LMx6z9dDfA1YIdZ1iNJkiTNu9mG6G2r6mqA9veeq7OTNo3j+cBnpmlzSJJlSZatXLlytYqVJEmS5sKGq2qQ5LPAvcZcdPgc1vEO4ItV9aWpGlTVccBxAIsXL645PLYkSZLUZZUhuqr2muqyJD9Osl1VXZ1kO+Ca3gKSHAlsA/xV77aSJEnSQpjtdI7TgCXt/BLg1J6NkxwM7As8t6p+O8taJEmSpDVitiH6WGDvJMuBvdsySRYnOX6iUZIvAacAeya5Msm+7aJ3AtsCX0lyQZIjZlmPJEmSNO9WOZ1jOlV1HbDnmPXLGPm6uqp67BTbz+r4kiRJ0kLwFwslSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6bbjQBaxLVhz75IUuQZIkSWsBR6IlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqVOqaqFr6JZkJXD5QtcxR7YGrl3oIubI+tQXsD9rs/WpL7B+9Wd96gvYn7XZ+tQXWL/6sz715b5Vtc24C9bJEL0+SbKsqhYvdB1zYX3qC9iftdn61BdYv/qzPvUF7M/abH3qC6xf/Vmf+jIdp3NIkiRJnQzRkiRJUidD9MI7bqELmEPrU1/A/qzN1qe+wPrVn/WpL2B/1mbrU19g/erP+tSXKTknWpIkSerkSLQkSZLUyRCtLkmOT7JLO3/TQtezkJIsSnLJQtcBkORlSb6R5P0LXctMzHe9SY5K8sr52Pd8S7JHkk8udB2SpOkZotWlqg6uqssWuo41KcmGC13DDLwEeFJVPW9VDdeS/sy4Xq09khyY5O2z3MeKJFvPsO29knwoyXeTXJbk00nuP5vjj+x7TfdlhySnJlme5HtJ3p5k49kcf9L+13R/Dk9yaZKLklyQZPdZHnveX/gmOSHJNWti8GO++5NkxySfb4MRlyb5m/k6VjvefPfnLkm+luTC1p/Xzdex5pIhep4l+USS89qd4pC27qAk305ydpL/mPjHl2SbJB9Ncm47/ekC1363JJ9qd+pLkjy71bx4pM1bk5yfZGmSbdq6l7UnvIuSfKitOyrJe5N8rj2JvGgN92VRkm8mOanV9ZEkd01yRLuuL0lyXJK09mcneWOSLwB/k2TbJB9v18WFSf6k7XqDdhtemuTMJJusyX61Wt8J3A84Lcmrkvx3kq+3vw9obQ5MckqS04Ez27q/a32/aE3+w5pU7+Htie3cVvP+I/V+IsnpSb6f5KVJXtHafDXJVq3di9q2F7bHzl3HHG+nJJ9pj8MvJXngGujjxP3t+Hbfen+SvZKc0+7/u7XT791Wk/Zzt3HXz7oqyQYdbQN8HDi7qnaqql2Avwe2na/6eqxGXz4GfKKqdgZ2BjYB/nGeyuvW2Z9HA/8L+KOqehiwF3DFfNU2h04E9lvoIubIb4C/raoHAY8C/nfau8TrqF8CT6iqhwO7AvsledQC17RqVeVpHk/AVu3vJsAlwPbACmArYCPgS8DbW5sPAI9p5+8DfGOBa3868B8jy5sDZwOL23IBz2vnjxjpx1XAxu38Fu3vUcCF7XrYmuEf7r3XYF8WtXr/tC2fALxy4vZp694LPKWdPxt4x8hlHwZe3s5v0K6LRQz/yHZt608G/nKBbqsV7Xq9O7BhW7cX8NF2/kDgypH74z4Mn54Ow4vpTwKPW4B63zhxnQFbAN8G7tbq/Q6wGbAN8BPg0NbubSO3xT1G9vl64K9H7m+vbOeXAju380VFupAAAAoBSURBVLsDn1tD97ffAA9t1+957T4XYH/gE9PcVnsAn2znx14/c1DfXwJfAy4A3tXu0zcBb2q1fhbYrT0Ovgf8+cj96FTgM8C3gCNH9vmJtu2lwCEj628Cjgb+B3jMyG2/SdvPi6ao8QnAF9eTvuw5uS/t9r8B2HQd7M/TgNNneR98AXARw/PCe7n9Y/ZFwLntso8Cd23rn8nwPHrhxPUJPHjk+rqI9lhfxWPzknl4zC9If0aOfyqw9/rQH+CuwPnA7nN9O8357b7QBazvJ24LjxcyBIHDgJNGLn8Zt4XPa9odbeL0Q2CzBaz9/sD3Gf55P7atO5vbQvSt3BYC7gdc0M5/BvgIw5PBpiPXw9Ej+34PcMAa7Msi4Acjy09geGJ5OsMTyMXt+j5spJ+PH2m/kvbCYNI+l48svwp4zQLdVisYnvx2ZBi9u6T16Zvt8gOB/xxp/5a2zcR97TvAQQtQ77JW60QdPwAe1OodfQH3A2D7dv6FwD+1849neCF6cbuvvnPk/vZKYFPg55MeV/P+4nTMfeM93PaC836tjqluqz24LUSPvX5mWduDgNOBjdryOxieMAt4Ylv3cYZ3LDYCHs5tj+0DgauBe3DbwMDE/4PJAwb3aMsFPGvSbb+IIQy+YJo6Xwa8bX3uC/B12ovwdaw/mzLcH7/danx8533wwQxBf+uJ+rh9SJvqxfHF3PZ/YGKA5l+57bF1Z2CTGTw25zREL2R/Rvr0A+Du63J/GF4wXkB70TiXt9F8ndaGuZHrrSR7MIwwPbqqbklyNsMd80FTbHKn1vbna6bC6VXVt5M8EngS8A9JzlzVJu3vk4HHAX8OvDbJgyddPrn9mjLu+O9geKK5IslRwF1GLr95Bvv85cj5WxmepBbSMcDnq+qpSRYxvBiYMNqfAP9QVe9ac6WNFeDpVfWt260c5leOXre/HVn+Lfzuf9eJDC/GLkxyIEMAHXUn4Maq2nVuy56RVdU/3W01Yez1M0t7Ao8Ezm2zlzZheAH/K4YXwDA8Gf6yqn6d5GKGJ+kJZ1XVdQBJPsYwgrkMeFmSp7Y2OzJMWbiO4XHx0Uk1nAr8Y1XN9oOl60pfwvj/d1kX+1NVN7XnhscCfwZ8OMlhVXXiVNtM8gTgI1V1bdvf9a2/Ex6S5PUM775sCpzR1p8DnJjkZIbpMQBfAQ5PsgPwsapaPsMa5tKC9SfJpgy34cur6qfrcn+q6lZg1yRbAB9P8pCqWis+vD8V50TPr82BG1qAfiDDvKW7Ao9PsmWGD3g9faT9mcBLJxaSLMQT/+8kuTdwS1W9j2Hk8o8mNbkT8Ix2/i+ALye5E7BjVX0e+D/c9iAD2D/DhwfuwRB2zp3nLkx2nzaXD+C5wJfb+WvbP6JnjN8MGKYEvBiGuYNJ7j5/Zc7K5gwj6jCMTE3lDOCFrd8k2T7JPee5tqnq+Os2Z5Qkj+jcfjPg6iQbAb/3IcX2pPL9JM9s+0+Sh8+y5rkyk9tqttfPOGF4N2zXdnpAVR0F/LracBAjob+qRl+0wJgXo5MGDB7OMMI68YL0F+3JcdQ5wBMn+jWFSxkC5frSl8WjK9r/kG0ZBlbWtf5QVbdW1dlVdSTD89bTp2s/yVQvKiacCLy0qh4KvG6i3qo6FHgNwwuBC5Lco6o+wDBg83PgjCRP6KhjrixIf9r/vY8C76+qj03VbjUs6O1TVTcyDCqs9fPXDdHz6zPAhkkuYhh1+irDk+YbGaYQfBa4jGGaBwxv+S3O8EGvy4BD13zJt/NQ4GtJLgAOZ3jbZtTNwIOTnMfwyvVohrdj3tdGSL7O8Bbmja3914BPMVwPx1TVVWugD6O+ASxpt8dWwL8D/8EwsvMJpg/1fwP8WevXeQxvd62N/pHhXYNzGG6LsarqTIY5+F9pffoIQyBd045heFv6ogyfmD+mc/vXMjyWzgK+OUWb5wEHJbmQIcysLR/Om8ltNdvrZ5ylwDMmXjQl2SrJfTu237ttswlwAEPoGjdgMJ0jGEZC3zFNm88BG2fkQ8hJ/jjJ49fBviwF7prkBa3ODYC3Mkzl+/mkdmt9f5I8IMnOI6t2BS7vqHMp8Kw2oELaB4VHjH1xnGSnqvqfqjoCuBbYMcn9gO9V1b8ApwEP66hjrqzx/rQXOe9mmJ72f9eD/mzTRqBp99+9mPp/+tpjoeaR3JFP3DZPeEOG+W9PXeia1kCfj6LNp1qg4y9iHj5M4snTungCns1tH/Q5jyFY3TRy+e0erxOXMYyYn8zwYvh3H14DNgb+q+3vFIZRpD1Gtx3Z1wqG+fAB/pNh6sBUdd67He+7DC+APsWkDyatQ33ZkSFELAduBN61rt42DO8Q/DfDINBFDG/db915H1zCbR9CO5Hbz7l9McNnHM5mmFN7Ylv/MYZBj0uAf251vrrdNy5gGLjaappjfpBh3vivGT5oPWefA1nT/WGYqlPt+p/4zMST1uH+PIxh4O2itv0Rc9WX+Tz5s98LIMlbGF5l3YVhCsff1Hp+Q2SYb3xTVb1lgY6/iOHDWg9ZiONL0oQMX5H5QeBpVXXeQtcjafUYoiVJkqROfjuHJGnBtHmXS8dctGe1b5pYV6xPfYF1sz/rYs3TsT9rN0eiJUmSpE5+O4ckSZLUyRAtSZIkdTJES9JaKMmtSS4YOS1ajX1skeQlc1+dJMk50ZK0FkpyU1VtuuqW0+5jEavx1Y5JNqjf/yU7SdIIR6IlaR3RfnL+zUnObb9s+ldt/aZJliY5P8nFSSZ+lfFYYKc2kv3mJHsk+eTI/t6e5MB2fkWSI5J8GXhmkp2SfCbJeUm+1H7tjiTPTHJJkguTfHHNXgOStPbwK+4kae20SZIL2vnvV9VTgYOAn1TVHyfZGDgnyZnAFQy/fPrTJFsDX01yGnAY8JCq2hUgyR6rOOYvquoxre1S4NCqWp5kd4afgX4Cw89C71tVP5z4mV5JuiMyREvS2unnE+F3xD7Aw5I8oy1vDuzM8BPGb0zyOOC3wPbAtqtxzA/DMLIN/AlwSpKJyzZuf88BTkxyMsPP/ErSHZIhWpLWHQH+uqrOuN3KYUrGNsAjq+rXSVYAdxmz/W+4/TS+yW1ubn/vBNw4JsRTVYe2keknAxck2XVd/JEESZot50RL0rrjDODFSTYCSHL/JHdjGJG+pgXoPwPu29r/DNhsZPvLgV2SbJxkc2DPcQepqp8C30/yzHacJHl4O79TVf1PVR0BXAvsOPfdlKS1nyPRkrTuOB5YBJyfYZ7FSuAA4P3A6UmWARcA3wSoquuSnJPkEuC/qurv2jSMi4DlwNenOdbzgH9P8hpgI+BDwIXAm5PszDAqvrStk6Q7HL/iTpIkSerkdA5JkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqRO/w+14Opc/9oYTQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "visualize_importances(feature_names, neuron_cond_vals_0.mean(dim=0).detach().numpy(), title=\"Average Feature Importances for Neuron 0\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "From the data above, it appears that the primary input feature used by neuron 0 is age, with limited importance for all other features." + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Feature Importances for Neuron 10\n", + "age : -0.012\n", + "sibsp : 0.003\n", + "parch : 0.000\n", + "fare : -0.015\n", + "female : 0.055\n", + "male : -0.210\n", + "embark_C : 0.016\n", + "embark_Q : -0.006\n", + "embark_S : -0.052\n", + "class_1 : 0.022\n", + "class_2 : 0.009\n", + "class_3 : -0.116\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtEAAAGECAYAAADwYnzeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3debhkVX3u8e8rDYgyyNAiMtjqxQETJdrBJNeBMDgmQSMOCdHmBiXGqyaPMRGjIo7BmMTcxOtVnMAZHMEhCrZilDjQaDM6gIiCoMyOSER/94+9jhTHM62uM/Q5fD/PU8/Zw9p7r7Wr6tRbq9auSlUhSZIkae5us9QVkCRJkpYbQ7QkSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVInQ7QkSZLUyRAtSVp2krw8ydVJvrfUdZF062SIlla4JKcnuS7J1ktdl/nQ2vOzJD8euf3umPtck6SSrJqves7hmMckecdiHW8mSQ5P8rmlrsdcJdkT+Btgn6q60zzts5Kcm+Q2I8tenuT4+dj/QkqyW5JTklze2rFm0vqtk7wlyQ+TfC/Jc5amptLKYoiWVrD2YvpgoIA/WqBjLFrwHPHMqtp25Pb5JajDr2SwLP+fLtH9N667ANdU1ZW9G87S3jsDT9rkWs1PHTbFL4GPA4+bZv0xwN4M5+33gb9L8oh5roN0q7Ms/+lLmrOnAF8AjgfWTSxM8jutR2qLkWWPTXJOm75NkqOSfDPJNUlOSrJTWzfRa3tEku8An2rL39v2+YMk/5nkPiP73jnJh1tP2Jmth+9zI+vvleS0JNcm+XqSJ2xKY2faT5JHJ/lKq8OlSY4Z2fQ/29/rJ3q2J/cUT+6tbj3ir0hyBvBT4G5Jdkjy5iRXJPlua+cWzEHb9zOSXJjkR0leluTuST7f6nxSkq1a2f2TXJbk79uQhkuSHDayrx2SvC3JVUm+neSFEyG/9TqfkeQ1Sa4FTgReD/xua/v1s52vkXOxLsl3Wh1eMLJ+i1a3b7a2nJWh93i2++hRSS5o23w3yXOnOE8HAacBd271Pb4t/6Mk5ye5vt039x7Z5pIkz2uP75/MEGL/EXjJdOvb8+a/2jHOTrL/pGMcNDL/q8fPDM+Z2er83CTnZHhOnZjktlPVq6q+X1WvA86cpl1PAV5WVddV1VeBNwKHT1NW0lxVlTdv3lboDbgIeAbwAODnwK4j674JHDwy/17gqDb91wzhew9ga+ANwLvbujUMPdtvA24PbNOW/zmwXSv/r8DGkX2/p91uB+wDXAp8rq27fZv/X8Aq4P7A1cB9pmnT6cBTp1g+436A/YHfZOg8uC/wfeAxk9q0amR/xwDvGJm/RZlWj+8A92nH2xL4UDtXtwfuCHwJ+Itp2jF5/wWcAmzf9nkjsB64G7ADcAGwbqQtNwH/0s73Q4GfAPds698GnNzujzXAN4Aj2rrD27bPavXepi373KT6zeV8vbFtf79W33u39X8LnAvcE0hbv/Mc7qMrgAe36R2B+09z7vYHLhuZv0dr/8Htfvg7hsf+Vm39JcBGYE/a43WKfRZDb+1ZtMcX8HLg+Da9O3AN8Kh2Tg5u86tHjnHQVPcvUzxn5ljnLzH0ju8EfBV4+izP91XtOGtGlu3Ylo0+9w8Fzl3q/0/evC33mz3R0gqV5EEMH9+eVFVnMYTmPx0p8m7gT1rZ7RjCwbvbur8AXlBVl1XVjQyB4NBJPXTHVNVPquoGgKp6S1X9aKT8/VqP6BYMHzO/uKp+WlUXACeM7OcPgEuq6q1VdVNVfRl4P8ML/XT+rfXeXZ/ky3PZT1WdXlXnVtUvq+qc1taHzulkTu/4qjq/qm5iCDqPBP66nZcrgdfQNzzgVVX1w6o6HzgPOLWqLq6qHwD/AfzWpPIvqqobq+ozwEeBJ7Tz/UTg+e3+uAT4Z+DJI9tdXlX/3s7TDVNVZI7n6yVVdUNVnQ2czRCWAZ4KvLCqvl6Ds6vqGma/r38O7JNk+xp6Tb/M3DwR+GhVnVZVPwf+iSGo/t5ImX+rqkuna+9Es4EXAUfn168h+DPgY1X1sXZOTgM2MDxv5mr0OTPXOl9eVdcCHwb27TjWhG3b3x+MLPsBwxssSWMwREsr1zqGEHZ1m38XI0M62vwft7Dwx8CXq+rbbd1dgA9OBFWGXrBfALuObH/pxET7+P7Y9vH9Dxl60QB2AVYz9JBdOtW27VgPHAnF1wOHATNdMPbsqrpDu91/LvtJ8sAkn25DHH4APL3VbxyT27ElcMXI8d/A0CM9V98fmb5hivltR+avq6qfjMx/m6HXchdgqzY/um73aeo9pTmer9FvxvjpSP32ZHjTNtls9/XjGELpt5N8JnO/YPTOjLS3qn7J0MauNrdtP8bwCcORU9T98ZPq/iBgtznWcXId5lLn6c5vjx+3v9uPLNse+NEm7EvSiOV4QYmkWSTZBngCsEVu/gqwrYE7JLlf6xm8IMm3GXpP/5QhVE+4FPjzqjpjin2vaZM1svhPgUOAgxgC9A7AdQwf5V/FMHxgD4ZhBTCErNFjfaaqDt6kxs59P+8CXgs8sqp+luRfuTkU1hTlf8Iw/GTCVKF+dLtLGYY07NJ6phfajkluPxKk92Lovb6aoUf3LgxDQCbWfXdk28ntnar9M52v2VwK3L3VZ/Lyae+jqjoTOCTJlsAzgZO45WNlOpczDD0Bhgs923YztXkmL2QYfjT5OfH2qnraNNv0Pl7mUuexVdV1Sa5g+JTgtLb4fsD583kc6dbInmhpZXoMQ8/xPgwfAe8L3Bv4LMNFRhPeBTwbeAjDmOgJrwdekeQuAElWJzlkhuNtxxAgr2EIEq+cWFFVvwA+AByT5HZJ7jWpDh8B7pHkyUm2bLffHr3Iao5m2892wLUtEO7HLYe2XMXwDQd3G1m2EXhIkr2S7AA8f6aDV9UVwKnAPyfZPsPFmXdPMu6QkZm8JMlWSR7MMFTive18n8Rw/23X7sPnADN9nd73gT3SLlxsZjpfs3kT8LIke2dw3yQ7M8N91NpxWJId2vCGHzI8hufiJODRSQ5sAfxvGB6P/9VR51+pqtMZxnSPfnLzDuAPkzy8ffJy2wwXeO7R1m8EntTatJaZhyPNe53bRYcTQ1C2nnQR4tuAFybZsT3/nsZwsbGkMRiipZVpHfDWqvpOVX1v4sbQs3jYyNjmdzNcpPWpkWEfAP+H4SK3U5P8iOEiwwfOcLy3MXw0/V2G3s8vTFr/TIbe6e8Bb2/HvRGgqn4EPIxh7PDlrcyruDkQzMkc9vMM4KWtPUczhJiJbX8KvAI4o31U/zttzOuJwDkMF5t9ZA7VeArDUIoLGHri30ffx/09vteOcTnwToaLzr7W1j2LoWf0YuBzDG+W3jLDvj7F0DP5vSQTj4Npz9cc/EsrfypDGH4zwwV9s91HTwYuaUOCns4wDnlWVfX1VvbfGXri/xD4w6r67446T/ZChnHuE8e4lOHTlr9neNN1KcMFlBOvoy9i6H2/DngJt+zFXow638DNQze+1uYnvJhheM23gc8Ar66qj2/icSQ1qer5hEuSxpfkVcCdqmrdrIX1azJ8tdo7qmqP2cpKkhaGPdGSFlyG7wa+b/tofz/gCOCDS10vSZI2lRcWSloM2zEM4bgzcCXDV66dvKQ1kiRpDA7nkCRJkjo5nEOSJEnqZIiWJEmSOi3LMdG77LJLrVmzZqmrIUmSpBXsrLPOurqqVk+1blmG6DVr1rBhw4alroYkSZJWsPbLvlNyOIckSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVInQ7QkSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVInQ7QkSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVInQ7QkSZLUadVSV0DSyrfmqI8udRVmdcmxj17qKkiSlhF7oiVJkqRO8xKikzwiydeTXJTkqCnWb53kxLb+i0nWtOVrktyQZGO7vX4+6iNJkiQtpLGHcyTZAvi/wMHAZcCZSU6pqgtGih0BXFdV/yPJk4BXAU9s675ZVfuOWw9JkiRpscxHT/R+wEVVdXFV/TfwHuCQSWUOAU5o0+8DDkySeTi2JEmStOjmI0TvDlw6Mn9ZWzZlmaq6CfgBsHNbd9ckX0nymSQPnu4gSY5MsiHJhquuumoeqi1JkiRtmvkI0VP1KNccy1wB7FVVvwU8B3hXku2nOkhVHVdVa6tq7erVq8eqsCRJkjSO+QjRlwF7jszvAVw+XZkkq4AdgGur6saqugagqs4CvgncYx7qJEmSJC2Y+QjRZwJ7J7lrkq2AJwGnTCpzCrCuTR8KfKqqKsnqdmEiSe4G7A1cPA91kiRJkhbM2N/OUVU3JXkm8AlgC+AtVXV+kpcCG6rqFODNwNuTXARcyxC0AR4CvDTJTcAvgKdX1bXj1kmSJElaSPPyi4VV9THgY5OWHT0y/TPg8VNs937g/fNRB0mSJGmx+IuFkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUqdVS10BSZK08q056qNLXYVZXXLso5e6ClpG7ImWJEmSOhmiJUmSpE6GaEmSJKmTIVqSJEnqZIiWJEmSOhmiJUmSpE5+xZ0k3Yr5tWOStGnsiZYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZPfEy1JktTB71cX2BMtSZIkdTNES5IkSZ0M0ZIkSVInQ7QkSZLUyRAtSZIkdfLbOW7FvLpYkiRp08xLT3SSRyT5epKLkhw1xfqtk5zY1n8xyZqRdc9vy7+e5OHzUR9JkiRpIY0dopNsAfxf4JHAPsCfJNlnUrEjgOuq6n8ArwFe1bbdB3gScB/gEcDr2v4kSZKkzdZ89ETvB1xUVRdX1X8D7wEOmVTmEOCENv0+4MAkacvfU1U3VtW3gIva/iRJkqTN1nyMid4duHRk/jLggdOVqaqbkvwA2Lkt/8KkbXef6iBJjgSOBNhrr73modr9lsMYYpj7OGLHG2++lsNjrefxs5Iea943m6/lcN/A3M/5cmjPrfWxtpLaAivvsbZY5qMnOlMsqzmWmcu2w8Kq46pqbVWtXb16dWcVJUmSpPkzHyH6MmDPkfk9gMunK5NkFbADcO0ct5UkSZI2K/MRos8E9k5y1yRbMVwoeMqkMqcA69r0ocCnqqra8ie1b++4K7A38KV5qJMkSZK0YMYeE93GOD8T+ASwBfCWqjo/yUuBDVV1CvBm4O1JLmLogX5S2/b8JCcBFwA3Af+7qn4xbp0kSZKkhTQvP7ZSVR8DPjZp2dEj0z8DHj/Ntq8AXjEf9ZAkSZIWgz/7LUmSJHXyZ78lSSvC5vgVWJJWLnuiJUmSpE6GaEmSJKmTIVqSJEnqZIiWJEmSOhmiJUmSpE6GaEmSJKmTX3HXwa9PkiRJEtgTLUmSJHUzREuSJEmdHM4hbYYcOiRJ0ubNnmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjqNFaKT7JTktCQXtr87TlNuXStzYZJ1I8tPT/L1JBvb7Y7j1EeSJElaDOP2RB8FrK+qvYH1bf4WkuwEvBh4ILAf8OJJYfuwqtq33a4csz6SJEnSghs3RB8CnNCmTwAeM0WZhwOnVdW1VXUdcBrwiDGPK0mSJC2ZcUP0rlV1BUD7O9VwjN2BS0fmL2vLJry1DeV4UZJMd6AkRybZkGTDVVddNWa1JUmSpE23arYCST4J3GmKVS+Y4zGmCsbV/h5WVd9Nsh3wfuDJwNum2klVHQccB7B27dqaqowkSZK0GGYN0VV10HTrknw/yW5VdUWS3YCpxjRfBuw/Mr8HcHrb93fb3x8leRfDmOkpQ7QkSbcmlxz76KWugqQZjDuc4xRg4ts21gEnT1HmE8DDkuzYLih8GPCJJKuS7AKQZEvgD4DzxqyPJEmStODGDdHHAgcnuRA4uM2TZG2SNwFU1bXAy4Az2+2lbdnWDGH6HGAj8F3gjWPWR5IkSVpwsw7nmElVXQMcOMXyDcBTR+bfArxlUpmfAA8Y5/iSJEnSUvAXCyVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6mSIliRJkjoZoiVJkqROhmhJkiSpkyFakiRJ6jRWiE6yU5LTklzY/u44TbmPJ7k+yUcmLb9rki+27U9MstU49ZEkSZIWw7g90UcB66tqb2B9m5/Kq4EnT7H8VcBr2vbXAUeMWR9JkiRpwY0bog8BTmjTJwCPmapQVa0HfjS6LEmAA4D3zba9JEmStDkZN0TvWlVXALS/d+zYdmfg+qq6qc1fBuw+XeEkRybZkGTDVVddtckVliRJksa1arYCST4J3GmKVS8Y89iZYllNV7iqjgOOA1i7du205SRJkqSFNmuIrqqDpluX5PtJdquqK5LsBlzZceyrgTskWdV6o/cALu/YXpIkSVoS4w7nOAVY16bXASfPdcOqKuDTwKGbsr0kSZK0VMYN0ccCBye5EDi4zZNkbZI3TRRK8lngvcCBSS5L8vC26nnAc5JcxDBG+s1j1keSJElacLMO55hJVV0DHDjF8g3AU0fmHzzN9hcD+41TB0mSJGmx+YuFkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnVYtdQUkSZK0dC459tFLXYVlyZ5oSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSepkiJYkSZI6GaIlSZKkToZoSZIkqZMhWpIkSeo0VohOslOS05Jc2P7uOE25jye5PslHJi0/Psm3kmxst33HqY8kSZK0GMbtiT4KWF9VewPr2/xUXg08eZp1f1tV+7bbxjHrI0mSJC24cUP0IcAJbfoE4DFTFaqq9cCPxjyWJEmStFkYN0TvWlVXALS/d9yEfbwiyTlJXpNk6+kKJTkyyYYkG6666qpNra8kSZI0tllDdJJPJjlvitsh83D85wP3An4b2Al43nQFq+q4qlpbVWtXr149D4eWJEmSNs2q2QpU1UHTrUvy/SS7VdUVSXYDruw5+EQvNnBjkrcCz+3ZXpIkSVoK4w7nOAVY16bXASf3bNyCN0nCMJ76vDHrI0mSJC24cUP0scDBSS4EDm7zJFmb5E0ThZJ8FngvcGCSy5I8vK16Z5JzgXOBXYCXj1kfSZIkacHNOpxjJlV1DXDgFMs3AE8dmX/wNNsfMM7xJUmSpKXgLxZKkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUqexQnSSnZKcluTC9nfHKcrsm+TzSc5Pck6SJ46su2uSL7btT0yy1Tj1kSRJkhbDuD3RRwHrq2pvYH2bn+ynwFOq6j7AI4B/TXKHtu5VwGva9tcBR4xZH0mSJGnBjRuiDwFOaNMnAI+ZXKCqvlFVF7bpy4ErgdVJAhwAvG+m7SVJkqTNzbghetequgKg/b3jTIWT7AdsBXwT2Bm4vqpuaqsvA3Yfsz6SJEnSgls1W4EknwTuNMWqF/QcKMluwNuBdVX1y9YTPVnNsP2RwJEAe+21V8+hJUmSpHk1a4iuqoOmW5fk+0l2q6orWki+cppy2wMfBV5YVV9oi68G7pBkVeuN3gO4fIZ6HAccB7B27dppw7YkSZK00MYdznEKsK5NrwNOnlygfePGB4G3VdV7J5ZXVQGfBg6daXtJkiRpczNuiD4WODjJhcDBbZ4ka5O8qZV5AvAQ4PAkG9tt37buecBzklzEMEb6zWPWR5IkSVpwsw7nmElVXQMcOMXyDcBT2/Q7gHdMs/3FwH7j1EGSJElabP5ioSRJktTJEC1JkiR1MkRLkiRJnQzRkiRJUidDtCRJktTJEC1JkiR1Gusr7iTp1uaSYx+91FWQJG0G7ImWJEmSOhmiJUmSpE6GaEmSJKmTIVqSJEnqZIiWJEmSOhmiJUmSpE6GaEmSJKmTIVqSJEnqZIiWJEmSOhmiJUmSpE6GaEmSJKmTIVqSJEnqZIiWJEmSOhmiJUmSpE6pqqWuQ7ckVwHfXup6zJNdgKuXuhLzZCW1BWzP5mwltQVWVntWUlvA9mzOVlJbYGW1ZyW15S5VtXqqFcsyRK8kSTZU1dqlrsd8WEltAduzOVtJbYGV1Z6V1BawPZuzldQWWFntWUltmYnDOSRJkqROhmhJkiSpkyF66R231BWYRyupLWB7NmcrqS2wstqzktoCtmdztpLaAiurPSupLdNyTLQkSZLUyZ5oSZIkqZMhWl2SvCnJPm36x0tdn6WUZE2S85a6HgBJnp3kq0neudR1mYuFrm+SY5I8dyH2vdCS7J/kI0tdD0nSzAzR6lJVT62qC5a6HospyaqlrsMcPAN4VFUdNlvBzaQ9c66vNh9JDk/y2jH3cUmSXeZY9k5J3pPkm0kuSPKxJPcY5/gj+17stuyR5OQkFya5OMlrk2w9zvEn7X+x2/OCJOcnOSfJxiQPHPPYC/7GN8lbkly5GJ0fC92eJHsm+XTrjDg/yV8t1LHa8Ra6PbdN8qUkZ7f2vGShjjWfDNELLMmHkpzVHhRHtmVHJPlGktOTvHHiH1+S1Unen+TMdvufS1z32yf5aHtQn5fkia3Oa0fK/HOSLydZn2R1W/bs9oJ3TpL3tGXHJHl7kk+1F5GnLXJb1iT5WpITWr3el+R2SY5u5/q8JMclSSt/epJXJvkM8FdJdk3ywXYuzk7ye23XW7T78PwkpybZZjHb1er6euBuwClJnpfkv5J8pf29ZytzeJL3JvkwcGpb9ret7ecs5j+sSfV9QXthO7PV+ZCR+n4oyYeTfCvJM5M8p5X5QpKdWrmntW3Pbs+d201xvLsn+Xh7Hn42yb0WoY0Tj7c3tcfWO5MclOSM9vjfr91+7b6atJ/bT3V+lqskW3SUDfBB4PSquntV7QP8PbDrQtWvxya05QPAh6pqb2BvYBvgHxeoet062/O7wB8A96+q+wIHAZcuVN3m0fHAI5a6EvPkJuBvqurewO8A/zvtU+Jl6kbggKq6H7Av8Igkv7PEdZpdVXlbwBuwU/u7DXAesDtwCbATsCXwWeC1rcy7gAe16b2Ary5x3R8HvHFkfgfgdGBtmy/gsDZ99Eg7Lge2btN3aH+PAc5u52EXhn+4d17Etqxp9f2fbf4twHMn7p+27O3AH7bp04HXjaw7EfjrNr1FOxdrGP6R7duWnwT82RLdV5e087o9sKotOwh4f5s+HLhs5PH4MIarp8PwZvojwEOWoL6vnDhnwB2AbwC3b/W9CNgOWA38AHh6K/eakfti55F9vhx41sjj7bltej2wd5t+IPCpRXq83QT8Zju/Z7XHXIBDgA/NcF/tD3ykTU95fuahfn8GfAnYCLyhPaZ/DLyq1fWTwH7teXAx8Ecjj6OTgY8DXwdePLLPD7VtzweOHFn+Y+ClwBeBB43c99u0/TxtmjoeAPznCmnLgZPb0u7/64Btl2F7/hj48JiPwacA5zC8LrydWz5nnwac2da9H7hdW/54htfRsyfOJ3CfkfN1Du25Pstz87wFeM4vSXtGjn8ycPBKaA9wO+DLwAPn+36a9/t9qSuw0m/cHB7PZggCRwEnjKx/NjeHzyvbA23i9l1guyWs+z2AbzH8835wW3Y6N4foX3BzCLgbsLFNfxx4H8OLwbYj5+GlI/t+G/CYRWzLGuA7I/MHMLywPI7hBeTcdr6PGmnnQ0VcMbMAAAjYSURBVEfKX0V7YzBpnxeOzD8PeOES3VeXMLz47cnQe3dea9PX2vrDgbeOlP+nts3EY+0i4IglqO+GVteJenwHuHer7+gbuO8Au7fpPwf+tU0/lOGN6Lntsfr6kcfbc4FtgRsmPa8W/M3pFI+Nt3HzG867tXpMd1/tz80hesrzM2bd7g18GNiyzb+O4QWzgEe2ZR9k+MRiS+B+3PzcPhy4AtiZmzsGJv4fTO4w2LnNF/CESff9GoYw+JQZ6vls4DUruS3AV2hvwpdZe7ZleDx+o9XxoZ2PwfswBP1dJurHLUPadG+Oz+Xm/wMTHTT/zs3Pra2Abebw3JzXEL2U7Rlp03eA7ZdzexjeMG6kvWmcz/tooW6bw9jIFSvJ/gw9TL9bVT9NcjrDA/Pe02xym1b2hsWp4cyq6htJHgA8CviHJKfOtkn7+2jgIcAfAS9Kcp9J6yeXXyxTHf91DC80lyY5BrjtyPqfzGGfN45M/4LhRWopvQz4dFU9NskahjcDE0bbE+AfquoNi1e1KQV4XFV9/RYLh/GVo+f2lyPzv4Rf/e86nuHN2NlJDmcIoKNuA1xfVfvOb7XnZLb6z3RfTZjy/IzpQOABwJlt9NI2DG/g/5vhDTAML4Y3VtXPk5zL8CI94bSqugYgyQcYejA3AM9O8thWZk+GIQvXMDwv3j+pDicD/1hV415YulzaEqb+f5fl2J6q+nF7bXgw8PvAiUmOqqrjp9tmkgOA91XV1W1/17b2TviNJC9n+PRlW+ATbfkZwPFJTmIYHgPweeAFSfYAPlBVF86xDvNpydqTZFuG+/Cvq+qHy7k9VfULYN8kdwA+mOQ3qmqzuHh/Oo6JXlg7ANe1AH0vhnFLtwMemmTHDBd4PW6k/KnAMydmkizFC/+vJLkz8NOqegdDz+X9JxW5DXBom/5T4HNJbgPsWVWfBv6Om59kAIdkuHhgZ4awc+YCN2GyvdpYPoA/AT7Xpq9u/4gOnXozYBgS8JcwjB1Msv3CVXMsOzD0qMPQMzWdTwB/3tpNkt2T3HGB6zZdPZ7VxoyS5Lc6t98OuCLJlsCvXaTYXlS+leTxbf9Jcr8x6zxf5nJfjXt+phKGT8P2bbd7VtUxwM+rdQcxEvqravRNC0zxZnRSh8H9GHpYJ96Q/qy9OI46A3jkRLumcT5DoFwpbVk7uqD9D9mVoWNlubWHqvpFVZ1eVS9meN163EzlJ5nuTcWE44FnVtVvAi+ZqG9VPR14IcMbgY1Jdq6qdzF02NwAfCLJAR31mC9L0p72f+/9wDur6gPTldsES3r/VNX1DJ0Km/34dUP0wvo4sCrJOQy9Tl9geNF8JcMQgk8CFzAM84DhI7+1GS70ugB4+uJX+RZ+E/hSko3ACxg+thn1E+A+Sc5ieOf6UoaPY97Reki+wvAR5vWt/JeAjzKch5dV1eWL0IZRXwXWtftjJ+D/AW9k6Nn5EDOH+r8Cfr+16yyGj7s2R//I8KnBGQz3xZSq6lSGMfifb216H0MgXWwvY/hY+pwMV8y/rHP7FzE8l04DvjZNmcOAI5KczRBmNpeL8+ZyX417fqayHjh04k1Tkp2S3KVj+4PbNtsAj2EIXVN1GMzkaIae0NfNUOZTwNYZuQg5yW8neegybMt64HZJntLquQXwzwxD+W6YVG6zb0+SeybZe2TRvsC3O+q5HnhC61Ah7ULhEVO+OU5y96r6YlUdDVwN7JnkbsDFVfVvwCnAfTvqMV8WvT3tTc6bGYan/csKaM/q1gNNe/wexPT/0zcfSzWO5NZ84+ZxwqsYxr89dqnrtAhtPoY2nmqJjr+GBbiYxJu35XgDnsjNF/qcxRCsfjyy/hbP14l1DD3mJzG8Gf7VxWvA1sB/tP29l6EXaf/RbUf2dQnDePgAb2UYOjBdPe/cjvdNhjdAH2XShUnLqC17MoSIC4HrgTcs1/uG4ROC/2LoBDqH4aP7XTofg+u4+SK047nlmNu/ZLjG4XSGMbXHt+UfYOj0OA/4P62ez2+PjY0MHVc7zXDMdzOMG/85w4XW83YdyGK3h2GoTrXzP3HNxKOWcXvuy9Dxdk7b/uj5astC3vzZ7yWQ5J8Y3mXdlmEIx1/VCr8jMow3/nFV/dMSHX8Nw8Vav7EUx5ekCRm+IvPdwB9X1VlLXR9Jm8YQLUmSJHXy2zkkSUumjbtcP8WqA6t908RysZLaAsuzPcuxzjOxPZs3e6IlSZKkTn47hyRJktTJEC1JkiR1MkRL0mYoyS+SbBy5rdmEfdwhyTPmv3aSJMdES9JmKMmPq2rb2UvOuI81bMJXOybZon79l+wkSSPsiZakZaL95Pyrk5zZftn0L9rybZOsT/LlJOcmmfhVxmOBu7ee7Fcn2T/JR0b299okh7fpS5IcneRzwOOT3D3Jx5OcleSz7dfuSPL4JOclOTvJfy7uGZCkzYdfcSdJm6dtkmxs09+qqscCRwA/qKrfTrI1cEaSU4FLGX759IdJdgG+kOQU4CjgN6pqX4Ak+89yzJ9V1YNa2fXA06vqwiQPZPgZ6AMYfhb64VX13Ymf6ZWkWyNDtCRtnm6YCL8jHgbcN8mhbX4HYG+GnzB+ZZKHAL8Edgd23YRjnghDzzbwe8B7k0ys27r9PQM4PslJDD/zK0m3SoZoSVo+Ajyrqj5xi4XDkIzVwAOq6udJLgFuO8X2N3HLYXyTy/yk/b0NcP0UIZ6qenrrmX40sDHJvsvxRxIkaVyOiZak5eMTwF8m2RIgyT2S3J6hR/rKFqB/H7hLK/8jYLuR7b8N7JNk6yQ7AAdOdZCq+iHwrSSPb8dJkvu16btX1Rer6mjgamDP+W+mJG3+7ImWpOXjTcAa4MsZxllcBTwGeCfw4SQbgI3A1wCq6pokZyQ5D/iPqvrbNgzjHOBC4CszHOsw4P8leSGwJfAe4Gzg1Un2ZugVX9+WSdKtjl9xJ0mSJHVyOIckSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVInQ7QkSZLUyRAtSZIkdTJES5IkSZ0M0ZIkSVKn/w/BK4o3Y/Bo5AAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "visualize_importances(feature_names, neuron_cond_vals_10.mean(dim=0).detach().numpy(), title=\"Average Feature Importances for Neuron 10\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "From the visualization above, it is evident that neuron 10 primarily relies on the gender and class features, substantially different from the focus of neuron 0." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this demo, we have applied different attribution techniques in Captum including Integrated Gradients for feature attribution and Conductance for layer and neuron attribution in order to better understand the neural network predicting survival. Although larger networks are more difficult to analyze than this simple network, these basic building blocks for attribution can be utilized to improve model interpretability, breaking the traditional \"black-box\" characterization of neural networks and delving deeper into understanding how and why they make their decisions. " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/attr/test_conductance.py b/tests/attr/test_layer_conductance.py similarity index 96% rename from tests/attr/test_conductance.py rename to tests/attr/test_layer_conductance.py index 54bbc66c0d..9723b11ec0 100644 --- a/tests/attr/test_conductance.py +++ b/tests/attr/test_layer_conductance.py @@ -3,7 +3,7 @@ import unittest import torch -from captum.attr._core.conductance import Conductance +from captum.attr._core.layer_conductance import LayerConductance from .helpers.basic_models import TestModel_ConvNet, TestModel_MultiLayer from .helpers.conductance_reference import ConductanceReference @@ -67,7 +67,7 @@ def test_matching_conv_with_baseline_conductance(self): def _conductance_test_helper( self, model, target_layer, test_input, expected_conductance ): - cond = Conductance(model, target_layer) + cond = LayerConductance(model, target_layer) attributions = cond.attribute( test_input, target=0, n_steps=500, method="gausslegendre" ) @@ -88,7 +88,7 @@ def forward_hook(module, inp, out): final_output = model(test_input) hook.remove() target_index = torch.argmax(torch.sum(final_output, 0)) - cond = Conductance(model, target_layer) + cond = LayerConductance(model, target_layer) cond_ref = ConductanceReference(model, target_layer) attributions = cond.attribute( test_input, diff --git a/tests/attr/test_neuron_conductance.py b/tests/attr/test_neuron_conductance.py index 3008b064ef..fe1d8a23fe 100644 --- a/tests/attr/test_neuron_conductance.py +++ b/tests/attr/test_neuron_conductance.py @@ -3,7 +3,7 @@ import unittest import torch -from captum.attr._core.conductance import Conductance +from captum.attr._core.layer_conductance import LayerConductance from captum.attr._core.neuron_conductance import NeuronConductance from .helpers.basic_models import TestModel_ConvNet, TestModel_MultiLayer @@ -59,7 +59,7 @@ def _conductance_input_test_assert( def _conductance_input_sum_test_assert( self, model, target_layer, test_input, test_baseline=None ): - layer_cond = Conductance(model, target_layer) + layer_cond = LayerConductance(model, target_layer) attributions = layer_cond.attribute( test_input, baselines=test_baseline,