demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Thermostat demo\n",
    "### This notebook is an introduction to Thermostat, to both the data hub and collection of explanation data (feature attribution maps) and the accompanying convenience functions for analysis of the maps."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To start off, we have to install the thermostat-datasets package using pip."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: thermostat-datasets in d:\\working student\\repo\\thermostat\\src (1.0.2)\n",
      "Requirement already satisfied: captum>=0.3 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from thermostat-datasets) (0.5.0)\n",
      "Requirement already satisfied: datasets>=1.5 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from thermostat-datasets) (1.18.4)\n",
      "Requirement already satisfied: jsonnet-binary in c:\\users\\49176\\anaconda3\\lib\\site-packages (from thermostat-datasets) (0.17.0)\n",
      "Requirement already satisfied: numpy>=1.20 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from thermostat-datasets) (1.20.3)\n",
      "Requirement already satisfied: overrides in c:\\users\\49176\\anaconda3\\lib\\site-packages (from thermostat-datasets) (6.1.0)\n",
      "Requirement already satisfied: pandas in c:\\users\\49176\\anaconda3\\lib\\site-packages (from thermostat-datasets) (1.3.4)\n",
      "Requirement already satisfied: protobuf in c:\\users\\49176\\anaconda3\\lib\\site-packages (from thermostat-datasets) (3.19.4)\n",
      "Requirement already satisfied: pytorch-ignite in c:\\users\\49176\\anaconda3\\lib\\site-packages (from thermostat-datasets) (0.4.8)\n",
      "Requirement already satisfied: scipy in c:\\users\\49176\\anaconda3\\lib\\site-packages (from thermostat-datasets) (1.7.1)\n",
      "Requirement already satisfied: sentencepiece in c:\\users\\49176\\anaconda3\\lib\\site-packages (from thermostat-datasets) (0.1.96)\n",
      "Requirement already satisfied: sklearn in c:\\users\\49176\\anaconda3\\lib\\site-packages (from thermostat-datasets) (0.0)\n",
      "Requirement already satisfied: spacy>=3.0 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from thermostat-datasets) (3.2.3)\n",
      "Requirement already satisfied: torch in c:\\users\\49176\\anaconda3\\lib\\site-packages (from thermostat-datasets) (1.11.0+cu113)\n",
      "Requirement already satisfied: tqdm>=4.49 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from thermostat-datasets) (4.62.3)\n",
      "Requirement already satisfied: transformers>=4.5 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from thermostat-datasets) (4.16.2)\n",
      "Requirement already satisfied: matplotlib in c:\\users\\49176\\anaconda3\\lib\\site-packages (from captum>=0.3->thermostat-datasets) (3.4.3)\n",
      "Requirement already satisfied: dill in c:\\users\\49176\\anaconda3\\lib\\site-packages (from datasets>=1.5->thermostat-datasets) (0.3.4)\n",
      "Requirement already satisfied: xxhash in c:\\users\\49176\\anaconda3\\lib\\site-packages (from datasets>=1.5->thermostat-datasets) (3.0.0)\n",
      "Requirement already satisfied: responses<0.19 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from datasets>=1.5->thermostat-datasets) (0.18.0)\n",
      "Requirement already satisfied: requests>=2.19.0 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from datasets>=1.5->thermostat-datasets) (2.26.0)\n",
      "Requirement already satisfied: packaging in c:\\users\\49176\\anaconda3\\lib\\site-packages (from datasets>=1.5->thermostat-datasets) (21.0)\n",
      "Requirement already satisfied: fsspec[http]>=2021.05.0 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from datasets>=1.5->thermostat-datasets) (2021.10.1)\n",
      "Requirement already satisfied: pyarrow!=4.0.0,>=3.0.0 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from datasets>=1.5->thermostat-datasets) (7.0.0)\n",
      "Requirement already satisfied: aiohttp in c:\\users\\49176\\anaconda3\\lib\\site-packages (from datasets>=1.5->thermostat-datasets) (3.8.1)\n",
      "Requirement already satisfied: huggingface-hub<1.0.0,>=0.1.0 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from datasets>=1.5->thermostat-datasets) (0.4.0)\n",
      "Requirement already satisfied: multiprocess in c:\\users\\49176\\anaconda3\\lib\\site-packages (from datasets>=1.5->thermostat-datasets) (0.70.12.2)\n",
      "Requirement already satisfied: setuptools in c:\\users\\49176\\anaconda3\\lib\\site-packages (from spacy>=3.0->thermostat-datasets) (58.0.4)\n",
      "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from spacy>=3.0->thermostat-datasets) (1.0.1)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from spacy>=3.0->thermostat-datasets) (1.8.2)\n",
      "Requirement already satisfied: typer<0.5.0,>=0.3.0 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from spacy>=3.0->thermostat-datasets) (0.4.0)\n",
      "Requirement already satisfied: thinc<8.1.0,>=8.0.12 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from spacy>=3.0->thermostat-datasets) (8.0.14)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.1 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from spacy>=3.0->thermostat-datasets) (2.4.2)\n",
      "Requirement already satisfied: jinja2 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from spacy>=3.0->thermostat-datasets) (2.11.3)\n",
      "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from spacy>=3.0->thermostat-datasets) (3.3.0)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from spacy>=3.0->thermostat-datasets) (2.0.6)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.4.0 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from spacy>=3.0->thermostat-datasets) (0.7.6)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from spacy>=3.0->thermostat-datasets) (0.9.0)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.8 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from spacy>=3.0->thermostat-datasets) (3.0.9)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from spacy>=3.0->thermostat-datasets) (3.0.6)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from spacy>=3.0->thermostat-datasets) (1.0.6)\n",
      "Requirement already satisfied: pathy>=0.3.5 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from spacy>=3.0->thermostat-datasets) (0.6.1)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from spacy>=3.0->thermostat-datasets) (2.0.6)\n",
      "Requirement already satisfied: typing-extensions in c:\\users\\49176\\anaconda3\\lib\\site-packages (from torch->thermostat-datasets) (3.10.0.2)\n",
      "Requirement already satisfied: colorama in c:\\users\\49176\\anaconda3\\lib\\site-packages (from tqdm>=4.49->thermostat-datasets) (0.4.4)\n",
      "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from transformers>=4.5->thermostat-datasets) (6.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from transformers>=4.5->thermostat-datasets) (2021.8.3)\n",
      "Requirement already satisfied: tokenizers!=0.11.3,>=0.10.1 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from transformers>=4.5->thermostat-datasets) (0.11.6)\n",
      "Requirement already satisfied: sacremoses in c:\\users\\49176\\anaconda3\\lib\\site-packages (from transformers>=4.5->thermostat-datasets) (0.0.47)\n",
      "Requirement already satisfied: filelock in c:\\users\\49176\\anaconda3\\lib\\site-packages (from transformers>=4.5->thermostat-datasets) (3.3.1)\n",
      "Requirement already satisfied: typing-utils>=0.0.3 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from overrides->thermostat-datasets) (0.1.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from pandas->thermostat-datasets) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2017.3 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from pandas->thermostat-datasets) (2021.3)\n",
      "Requirement already satisfied: scikit-learn in c:\\users\\49176\\anaconda3\\lib\\site-packages (from sklearn->thermostat-datasets) (0.24.2)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from packaging->datasets>=1.5->thermostat-datasets) (3.0.4)\n",
      "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from pathy>=0.3.5->spacy>=3.0->thermostat-datasets) (5.2.1)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from python-dateutil>=2.7.3->pandas->thermostat-datasets) (1.16.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from requests>=2.19.0->datasets>=1.5->thermostat-datasets) (2021.10.8)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from requests>=2.19.0->datasets>=1.5->thermostat-datasets) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from requests>=2.19.0->datasets>=1.5->thermostat-datasets) (3.2)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from requests>=2.19.0->datasets>=1.5->thermostat-datasets) (1.26.7)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from typer<0.5.0,>=0.3.0->spacy>=3.0->thermostat-datasets) (8.0.3)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from aiohttp->datasets>=1.5->thermostat-datasets) (1.3.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from aiohttp->datasets>=1.5->thermostat-datasets) (1.2.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from aiohttp->datasets>=1.5->thermostat-datasets) (6.0.2)\n",
      "Requirement already satisfied: attrs>=17.3.0 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from aiohttp->datasets>=1.5->thermostat-datasets) (21.2.0)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from aiohttp->datasets>=1.5->thermostat-datasets) (1.7.2)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from aiohttp->datasets>=1.5->thermostat-datasets) (4.0.2)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from jinja2->spacy>=3.0->thermostat-datasets) (1.1.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from matplotlib->captum>=0.3->thermostat-datasets) (8.4.0)\n",
      "Requirement already satisfied: cycler>=0.10 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from matplotlib->captum>=0.3->thermostat-datasets) (0.10.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from matplotlib->captum>=0.3->thermostat-datasets) (1.3.1)\n",
      "Requirement already satisfied: joblib in c:\\users\\49176\\anaconda3\\lib\\site-packages (from sacremoses->transformers>=4.5->thermostat-datasets) (1.1.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\users\\49176\\anaconda3\\lib\\site-packages (from scikit-learn->sklearn->thermostat-datasets) (2.2.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install thermostat-datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we import some utilities."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "# Suppress warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "\n",
    "import sys\n",
    "# Include root directory in module path\n",
    "sys.path.append('src')\n",
    "\n",
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can import our library."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import thermostat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load dataset\n",
    "\n",
    "Let's use the `load` method which is a wrapper around the `load_dataset` function from HF `datasets`.\n",
    "In the background, this uses the dataset script (\"hf_dataset.py\") in the \"thermostat\" directory.\n",
    "\n",
    "In this example, we use the `imdb-bert-lig` configuration.\n",
    "This refers to **Layer Integrated Gradients** (LIG) explanations of the predictions by a **BERT** model that has been fine-tuned on the **IMDb** (train) dataset and evaluated on the **IMDb** test dataset.\n",
    "In other words, we load the 25k test examples from the IMDb test plus the BERT predictions and the feature attributions from a Layer Integrated Gradients explainer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset thermostat (C:\\Users\\49176\\.cache\\huggingface\\datasets\\thermostat\\imdb-bert-lig\\1.0.1\\0cbe93e1fbe5b8ed0217559442d8b49a80fd4c2787185f2d7940817c67d8707b)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Thermostat configuration: imdb-bert-lig\n"
     ]
    }
   ],
   "source": [
    "lig = thermostat.load(\"imdb-bert-lig\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's see what's inside the loaded dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IMDb dataset, BERT model, Layer Integrated Gradients explanations\n",
      "Dataset: imdb\n",
      "Model: textattack/bert-base-uncased-imdb\n",
      "Explainer: LayerIntegratedGradients\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(lig)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's inspect a single instance of the loaded dataset.\n",
    "For readability purposes, we will not print the whole content of that instance.\n",
    "Instead, we're showing only the first few entries of the attributions and the input_ids, respectively."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index: 429\n",
      "Attributions (first 4): [0.0, 2.3141794204711914, 0.06655970215797424, -0.47832658886909485]\n",
      "True label: pos\n",
      "Predicted label: pos\n"
     ]
    }
   ],
   "source": [
    "example = lig[429]\n",
    "\n",
    "print(f'Index: {example.idx}')\n",
    "print(f'Attributions (first 4): {example.attributions[:4]}')\n",
    "print(f'True label: {example.true_label}')\n",
    "print(f'Predicted label: {example.predicted_label}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also print a heatmap of token-attribution tuples via the `explanation` attribute of an instance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('[CLS]', 0.0, 0),\n",
      " ('amazing', 2.3141794204711914, 1),\n",
      " ('movie', 0.06655970215797424, 2),\n",
      " ('.', -0.47832658886909485, 3),\n",
      " ('some', 0.15708176791667938, 4),\n",
      " ('of', -0.02931656688451767, 5),\n",
      " ('the', -0.08834744244813919, 6),\n",
      " ('script', -0.2660972774028778, 7),\n",
      " ('writing', -0.4021594822406769, 8),\n",
      " ('could', -0.19280624389648438, 9),\n",
      " ('have', -0.015477157197892666, 10),\n",
      " ('been', -0.21898044645786285, 11),\n",
      " ('better', -0.4095713794231415, 12),\n",
      " ('(', 0.05475223436951637, 13),\n",
      " ('some', 0.0466572567820549, 14),\n",
      " ('cl', 0.08523529022932053, 15),\n",
      " ('##iche', 0.05406142398715019, 16),\n",
      " ('##d', -0.031489163637161255, 17),\n",
      " ('language', -0.3399031162261963, 18),\n",
      " (')', -0.11275435984134674, 19),\n",
      " ('.', -0.22217823565006256, 20),\n",
      " ('joyce', 0.6259628534317017, 21),\n",
      " (\"'\", -0.20313552021980286, 22),\n",
      " ('s', -0.22971349954605103, 23),\n",
      " ('\"', -0.28431516885757446, 24),\n",
      " ('the', 0.13832062482833862, 25),\n",
      " ('dead', -0.09080619364976883, 26),\n",
      " ('\"', 0.008070609532296658, 27),\n",
      " ('is', -0.09763125330209732, 28),\n",
      " ('all', 0.31558454036712646, 29),\n",
      " ('##uded', -0.7008199691772461, 30),\n",
      " ('to', 0.09101452678442001, 31),\n",
      " ('throughout', 0.11561280488967896, 32),\n",
      " ('the', 0.1659221053123474, 33),\n",
      " ('movie', -0.40316757559776306, 34),\n",
      " ('.', -0.42794814705848694, 35),\n",
      " ('beautiful', 2.0079877376556396, 36),\n",
      " ('scenery', -0.7362574934959412, 37),\n",
      " ('and', 0.22411005198955536, 38),\n",
      " ('great', 1.8532911539077759, 39),\n",
      " ('acting', -0.25186970829963684, 40),\n",
      " ('.', -0.42279019951820374, 41),\n",
      " ('very', 1.5215049982070923, 42),\n",
      " ('poetic', 0.6973422765731812, 43),\n",
      " ('.', -0.4415672719478607, 44),\n",
      " ('highly', 0.5367811322212219, 45),\n",
      " ('recommend', 1.4894026517868042, 46),\n",
      " ('.', 0.14289245009422302, 47),\n",
      " ('[SEP]', 0.0, 48)]\n"
     ]
    }
   ],
   "source": [
    "pprint(example.explanation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Another option is printing it as a `pandas` DataFrame which is accessible via the `.heatmap` attribute."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "token_index     0        1         2         3         4         5         6   \\\n",
      "token        [CLS]  amazing     movie         .      some        of       the   \n",
      "attribution    0.0      1.0  0.028762 -0.206694  0.067878 -0.012668 -0.038177   \n",
      "text_field    text     text      text      text      text      text      text   \n",
      "\n",
      "token_index        7         8         9         10        11        12  \\\n",
      "token          script   writing     could      have      been    better   \n",
      "attribution -0.114986 -0.173781 -0.083315 -0.006688 -0.094626 -0.176983   \n",
      "text_field       text      text      text      text      text      text   \n",
      "\n",
      "token_index        13        14        15        18        19        20  \\\n",
      "token               (      some   cliched  language         )         .   \n",
      "attribution  0.023659  0.020161  0.036832 -0.146878 -0.048723 -0.096007   \n",
      "text_field       text      text      text      text      text      text   \n",
      "\n",
      "token_index       21        22        23        24        25        26  \\\n",
      "token          joyce         '         s         \"       the      dead   \n",
      "attribution  0.27049 -0.087779 -0.099263 -0.122858  0.059771 -0.039239   \n",
      "text_field      text      text      text      text      text      text   \n",
      "\n",
      "token_index        27        28        29        31          32        33  \\\n",
      "token               \"        is   alluded        to  throughout       the   \n",
      "attribution  0.003487 -0.042188 -0.302837  0.039329    0.049958  0.071698   \n",
      "text_field       text      text      text      text        text      text   \n",
      "\n",
      "token_index        34        35         36        37        38        39  \\\n",
      "token           movie         .  beautiful   scenery       and     great   \n",
      "attribution -0.174216 -0.184924   0.867689 -0.318151  0.096842  0.800842   \n",
      "text_field       text      text       text      text      text      text   \n",
      "\n",
      "token_index        40        41        42        43        44        45  \\\n",
      "token          acting         .      very    poetic         .    highly   \n",
      "attribution -0.108838 -0.182696  0.657471  0.301335 -0.190809  0.231953   \n",
      "text_field       text      text      text      text      text      text   \n",
      "\n",
      "token_index         46        47     48  \n",
      "token        recommend         .  [SEP]  \n",
      "attribution   0.643599  0.061746    0.0  \n",
      "text_field        text      text   text  \n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "pd.set_option('display.max_columns', None)\n",
    "\n",
    "print(example.heatmap)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualize data\n",
    "Now the much more visually pleasing way is to turn the attribution scores into colors and display the heatmap using the displaCy (spaCy) library. We can do this with the `.render()` function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "            <mark class=\"entity\" style=\"background: #ffffff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                [CLS]\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff0000; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                amazing\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fff7f7; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                movie\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #cacaff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                .\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffeded; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                some\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fbfbff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                of\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #f5f5ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                the\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #e1e1ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                script\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #d2d2ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                writing\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #e9e9ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                could\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fdfdff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                have\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #e6e6ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                been\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #d1d1ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                better\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fff8f8; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                (\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fff9f9; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                some\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fff5f5; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                cliched\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #d9d9ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                language\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #f2f2ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                )\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #e6e6ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                .\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffbaba; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                joyce\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #e8e8ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                '\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #e5e5ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                s\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #dfdfff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                &quot;\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffefef; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                the\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #f4f4ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                dead\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fffefe; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                &quot;\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #f4f4ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                is\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #b1b1ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                alluded\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fff4f4; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                to\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fff2f2; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                throughout\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffecec; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                the\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #d2d2ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                movie\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #cfcfff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                .\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff2121; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                beautiful\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #adadff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                scenery\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffe6e6; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                and\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff3232; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                great\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #e3e3ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                acting\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #d0d0ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                .\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff5757; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                very\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffb2b2; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                poetic\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ceceff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                .\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffc3c3; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                highly\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff5a5a; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                recommend\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffefef; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                .\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffffff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                [SEP]\n",
       "            </mark>\n",
       "            </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "example.render()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Aggregate data\n",
    "Let us first compare the heatmaps of two different models on the same dataset+explainer configuration, MNLI + Occlusion."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset thermostat (C:\\Users\\49176\\.cache\\huggingface\\datasets\\thermostat\\multi_nli-bert-occ\\1.0.1\\0cbe93e1fbe5b8ed0217559442d8b49a80fd4c2787185f2d7940817c67d8707b)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Thermostat configuration: multi_nli-bert-occ\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset thermostat (C:\\Users\\49176\\.cache\\huggingface\\datasets\\thermostat\\multi_nli-electra-occ\\1.0.1\\0cbe93e1fbe5b8ed0217559442d8b49a80fd4c2787185f2d7940817c67d8707b)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Thermostat configuration: multi_nli-electra-occ\n"
     ]
    }
   ],
   "source": [
    "bert = thermostat.load(\"multi_nli-bert-occ\")\n",
    "electra = thermostat.load(\"multi_nli-electra-occ\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's build a list of instances for which the predicted labels of BERT and ELECTRA do not align:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "disagreement = [(b, e) for (b, e) in zip(bert, electra) if b.predicted_label != e.predicted_label]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We choose one instance (index: 51) and render both heatmaps:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Instance index: 378\n",
      "Model: textattack/bert-base-uncased-MNLI | Pred: entailment | True: contradiction\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "            <mark class=\"entity\" style=\"background: #f0f0ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                [CLS]\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #e8e8ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                as\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #e6e6ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                recent\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #e7e7ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                events\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffbaba; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                illustrate\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff5b5b; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                ,\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff0000; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                trust\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff3c3c; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                takes\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff6767; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                years\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff9797; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                to\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff9090; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                gain\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffc8c8; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                but\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fff4f4; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                can\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #e7e7ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                be\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #f1f1ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                lost\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #efefff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                in\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fffafa; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                an\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fff1f1; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                instant\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffe0e0; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                .\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fff8f8; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                [SEP]\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffebeb; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                trust\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #e6e6ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                ,\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ccccff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                once\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #7a7aff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                built\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #9898ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                ,\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #c5c5ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                is\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffcccc; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                hard\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffc9c9; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                to\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffd3d3; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                lose\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffdfdf; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                .\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffd7d7; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                [SEP]\n",
       "            </mark>\n",
       "            </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: howey/electra-base-mnli | Pred: contradiction | True: contradiction\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "            <mark class=\"entity\" style=\"background: #c8c8ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                [CLS]\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ccccff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                as\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #e1e1ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                recent\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #eeeeff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                events\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffc0c0; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                illustrate\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff9595; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                ,\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff7272; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                trust\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffc5c5; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                takes\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #e9e9ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                years\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #8181ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                to\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #5555ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                gain\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #4949ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                but\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ddddff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                can\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff8686; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                be\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff0000; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                lost\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff2323; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                in\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff5353; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                an\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff7878; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                instant\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffb3b3; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                .\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffe7e7; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                [SEP]\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ababff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                trust\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #a6a6ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                ,\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #a4a4ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                once\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #9f9fff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                built\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #f5f5ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                ,\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffbdbd; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                is\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffafaf; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                hard\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffc5c5; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                to\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffd2d2; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                lose\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffd9d9; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                .\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fff3f3; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                [SEP]\n",
       "            </mark>\n",
       "            </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "u_b, u_e = disagreement[51]\n",
    "print(f'Instance index: {u_b.idx}')\n",
    "\n",
    "print(f'Model: {u_b.model_name} | Pred: {u_b.predicted_label} | True: {u_b.true_label}')\n",
    "u_b.render()\n",
    "print(f'Model: {u_e.model_name} | Pred: {u_e.predicted_label} | True: {u_e.true_label}')\n",
    "u_e.render()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We observe that the Occlusion explainer does not attribute much importance to the phrase *can be lost in an instant*. This is plausible since the heatmap explains a misclassification: the maximum output activation stands for `entailment`, but the correct label is `contradiction` and the phrase certainly is a signal for `contradiction`. In contrast, in the case of ELECTRA which correctly classified the instance the signal phrase receives much higher importance scores."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bonus: Print classification report from sklearn\n",
    "We also added the classification report function from sklearn as a method to apply to a Thermopack."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "contradiction       0.86      0.87      0.86      3213\n",
      "   entailment       0.89      0.85      0.87      3479\n",
      "      neutral       0.79      0.82      0.81      3123\n",
      "\n",
      "     accuracy                           0.85      9815\n",
      "    macro avg       0.85      0.85      0.85      9815\n",
      " weighted avg       0.85      0.85      0.85      9815\n",
      "\n",
      "=====================\n",
      "\n",
      "\n",
      "electra\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "   entailment       0.93      0.87      0.90      3479\n",
      "      neutral       0.83      0.87      0.85      3123\n",
      "contradiction       0.91      0.92      0.91      3213\n",
      "\n",
      "     accuracy                           0.89      9815\n",
      "    macro avg       0.89      0.89      0.89      9815\n",
      " weighted avg       0.89      0.89      0.89      9815\n",
      "\n",
      "=====================\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model_name, data in zip([\"bert\", \"electra\"], [bert, electra]):\n",
    "    print(model_name)\n",
    "    data.classification_report()\n",
    "    print('=====================\\n\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Explainer comparison"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can only compare the heatmaps between multiple explainers. For this, let us load both the MultiNLI-BERT-Occlusion dataset plus the associated LIG and LIME explainers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset thermostat (C:\\Users\\49176\\.cache\\huggingface\\datasets\\thermostat\\multi_nli-bert-occ\\1.0.1\\0cbe93e1fbe5b8ed0217559442d8b49a80fd4c2787185f2d7940817c67d8707b)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Thermostat configuration: multi_nli-bert-occ\n",
      "Loading Thermostat configuration: multi_nli-bert-lig\n",
      "Downloading and preparing dataset thermostat/multi_nli-bert-lig to C:\\Users\\49176\\.cache\\huggingface\\datasets\\thermostat\\multi_nli-bert-lig\\1.0.1\\0cbe93e1fbe5b8ed0217559442d8b49a80fd4c2787185f2d7940817c67d8707b...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "98a45c0de0b04800b1974202833db926",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/58.5M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset thermostat downloaded and prepared to C:\\Users\\49176\\.cache\\huggingface\\datasets\\thermostat\\multi_nli-bert-lig\\1.0.1\\0cbe93e1fbe5b8ed0217559442d8b49a80fd4c2787185f2d7940817c67d8707b. Subsequent calls will reuse this data.\n",
      "Loading Thermostat configuration: multi_nli-bert-lime\n",
      "Downloading and preparing dataset thermostat/multi_nli-bert-lime to C:\\Users\\49176\\.cache\\huggingface\\datasets\\thermostat\\multi_nli-bert-lime\\1.0.1\\0cbe93e1fbe5b8ed0217559442d8b49a80fd4c2787185f2d7940817c67d8707b...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "96404ef25f5040a5af1358793cc69d19",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/59.4M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset thermostat downloaded and prepared to C:\\Users\\49176\\.cache\\huggingface\\datasets\\thermostat\\multi_nli-bert-lime\\1.0.1\\0cbe93e1fbe5b8ed0217559442d8b49a80fd4c2787185f2d7940817c67d8707b. Subsequent calls will reuse this data.\n"
     ]
    }
   ],
   "source": [
    "unit_index = 378\n",
    "u_occ = thermostat.load(\"multi_nli-bert-occ\")[unit_index]\n",
    "u_intg = thermostat.load(\"multi_nli-bert-lig\")[unit_index]\n",
    "u_lime = thermostat.load(\"multi_nli-bert-lime\")[unit_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Occlusion --- same map as in previous example\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "            <mark class=\"entity\" style=\"background: #f0f0ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                [CLS]\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #e8e8ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                as\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #e6e6ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                recent\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #e7e7ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                events\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffbaba; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                illustrate\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff5b5b; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                ,\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff0000; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                trust\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff3c3c; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                takes\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff6767; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                years\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff9797; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                to\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff9090; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                gain\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffc8c8; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                but\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fff4f4; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                can\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #e7e7ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                be\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #f1f1ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                lost\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #efefff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                in\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fffafa; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                an\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fff1f1; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                instant\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffe0e0; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                .\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fff8f8; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                [SEP]\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffebeb; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                trust\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #e6e6ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                ,\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ccccff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                once\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #7a7aff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                built\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #9898ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                ,\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #c5c5ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                is\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffcccc; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                hard\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffc9c9; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                to\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffd3d3; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                lose\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffdfdf; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                .\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffd7d7; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                [SEP]\n",
       "            </mark>\n",
       "            </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LayerIntegratedGradients\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "            <mark class=\"entity\" style=\"background: #ffffff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                [CLS]\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #e8e8ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                as\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffdede; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                recent\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffcdcd; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                events\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #d7d7ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                illustrate\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #efefff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                ,\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff4040; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                trust\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #dadaff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                takes\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff2525; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                years\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #efefff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                to\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #c4c4ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                gain\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #e8e8ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                but\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #8484ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                can\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #b5b5ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                be\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #5353ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                lost\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #eeeeff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                in\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fdfdff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                an\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffacac; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                instant\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #eaeaff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                .\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffffff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                [SEP]\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff2626; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                trust\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffa9a9; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                ,\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff1c1c; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                once\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #7272ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                built\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffb2b2; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                ,\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffbcbc; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                is\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffabab; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                hard\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffa7a7; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                to\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #0000ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                lose\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #a1a1ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                .\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffffff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                [SEP]\n",
       "            </mark>\n",
       "            </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LimeBase\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "            <mark class=\"entity\" style=\"background: #ffffff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                [CLS]\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #0000ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                as\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ceceff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                recent\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fdfdff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                events\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fff4f4; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                illustrate\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffd4d4; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                ,\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fffbfb; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                trust\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ebebff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                takes\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff6868; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                years\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff8484; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                to\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffdfdf; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                gain\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #c2c2ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                but\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffe3e3; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                can\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #aeaeff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                be\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #c3c3ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                lost\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff9292; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                in\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffc8c8; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                an\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffe4e4; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                instant\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffe0e0; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                .\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffffff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                [SEP]\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fff3f3; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                trust\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #1f1fff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                ,\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #c4c4ff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                once\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #dbdbff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                built\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ff5454; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                ,\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #9d9dff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                is\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffaeae; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                hard\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffbfbf; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                to\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #fff8f8; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                lose\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #cdcdff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                .\n",
       "            </mark>\n",
       "            \n",
       "            <mark class=\"entity\" style=\"background: #ffffff; padding: 0.15em 0.3em; margin: 0 0.2em; line-height: 2.2;\n",
       "            border-radius: 0.25em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "                [SEP]\n",
       "            </mark>\n",
       "            </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(f'{u_occ.explainer_name} --- same map as in previous example')\n",
    "u_occ.render()\n",
    "print(u_intg.explainer_name)\n",
    "u_intg.render()\n",
    "print(u_lime.explainer_name)\n",
    "u_lime.render()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Rank correlation\n",
    "A far more interesting, empirical investigation is how well two explainers on the same dataset+model combination correlate regarding their attribution scores."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Thermostat configuration: imdb-bert-lime\n",
      "Downloading and preparing dataset thermostat/imdb-bert-lime to C:\\Users\\49176\\.cache\\huggingface\\datasets\\thermostat\\imdb-bert-lime\\1.0.1\\0cbe93e1fbe5b8ed0217559442d8b49a80fd4c2787185f2d7940817c67d8707b...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "017d3fe023654275a9bdf63662a4b2a6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/276M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset thermostat downloaded and prepared to C:\\Users\\49176\\.cache\\huggingface\\datasets\\thermostat\\imdb-bert-lime\\1.0.1\\0cbe93e1fbe5b8ed0217559442d8b49a80fd4c2787185f2d7940817c67d8707b. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset thermostat (C:\\Users\\49176\\.cache\\huggingface\\datasets\\thermostat\\imdb-bert-lig\\1.0.1\\0cbe93e1fbe5b8ed0217559442d8b49a80fd4c2787185f2d7940817c67d8707b)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Thermostat configuration: imdb-bert-lig\n"
     ]
    }
   ],
   "source": [
    "imdb_lime = thermostat.load(\"imdb-bert-lime\")\n",
    "imdb_intg = thermostat.load(\"imdb-bert-lig\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After loading LIME and LIG explainers for IMDb+BERT, we import the SciPy function that calculates Kendall's tau for rank correlation of attributions.\n",
    "\n",
    "We merge all attributions to a single list via `.flatten()` which is possible, because accessing the attributions of an entire dataset (Thermopack) via the `.attributions` attribute returns a NumPy array."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KendalltauResult(correlation=0.025657302000906455, pvalue=0.0)\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import kendalltau\n",
    "il_atts = imdb_lime.attributions.flatten()\n",
    "ig_atts = imdb_intg.attributions.flatten()\n",
    "\n",
    "kendall_imdb = kendalltau(il_atts, ig_atts)\n",
    "print(kendall_imdb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's also consider MultiNLI explanations on BERT plus LIME and LIG."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset thermostat (/home/nfel/.cache/huggingface/datasets/thermostat/multi_nli-bert-lime/1.0.0/d4c1fec14831f7d2677ccb8fba33151c9fea8119c4921e647af71cb81299899f)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Thermostat configuration: multi_nli-bert-lime\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset thermostat (/home/nfel/.cache/huggingface/datasets/thermostat/multi_nli-bert-lig/1.0.0/d4c1fec14831f7d2677ccb8fba33151c9fea8119c4921e647af71cb81299899f)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Thermostat configuration: multi_nli-bert-lig\n"
     ]
    }
   ],
   "source": [
    "mnli_lime = thermostat.load(\"multi_nli-bert-lime\")\n",
    "mnli_intg = thermostat.load(\"multi_nli-bert-lig\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KendalltauResult(correlation=0.10327941961925725, pvalue=0.0)\n"
     ]
    }
   ],
   "source": [
    "ml_atts = mnli_lime.attributions.flatten()\n",
    "mg_atts = mnli_intg.attributions.flatten()\n",
    "kendall_mnli = kendalltau(ml_atts, mg_atts)\n",
    "print(kendall_mnli)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We find that the correlation between LIG and LIME is higher for MultiNLI than it is for IMDb.\n",
    "This aligns well with the findings reported in the \"Order in the Court\" paper by Neely, Schouten et al. (2021): https://api.semanticscholar.org/CorpusID:234096057"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}