examples/anchor_text_movie.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Anchor explanations for movie sentiment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this example, we will explain why a certain sentence is classified by a logistic regression as having negative or positive sentiment. The logistic regression is trained on negative and positive movie reviews."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import spacy\n",
    "import string\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from alibi.explainers import AnchorText\n",
    "from alibi.datasets import fetch_movie_sentiment\n",
    "from alibi.utils.download import spacy_model\n",
    "from alibi.utils.lang_model import DistilbertBaseUncased, BertBaseUncased, RobertaBase"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load movie review dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `fetch_movie_sentiment` function returns a `Bunch` object containing the features, the targets and the target names for the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['data', 'target', 'target_names'])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies = fetch_movie_sentiment()\n",
    "movies.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = movies.data\n",
    "labels = movies.target\n",
    "target_names = movies.target_names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define shuffled training, validation and test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, test, train_labels, test_labels = train_test_split(data, labels, test_size=.2, random_state=42)\n",
    "train, val, train_labels, val_labels = train_test_split(train, train_labels, test_size=.1, random_state=42)\n",
    "train_labels = np.array(train_labels)\n",
    "test_labels = np.array(test_labels)\n",
    "val_labels = np.array(val_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Apply CountVectorizer to training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer()"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer = CountVectorizer(min_df=1)\n",
    "vectorizer.fit(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fit model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(solver='liblinear')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(0)\n",
    "clf = LogisticRegression(solver='liblinear')\n",
    "clf.fit(vectorizer.transform(train), train_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define prediction function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "predict_fn = lambda x: clf.predict(vectorizer.transform(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make predictions on train and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train accuracy: 0.980\n",
      "Validation accuracy: 0.754\n",
      "Test accuracy: 0.759\n"
     ]
    }
   ],
   "source": [
    "preds_train = predict_fn(train)\n",
    "preds_val = predict_fn(val)\n",
    "preds_test = predict_fn(test)\n",
    "print('Train accuracy: %.3f' % accuracy_score(train_labels, preds_train))\n",
    "print('Validation accuracy: %.3f' % accuracy_score(val_labels, preds_val))\n",
    "print('Test accuracy: %.3f' % accuracy_score(test_labels, preds_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load spaCy model\n",
    "\n",
    "English multi-task CNN trained on OntoNotes, with GloVe vectors trained on Common Crawl. Assigns word vectors, context-specific token vectors, POS tags, dependency parse and named entities."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = 'en_core_web_md'\n",
    "spacy_model(model=model)\n",
    "nlp = spacy.load(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Instance to be explained"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "* Text: a visually flashy but narratively opaque and emotionally vapid exercise in style and mystification .\n",
      "* Prediction: negative\n"
     ]
    }
   ],
   "source": [
    "class_names = movies.target_names\n",
    "\n",
    "# select instance to be explained\n",
    "text = data[4]\n",
    "print(\"* Text: %s\" % text)\n",
    "\n",
    "# compute class prediction\n",
    "pred = class_names[predict_fn([text])[0]]\n",
    "alternative =  class_names[1 - predict_fn([text])[0]]\n",
    "print(\"* Prediction: %s\" % pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize anchor text explainer with `unknown` sampling\n",
    "\n",
    "* `sampling_strategy='unknown'` means we will perturb examples by replacing words with UNKs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "explainer = AnchorText(\n",
    "    predictor=predict_fn, \n",
    "    sampling_strategy='unknown',\n",
    "    nlp=nlp,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Explanation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "explanation = explainer.explain(text, threshold=0.95)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us now take a look at the anchor. The word `flashy` basically guarantees a negative prediction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Anchor: flashy\n",
      "Precision: 0.99\n",
      "\n",
      "Examples where anchor applies and model predicts negative:\n",
      "a UNK flashy UNK UNK opaque and emotionally vapid exercise in style UNK mystification .\n",
      "a UNK flashy UNK UNK UNK and emotionally UNK exercise UNK UNK and UNK UNK\n",
      "a UNK flashy UNK narratively opaque UNK UNK UNK exercise in style and UNK UNK\n",
      "UNK visually flashy UNK narratively UNK and emotionally UNK UNK UNK UNK UNK mystification .\n",
      "UNK UNK flashy UNK UNK opaque and emotionally UNK UNK in UNK and UNK .\n",
      "a visually flashy but UNK UNK and UNK UNK UNK in style UNK mystification .\n",
      "a visually flashy but UNK opaque UNK emotionally vapid UNK in UNK and mystification .\n",
      "a UNK flashy but narratively UNK UNK emotionally vapid exercise in style UNK mystification UNK\n",
      "a UNK flashy but narratively opaque UNK emotionally vapid exercise in style and mystification .\n",
      "a visually flashy UNK UNK opaque UNK UNK UNK exercise in UNK UNK UNK .\n",
      "\n",
      "Examples where anchor applies and model predicts positive:\n",
      "UNK UNK flashy but narratively UNK and UNK UNK UNK in style and UNK UNK\n"
     ]
    }
   ],
   "source": [
    "print('Anchor: %s' % (' AND '.join(explanation.anchor)))\n",
    "print('Precision: %.2f' % explanation.precision)\n",
    "print('\\nExamples where anchor applies and model predicts %s:' % pred)\n",
    "print('\\n'.join([x for x in explanation.raw['examples'][-1]['covered_true']]))\n",
    "print('\\nExamples where anchor applies and model predicts %s:' % alternative)\n",
    "print('\\n'.join([x for x in explanation.raw['examples'][-1]['covered_false']]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize anchor text explainer with word `similarity` sampling\n",
    "\n",
    "Let's try this with another perturbation distribution, namely one that replaces words by similar words instead of UNKs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "explainer = AnchorText(\n",
    "    predictor=predict_fn, \n",
    "    sampling_strategy='similarity',   # replace masked words by simialar words\n",
    "    nlp=nlp,                          # spacy object\n",
    "    sample_proba=0.5,                 # probability of a word to be masked and replace by as similar word\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "explanation = explainer.explain(text, threshold=0.95)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The anchor now shows that we need more to guarantee the negative prediction:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Anchor: exercise AND emotionally\n",
      "Precision: 0.95\n",
      "\n",
      "Examples where anchor applies and model predicts negative:\n",
      "a visually ballsy but narratively green and emotionally illogical exercise in style and hysteria .\n",
      "an accurately flashy but godsend opaque and emotionally illogical exercise in flair and superstition .\n",
      "a fantastically pretentious but amazingly opaque and emotionally cartoonish exercise to style and mystification .\n",
      "a stunningly oddball but narratively green and emotionally uninteresting exercise in style and mystification .\n",
      "a visually flashy but narratively black and emotionally ephemeral exercise among waltz and cowardice .\n",
      "an importantly stylish but narratively opaque and emotionally vapid exercise in temperament and mystification .\n",
      "any visually ingenious but philosophically opaque and emotionally insufferable exercise midst style and mystification .\n",
      "a lovingly gaudy but narratively opaque and emotionally vapid exercise arround style and mystification .\n",
      "any surprisingly mischievous but suprisingly minimal and emotionally vapid exercise since style and uselessness .\n",
      "the digitally flashy but masterfully opaque and emotionally vapid exercise in style and mystification .\n",
      "\n",
      "Examples where anchor applies and model predicts positive:\n",
      "a visually brash but narratively realistic and emotionally fanciful exercise in jazz and mystification .\n",
      "an lovingly glamorous but narratively expandable and emotionally extravagant exercise in style and lament .\n",
      "any precisely spendy but tremendously opaque and emotionally unscientific exercise in culture and mystification .\n",
      "a excellently tasteless but deliciously realistic and emotionally formulaic exercise after style and mystification .\n"
     ]
    }
   ],
   "source": [
    "print('Anchor: %s' % (' AND '.join(explanation.anchor)))\n",
    "print('Precision: %.2f' % explanation.precision)\n",
    "print('\\nExamples where anchor applies and model predicts %s:' % pred)\n",
    "print('\\n'.join([x for x in explanation.raw['examples'][-1]['covered_true']]))\n",
    "print('\\nExamples where anchor applies and model predicts %s:' % alternative)\n",
    "print('\\n'.join([x for x in explanation.raw['examples'][-1]['covered_false']]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can make the token perturbation distribution sample words that are more similar to the ground truth word via the `top_n` argument. Smaller values (default=100) should result in sentences that are more coherent and thus more in the distribution of natural language which could influence the returned anchor. By setting the `use_proba` to True, the sampling distribution for perturbed tokens is proportional to the similarity score between the possible perturbations and the original word. We can also put more weight on similar words via the `temperature` argument. Lower values of `temperature` increase the sampling weight of more similar words. The following example will perturb tokens in the original sentence with probability equal to `sample_proba`. The sampling distribution for the perturbed tokens is proportional to the similarity score between the ground truth word and each of the `top_n` words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "explainer = AnchorText(\n",
    "    predictor=predict_fn, \n",
    "    sampling_strategy='similarity',  # replace masked words by simialar words\n",
    "    nlp=nlp,                         # spacy object\n",
    "    use_proba=True,                  # sample according to the similiary distribution\n",
    "    sample_proba=0.5,                # probability of a word to be masked and replace by as similar word\n",
    "    top_n=20,                        # consider only top 20 words most similar words\n",
    "    temperature=0.2                  # higher temperature implies more randomness when sampling\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "explanation = explainer.explain(text, threshold=0.95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Anchor: emotionally\n",
      "Precision: 0.95\n",
      "\n",
      "Examples where anchor applies and model predicts negative:\n",
      "a visually flashy but narratively opaque and emotionally vapid exercise arround style and mystification .\n",
      "any visually flashy but culturally translucent and emotionally ludicrous exercise in style and mystification .\n",
      "a visually flashy but functionally opaque and emotionally vapid isometric near design and mystification .\n",
      "an visually blocky but stylistically opaque and emotionally vapid exercise in style and immaturity .\n",
      "a visually snazzy but anatomically opaque and emotionally vacuous cardio in fashion and immorality .\n",
      "another aesthetically cumbersome but narratively visible and emotionally vapid weightloss arround style and mystification .\n",
      "a visually gaudy but narratively reflective and emotionally vapid workout in sass and mystification .\n",
      "another visually flashy but stylistically opaque and emotionally boilerplate training around style and mystification .\n",
      "a graphically flashy but strikingly preferable and emotionally banal exercise within sensibility and delusion .\n",
      "a visually flashy but visually visible and emotionally vapid bodybuilding in style and mystification .\n",
      "\n",
      "Examples where anchor applies and model predicts positive:\n",
      "a visually gaudy but narratively opaque and emotionally vapid excercise throughout style and delusion .\n",
      "any visually gaudy but stylistically reflective and emotionally litany exercise inside stylistic and mirage .\n",
      "a visually gaudy but stylistically opaque and emotionally nonsensical p90x throughout style and mystification .\n",
      "a visually clunky but visually translucent and emotionally vapid training of style and immaturity .\n",
      "a visually snazzy but narratively transparent and emotionally litany bodybuilding in style and illusory .\n",
      "a functionally snazzy but graphically solid and emotionally vapid treadmill in dress and mystification .\n",
      "a visually snazzy but graphically transparent and emotionally moronic endurance outside style and lunacy .\n"
     ]
    }
   ],
   "source": [
    "print('Anchor: %s' % (' AND '.join(explanation.anchor)))\n",
    "print('Precision: %.2f' % explanation.precision)\n",
    "print('\\nExamples where anchor applies and model predicts %s:' % pred)\n",
    "print('\\n'.join([x for x in explanation.raw['examples'][-1]['covered_true']]))\n",
    "print('\\nExamples where anchor applies and model predicts %s:' % alternative)\n",
    "print('\\n'.join([x for x in explanation.raw['examples'][-1]['covered_false']]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize language model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Because the Language Model is computationally demanding, we can run it on the GPU. Note that this is optional, and we can run the explainer on a non-GPU machine too."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the code runs for non-GPU machines too\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " We provide support for three transformer-based language models: `DistilbertBaseUncased`, `BertBaseUncased`, and `RobertaBase`. We initialize the language model as follows:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some layers from the model checkpoint at distilbert-base-uncased were not used when initializing TFDistilBertForMaskedLM: ['activation_13']\n",
      "- This IS expected if you are initializing TFDistilBertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFDistilBertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the layers of TFDistilBertForMaskedLM were initialized from the model checkpoint at distilbert-base-uncased.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFDistilBertForMaskedLM for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "# language_model = RobertaBase()\n",
    "# language_model = BertBaseUncased()\n",
    "language_model = DistilbertBaseUncased()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize anchor text explainer with `language_model` sampling (`parallel` filling)\n",
    "\n",
    "* `sampling_strategy='language_model'` means that the words will be sampled according to the output distribution predicted by the language model\n",
    "\n",
    "* `filling='parallel'` means the only one forward pass is performed. The words are the sampled independently of one another."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize explainer\n",
    "explainer = AnchorText(\n",
    "    predictor=predict_fn,\n",
    "    sampling_strategy=\"language_model\",   # use language model to predict the masked words\n",
    "    language_model=language_model,        # language model to be used\n",
    "    filling=\"parallel\",                   # just one pass through the transformer\n",
    "    sample_proba=0.5,                     # probability of masking a word\n",
    "    frac_mask_templates=0.1,              # fraction of masking templates (smaller value -> faster, less diverse)\n",
    "    use_proba=True,                       # use words distribution when sampling (if False sample uniform)\n",
    "    top_n=20,                             # consider the fist 20 most likely words\n",
    "    temperature=1.0,                      # higher temperature implies more randomness when sampling\n",
    "    stopwords=['and', 'a', 'but', 'in'],  # those words will not be sampled\n",
    "    batch_size_lm=32,                     # language model maximum batch size\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
      "WARNING:tensorflow:AutoGraph could not transform <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7f1760325460>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: module, class, method, function, traceback, frame, or code object was expected, got cython_function_or_method\n",
      "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
      "WARNING: AutoGraph could not transform <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7f1760325460>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: module, class, method, function, traceback, frame, or code object was expected, got cython_function_or_method\n",
      "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
      "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
      "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
      "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
      "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
      "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
      "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
      "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
     ]
    }
   ],
   "source": [
    "explanation = explainer.explain(text, threshold=0.95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Anchor: exercise AND emotionally AND flashy\n",
      "Precision: 0.96\n",
      "\n",
      "Examples where anchor applies and model predicts negative:\n",
      "a visually flashy but critically demanding and emotionally demanding exercise in style and mystification.\n",
      "a visually flashy but ultimately passionate and emotionally demanding exercise in style and mystification.\n",
      "a visually flashy but socially creative and emotionally challenging exercise in style and mystification.\n",
      "a visually flashy but emotionally passionate and emotionally stimulating exercise in style and mystification.\n",
      "a visually flashy but emotionally emotionally and emotionally motivated exercise in style and mystification.\n",
      "a visually flashy but socially challenging and emotionally inspired exercise in style and mystification.\n",
      "a visually flashy but emotionally creative and emotionally intense exercise in style and mystification.\n",
      "a visually flashy but emotionally conscious and emotionally inspired exercise in style and mystification.\n",
      "a visually flashy but visually mentally and emotionally vapid exercise in fitness and meditation.\n",
      "a visually flashy but emotionally spirited and emotionally vapid exercise in swimming and gymnastics.\n",
      "\n",
      "Examples where anchor applies and model predicts positive:\n",
      "a visually flashy but politically imaginative and emotionally challenging exercise in style and mystification.\n",
      "a visually flashy but highly provocative and emotionally challenging exercise in style and mystification.\n",
      "a visually flashy but narratively engaging and emotionally satisfying exercise in style and mystification.\n",
      "a visually flashy but narratively challenging and emotionally engaging exercise in style and mystification.\n",
      "a visually flashy but narratively accurate and emotionally engaging exercise in style and mystification.\n",
      "a visually flashy but narratively challenging and emotionally satisfying exercise in style and mystification.\n",
      "a visually flashy but narratively realistic and emotionally satisfying exercise in style and mystification.\n",
      "a visually flashy but visually opaque and emotionally satisfying exercise in beauty and mystification.\n",
      "a typically flashy but narratively accurate and emotionally stimulating exercise in style and mystification.\n",
      "a generally flashy but narratively detailed and emotionally demanding exercise in style and mystification.\n"
     ]
    }
   ],
   "source": [
    "print('Anchor: %s' % (' AND '.join(explanation.anchor)))\n",
    "print('Precision: %.2f' % explanation.precision)\n",
    "print('\\nExamples where anchor applies and model predicts %s:' % pred)\n",
    "print('\\n'.join([x for x in explanation.raw['examples'][-1]['covered_true']]))\n",
    "print('\\nExamples where anchor applies and model predicts %s:' % alternative)\n",
    "print('\\n'.join([x for x in explanation.raw['examples'][-1]['covered_false']]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize anchor text explainer with `language_model` sampling (`autoregressive` filling)\n",
    "\n",
    "* `filling='autoregressive'` means that the words are sampled one at the time (autoregressive). Thus, following words to be predicted will be conditioned one the previously generated words.\n",
    "* `frac_mask_templates=1` in this mode (overwriting it with any other value will not be considered).\n",
    "* **This procedure is computationally expensive**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# initialize explainer\n",
    "explainer = AnchorText(\n",
    "    predictor=predict_fn,\n",
    "    sampling_strategy=\"language_model\",  # use language model to predict the masked words\n",
    "    language_model=language_model,       # language model to be used\n",
    "    filling=\"autoregressive\",            # just one pass through the transformer\n",
    "    sample_proba=0.5,                    # probability of masking a word\n",
    "    use_proba=True,                      # use words distribution when sampling (if False sample uniform)\n",
    "    top_n=20,                            # consider the fist 20 most likely words\n",
    "    stopwords=['and', 'a', 'but', 'in']  # those words will not be sampled\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "explanation = explainer.explain(text, threshold=0.95, batch_size=10, coverage_samples=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Anchor: flashy AND vapid AND exercise AND opaque\n",
      "Precision: 1.00\n",
      "\n",
      "Examples where anchor applies and model predicts negative:\n",
      "a visually flashy but narratively opaque and occasionally vapid exercise in reflection and mystification.\n",
      "a visually flashy but narratively opaque and sometimes vapid exercise in inquiry and mystification.\n",
      "a somewhat flashy but somewhat opaque and emotionally vapid exercise in meditation and mystification.\n",
      "a very flashy but equally opaque and emotionally vapid exercise in style and technique.\n",
      "a visually flashy but visually opaque and emotionally vapid exercise in style and personality.\n",
      "a visually flashy but emotionally opaque and emotionally vapid exercise in style and mystification.\n",
      "a visually flashy but narratively opaque and emotionally vapid exercise in dialogue and storytelling.\n",
      "a visually flashy but narratively opaque and socially vapid exercise in style and substance.\n",
      "a somewhat flashy but narratively opaque and frequently vapid exercise in style and substance.\n",
      "a wholly flashy but narratively opaque and emotionally vapid exercise in experimentation and mystification.\n",
      "\n",
      "Examples where anchor applies and model predicts positive:\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Anchor: %s' % (' AND '.join(explanation.anchor)))\n",
    "print('Precision: %.2f' % explanation.precision)\n",
    "print('\\nExamples where anchor applies and model predicts %s:' % pred)\n",
    "print('\\n'.join([x for x in explanation.raw['examples'][-1]['covered_true']]))\n",
    "print('\\nExamples where anchor applies and model predicts %s:' % alternative)\n",
    "print('\\n'.join([x for x in explanation.raw['examples'][-1]['covered_false']]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}