examples/anchor_text_movie.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Anchor explanations for movie sentiment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this example, we will explain why a certain sentence is classified by a logistic regression as having negative or positive sentiment. The logistic regression is trained on negative and positive movie reviews."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "import spacy\n",
    "from alibi.explainers import AnchorText\n",
    "from alibi.datasets import fetch_movie_sentiment\n",
    "from alibi.utils.download import spacy_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load movie review dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `fetch_movie_sentiment` function returns a `Bunch` object containing the features, the targets and the target names for the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['data', 'target', 'target_names'])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies = fetch_movie_sentiment()\n",
    "movies.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = movies.data\n",
    "labels = movies.target\n",
    "target_names = movies.target_names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define shuffled training, validation and test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, test, train_labels, test_labels = train_test_split(data, labels, test_size=.2, random_state=42)\n",
    "train, val, train_labels, val_labels = train_test_split(train, train_labels, test_size=.1, random_state=42)\n",
    "train_labels = np.array(train_labels)\n",
    "test_labels = np.array(test_labels)\n",
    "val_labels = np.array(val_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Apply CountVectorizer to training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "                dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "                lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "                ngram_range=(1, 1), preprocessor=None, stop_words=None,\n",
       "                strip_accents=None, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       "                tokenizer=None, vocabulary=None)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer = CountVectorizer(min_df=1)\n",
    "vectorizer.fit(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fit model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
       "                   multi_class='auto', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='liblinear', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(0)\n",
    "clf = LogisticRegression(solver='liblinear')\n",
    "clf.fit(vectorizer.transform(train), train_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define prediction function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "predict_fn = lambda x: clf.predict(vectorizer.transform(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make predictions on train and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train accuracy 0.9801624284382905\n",
      "Validation accuracy 0.7544910179640718\n",
      "Test accuracy 0.7589841878294202\n"
     ]
    }
   ],
   "source": [
    "preds_train = predict_fn(train)\n",
    "preds_val = predict_fn(val)\n",
    "preds_test = predict_fn(test)\n",
    "print('Train accuracy', accuracy_score(train_labels, preds_train))\n",
    "print('Validation accuracy', accuracy_score(val_labels, preds_val))\n",
    "print('Test accuracy', accuracy_score(test_labels, preds_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load spaCy model\n",
    "\n",
    "English multi-task CNN trained on OntoNotes, with GloVe vectors trained on Common Crawl. Assigns word vectors, context-specific token vectors, POS tags, dependency parse and named entities."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = 'en_core_web_md'\n",
    "spacy_model(model=model)\n",
    "nlp = spacy.load(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize anchor text explainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "explainer = AnchorText(nlp, predict_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Explain a prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_names = movies.target_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a visually flashy but narratively opaque and emotionally vapid exercise in style and mystification .\n"
     ]
    }
   ],
   "source": [
    "text = data[4]\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prediction:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction: negative\n"
     ]
    }
   ],
   "source": [
    "pred = class_names[predict_fn([text])[0]]\n",
    "alternative =  class_names[1 - predict_fn([text])[0]]\n",
    "print('Prediction: %s' % pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Explanation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(0)\n",
    "explanation = explainer.explain(text, threshold=0.95, use_unk=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "use_unk=True means we will perturb examples by replacing words with UNKs. Let us now take a look at the anchor. The word 'exercise' basically guarantees a negative prediction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Anchor: flashy\n",
      "Precision: 0.99\n",
      "\n",
      "Examples where anchor applies and model predicts negative:\n",
      "a UNK flashy UNK UNK opaque and emotionally vapid exercise in style UNK mystification .\n",
      "a UNK flashy UNK UNK UNK and emotionally UNK exercise UNK UNK and UNK UNK\n",
      "a UNK flashy UNK narratively opaque UNK UNK UNK exercise in style and UNK UNK\n",
      "UNK visually flashy UNK narratively UNK and emotionally UNK UNK UNK UNK UNK mystification .\n",
      "UNK UNK flashy UNK UNK opaque and emotionally UNK UNK in UNK and UNK .\n",
      "a visually flashy but UNK UNK and UNK UNK UNK in style UNK mystification .\n",
      "a visually flashy but UNK opaque UNK emotionally vapid UNK in UNK and mystification .\n",
      "a UNK flashy but narratively UNK UNK emotionally vapid exercise in style UNK mystification UNK\n",
      "a UNK flashy but narratively opaque UNK emotionally vapid exercise in style and mystification .\n",
      "a visually flashy UNK UNK opaque UNK UNK UNK exercise in UNK UNK UNK .\n",
      "\n",
      "Examples where anchor applies and model predicts positive:\n",
      "UNK UNK flashy but narratively UNK and UNK UNK UNK in style and UNK UNK\n"
     ]
    }
   ],
   "source": [
    "print('Anchor: %s' % (' AND '.join(explanation.anchor)))\n",
    "print('Precision: %.2f' % explanation.precision)\n",
    "print('\\nExamples where anchor applies and model predicts %s:' % pred)\n",
    "print('\\n'.join([x for x in explanation.raw['examples'][-1]['covered_true']]))\n",
    "print('\\nExamples where anchor applies and model predicts %s:' % alternative)\n",
    "print('\\n'.join([x for x in explanation.raw['examples'][-1]['covered_false']]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Changing the perturbation distribution\n",
    "Let's try this with another perturbation distribution, namely one that replaces words by similar words instead of UNKs.\n",
    "\n",
    "Explanation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(0)\n",
    "explanation = explainer.explain(text, threshold=0.95, use_unk=False, sample_proba=0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The anchor now shows that we need more to guarantee the negative prediction:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Anchor: exercise AND emotionally\n",
      "Precision: 0.97\n",
      "\n",
      "Examples where anchor applies and model predicts negative:\n",
      "a visually ballsy but philosophically opaque and emotionally vapid exercise if signature and negation .\n",
      "a instantly flashy but narratively opaque and emotionally vapid exercise than style and mystification .\n",
      "a visually flashy but narratively opaque and emotionally vapid exercise in style and fear .\n",
      "some visually quirky but narratively dainty and emotionally patronising exercise until style and hopelessness .\n",
      "another visually decorate but narratively opaque and emotionally vapid exercise in outfit and mystification .\n",
      "both curiously unwieldy but wonderfully hollow and emotionally ridiculous exercise in style and mystification .\n",
      "a visually artsy but gloriously opaque and emotionally vapid exercise in style and mystification .\n",
      "a visually eclectic but narratively opaque and emotionally vapid exercise in vogue and mystification .\n",
      "another stunningly flashy but narratively bright and emotionally unscientific exercise on style and falsehood .\n",
      "a exceptionally whimsical but disturbingly opaque and emotionally vapid exercise about vibe and mystification .\n",
      "\n",
      "Examples where anchor applies and model predicts positive:\n",
      "both visually unconventional but socially opaque and emotionally caricature exercise around style and woe .\n",
      "both wonderfully artsy but similarly opaque and emotionally vapid exercise in style and mystification .\n",
      "a perfectly groovy but supremely smooth and emotionally vapid exercise towards style and mystification .\n",
      "a visually stylish but narratively truthful and emotionally moronic exercise in style and oxymoron .\n",
      "any remarkably inventive but narratively opaque and emotionally babble exercise despite style and naivety .\n"
     ]
    }
   ],
   "source": [
    "print('Anchor: %s' % (' AND '.join(explanation.anchor)))\n",
    "print('Precision: %.2f' % explanation.precision)\n",
    "print('\\nExamples where anchor applies and model predicts %s:' % pred)\n",
    "print('\\n'.join([x for x in explanation.raw['examples'][-1]['covered_true']]))\n",
    "print('\\nExamples where anchor applies and model predicts %s:' % alternative)\n",
    "print('\\n'.join([x for x in explanation.raw['examples'][-1]['covered_false']]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can make the token perturbation distribution sample words that are more similar to the ground truth word via the `top_n` argument. Smaller values (default=100) should result in sentences that are more coherent and thus more in the distribution of natural language which could influence the returned anchor. By setting the `use_probability_proba` to True, the sampling distribution for perturbed tokens is proportional to the similarity score between the possible perturbations and the original word. We can also put more weight on similar words via the `temperature` argument. Lower values of `temperature` increase the sampling weight of more similar words. The following example will perturb tokens in the original sentence with probability equal to `sample_proba`. The sampling distribution for the perturbed tokens is proportional to the similarity score between the ground truth word and each of the `top_n` words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Anchor: exercise AND emotionally\n",
      "Precision: 0.98\n",
      "\n",
      "Examples where anchor applies and model predicts negative:\n",
      "a visually exquisite but narratively opaque and emotionally vapid exercise before style and mystification .\n",
      "each mechanically eccentric but narratively transparent and emotionally unremarkable exercise in style and falsehood .\n",
      "a incredibly extravagant but artistically bright and emotionally vapid exercise of style and mystification .\n",
      "any visually shiny but artistically glide and emotionally vapid exercise around temperament and materialism .\n",
      "another clearly flashy but aesthetically opaque and emotionally vapid exercise whether flair and mystification .\n",
      "a visually snazzy but narratively opaque and emotionally mindless exercise within style and negation .\n",
      "a visually ingenious but narratively opaque and emotionally unimaginative exercise in artistry and mystification .\n",
      "a visually flashy but narratively colorful and emotionally vapid exercise than style and mystification .\n",
      "a graphically punchy but narratively opaque and emotionally vapid exercise of vibe and insanity .\n",
      "a artistically flashy but narratively opaque and emotionally vapid exercise in ballroom and mystification .\n",
      "\n",
      "Examples where anchor applies and model predicts positive:\n",
      "any vividly outlandish but supremely opaque and emotionally vapid exercise throughout streetwear and mystification .\n",
      "another precisely elaborate but delightfully realistic and emotionally muddled exercise in brevity and paranoia .\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(0)\n",
    "explanation = explainer.explain(text, threshold=0.95, use_similarity_proba=True, sample_proba=0.5,\n",
    "                                use_unk=False, top_n=20, temperature=.2)\n",
    "\n",
    "print('Anchor: %s' % (' AND '.join(explanation.anchor)))\n",
    "print('Precision: %.2f' % explanation.precision)\n",
    "print('\\nExamples where anchor applies and model predicts %s:' % pred)\n",
    "print('\\n'.join([x for x in explanation.raw['examples'][-1]['covered_true']]))\n",
    "print('\\nExamples where anchor applies and model predicts %s:' % alternative)\n",
    "print('\\n'.join([x for x in explanation.raw['examples'][-1]['covered_false']]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}