nl_explanations_exercise.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "nl-explanations_exercise",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/SIDN-IAP/nl-explanations/blob/master/nl_explanations_exercise.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PUh-ToqPfLc6",
        "colab_type": "text"
      },
      "source": [
        "Set up the environment (this will take a minute):"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "tmI5yq9vebIW",
        "colab": {}
      },
      "source": [
        "%%bash\n",
        "!(stat -t /usr/local/lib/*/dist-packages/google/colab > /dev/null 2>&1) && exit \n",
        "pip install ninja\n",
        "wget http://lingo.csail.mit.edu/demos/sidn-iap-2020/nl-explanations.tar.gz\n",
        "tar xzf nl-explanations.tar.gz"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MljjbPEaTFKv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import sys\n",
        "sys.path.append(\"/content/nl-explanations\")\n",
        "import pickle\n",
        "import random\n",
        "import collections\n",
        "\n",
        "import torch\n",
        "import torch.nn.functional as F\n",
        "import numpy as np\n",
        "\n",
        "import lab\n",
        "from lab import ImageCaptioner, ImageClassifier, TextClassifier, ImageRanker\n",
        "import torchdec\n",
        "\n",
        "from IPython.display import display, HTML\n",
        "\n",
        "DEVICE = torch.device(\"cpu\")\n",
        "lab.DEVICE = DEVICE"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1VAQ6fqofvlM",
        "colab_type": "text"
      },
      "source": [
        "Load models and data:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "52SVcQJmrmXF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open(\"/content/nl-explanations/vocab.json\") as reader:\n",
        "  vocab = torchdec.Vocab()\n",
        "  vocab.load(reader)\n",
        "\n",
        "with open(\"/content/nl-explanations/CUB_test_features.p\", \"rb\") as reader:\n",
        "  test_features = pickle.load(reader)\n",
        "\n",
        "# Keep 1000 only\n",
        "test_features = {k: test_features[k] for i, k in zip(range(1000), test_features)}\n",
        "test_names, test_images = zip(*test_features.items())\n",
        "test_images = torch.tensor(test_images)\n",
        "\n",
        "test_classes = collections.defaultdict(list)\n",
        "for name in test_names:\n",
        "  cls = name.split('.')[0]\n",
        "  test_classes[int(cls)].append(name)\n",
        "\n",
        "classifier = ImageClassifier()\n",
        "classifier.load_state_dict(torch.load(\"/content/nl-explanations/image_classifier-1.0.m\", map_location=DEVICE))\n",
        "classifier.eval()\n",
        "\n",
        "captioner = ImageCaptioner(vocab)\n",
        "captioner.load_state_dict(torch.load(\"/content/nl-explanations/image_captioner-1.0.m\", map_location=DEVICE))\n",
        "captioner.eval()\n",
        "\n",
        "text_classifier = TextClassifier(vocab)\n",
        "text_classifier.load_state_dict(torch.load(\"/content/nl-explanations/text_classifier.m\", map_location=DEVICE))\n",
        "text_classifier.eval()\n",
        "\n",
        "image_ranker = ImageRanker(vocab)\n",
        "image_ranker.load_state_dict(torch.load(\"/content/nl-explanations/image_ranker.m\", map_location=DEVICE))\n",
        "image_ranker.eval()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KnnE4an7FJcY",
        "colab_type": "text"
      },
      "source": [
        "Function for visualizing model predictions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hb6wLpLDUOQs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def url(key):\n",
        "  return f'http://lingo.csail.mit.edu/demos/sidn-iap-2020/CUB_200_2011/images/{key}'\n",
        "\n",
        "def show_classifier_predictions(classifier, explainers={}, n=10, with_class=None):\n",
        "  if not isinstance(classifier, dict):\n",
        "    classifier = {\"model\": classifier}\n",
        "  keys = list(enumerate(test_names[:n]))\n",
        "  for i, key in keys:\n",
        "    true_label, name = key.split(\"/\")[0].split(\".\")\n",
        "    true_label = int(true_label)\n",
        "    if with_class is not None and true_label not in with_class:\n",
        "      continue\n",
        "    name = name.replace(\"_\", \" \").lower()\n",
        "    features = torch.tensor([test_features[key]], device=DEVICE)\n",
        "    \n",
        "    display(HTML(f\"<img src='{url(key)}' width=200/>\"))\n",
        "    display(HTML(f\"<p><b>True label</b> {true_label} ({name})</p>\"))\n",
        "    for classifier_name in classifier:\n",
        "      scores = classifier[classifier_name](features)\n",
        "      pred_label = torch.argmax(scores, dim=1).item()\n",
        "      display(HTML(f\"<p><b>Predicted label ({classifier_name})</b> {pred_label}</p>\"))\n",
        "\n",
        "    for name, explainer in explainers.items():\n",
        "      explanation = explainer(i, features, pred_label)\n",
        "      display(HTML(f\"<p><b>({name}) because...</b>{explanation}</p>\"))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S_rUUeiggD4j",
        "colab_type": "text"
      },
      "source": [
        "## Fine-grained classification\n",
        "\n",
        "As a running example in this lab, we'll look at a model trained for a fine-grained bird species classification task. This is the [Caltech-UCSD Birds Dataset](http://www.vision.caltech.edu/visipedia/CUB-200-2011.html), a common benchmark for fine-grained and few-shot visual classification. It contains 200 species of birds, each with 40-60 images.\n",
        "\n",
        "We'd like this classifier to be able to _explain_ its predictions by describing the features of the input birds that are relevant to its classification decision.\n",
        "\n",
        "As a first step, let's look at some predictions from this classifier:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8KwNh5m3gDV5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "show_classifier_predictions(classifier)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6oitgDHJh28_",
        "colab_type": "text"
      },
      "source": [
        "## Generating natural language explanations\n",
        "\n",
        "There are a number of fine-grained distinctions between classes. Different features of birds can be discriminative for different reasons (e.g. the color of a wing might be important in one context, and the wing's shape in a different context.) We would like a tool for generating explanations that can make these fine-grained distinctions; the main idea in this lab is that language might provide such a tool.\n",
        "\n",
        "The Caltech Birds dataset is also accompanied by [natural language descriptions of all the birds](https://github.com/reedscot/cvpr2016). This language data will serve as the basis for the explanation techniques we develop.\n",
        "\n",
        "## Image captions as explanations\n",
        "\n",
        "The simplest way we can imagine using the bird descriptions is just to train a model to generate descriptions, and show these descriptions alongside explanations. We'll let the captioning model condition _on the classifier's representation of the image_, rather than raw pixels, so classifier behavior is already partly tied to model behavior.\n",
        "\n",
        "(For more information about image captioning models, check out https://arxiv.org/abs/1411.4389.)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X27g_SIEf0lU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def naive_explanation(i, features, pred_label):\n",
        "  (caption,), _ = captioner.decode(features, greedy=True)\n",
        "  return \" \".join(vocab.decode(caption))\n",
        "\n",
        "show_classifier_predictions(classifier, {\"naive explanation\": naive_explanation})"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qbtUFTRpiPzc",
        "colab_type": "text"
      },
      "source": [
        "**Question**: Are these captions explanations at all? (What's the relationship between captions and the classifier they're supposed to explain?)\n",
        "\n",
        "To answer this, we can start by exploring a related question: Can we distinguish good classifiers from bad classifiers on the basis of the quality of their descriptions?\n",
        "\n",
        "**Exercise 1**: Below we've trained two extra image captioning models. One is slightly worse than the model used above, and one is significantly worse. We've also trained models to generate bird descriptions from the representations computed by each of these models. Can you tell which model has the highest-quality representations based on the quality of the associated explanations?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LUGBlMktiuTc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ok_classifier = ImageClassifier()\n",
        "ok_classifier.load_state_dict(torch.load(\"/content/nl-explanations/image_classifier-0.03.m\", map_location=DEVICE))\n",
        "ok_captioner = ImageCaptioner(vocab)\n",
        "ok_captioner.load_state_dict(torch.load(\"/content/nl-explanations/image_captioner-0.03.m\", map_location=DEVICE))\n",
        "\n",
        "bad_classifier = ImageClassifier()\n",
        "bad_classifier.load_state_dict(torch.load(\"/content/nl-explanations/image_classifier-0.01.m\", map_location=DEVICE))\n",
        "bad_captioner = ImageCaptioner(vocab)\n",
        "bad_captioner.load_state_dict(torch.load(\"/content/nl-explanations/image_captioner-0.01.m\", map_location=DEVICE))\n",
        "\n",
        "def naive_explanation_for_captioner(captioner):\n",
        "  def fn(i, features, pred_label):\n",
        "    (caption,), _ = captioner.decode(features, greedy=False)\n",
        "    return \" \".join(vocab.decode(caption))\n",
        "  return fn\n",
        "\n",
        "show_classifier_predictions(\n",
        "  {\"good\": classifier, \"ok\": ok_classifier, \"bad\": bad_classifier},\n",
        "  {\"good\": naive_explanation, \"ok\": naive_explanation_for_captioner(ok_captioner), \"bad\": naive_explanation_for_captioner(bad_captioner)},\n",
        "  n=10\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FgrfTdzYisOd",
        "colab_type": "text"
      },
      "source": [
        "So we can already see that captions provide a proxy signal for model quality (even though the \"good\" and \"ok\" models get roughly the same accuracy on the test set we've been looking at!)\n",
        "\n",
        "\n",
        "However: suppose we want to understand _why_ a given class label has been assigned. Is the current approach to textual explanations good enough? Consider following examples:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LS7GIOKoi_B8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "show_classifier_predictions(classifier, {\"naive explanation\": naive_explanation}, n=200, with_class={88, 157})"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aqnFA6LojBIv",
        "colab_type": "text"
      },
      "source": [
        "Both descriptions apply reasonably well to both birds!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fE82Lo0I9jb1",
        "colab_type": "text"
      },
      "source": [
        "## Class-discriminative captions\n",
        "\n",
        "We should pick captions that not only apply to the given image, but that are *informative about the class label*.\n",
        "\n",
        "In the code below, we've trained a model (`text_classifier`) that assigns a probability to each bird class based on _description text alone_. \n",
        "\n",
        "Make sure you understand what the code below does. How does adding a text classifier term change the kind of explanations that get generated?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jr01rjXRiE2_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def discriminative_explanation(n_samples=20):\n",
        "  def fn(i, features, pred_label):\n",
        "    feature_batch = torch.cat([features] * n_samples)\n",
        "    captions, caption_scores = captioner.decode(feature_batch, greedy=False)\n",
        "    caption_batch = torchdec.batch_seqs(captions)\n",
        "    classifier_scores = text_classifier(caption_batch)[:, pred_label]\n",
        "\n",
        "    combined_scores = classifier_scores + torch.tensor(caption_scores)\n",
        "    best_caption_index = combined_scores.argmax()\n",
        "    return \" \".join(vocab.decode(captions[best_caption_index]))\n",
        "  return fn\n",
        "\n",
        "show_classifier_predictions(\n",
        "    classifier,\n",
        "    {\n",
        "        \"naive\": naive_explanation,\n",
        "        \"discriminative\": discriminative_explanation(),\n",
        "    }\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wC-lzmhfHOBE",
        "colab_type": "text"
      },
      "source": [
        "**Exercise 2**: how does changing the number of samples used in the reranking step affect explanation quality?\n",
        "\n",
        "**Exercise 3**: Add a parameter that trades off the weight of the image captioner and text classifier terms in the objective above. How does this weight affect explanation quality?"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E0Gh-gLTFcX1",
        "colab_type": "text"
      },
      "source": [
        "## Image-discriminative captions\n",
        "\n",
        "The explanations above are discriminative of the class of the bird, but not necessarily the actual contents of the image itself. We can circumvent this by generating descriptions that are not just discriminative of the birds class, but are discriminative of images as well."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yiCHj2kTGBvU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def image_discriminative_explanation(class_weight, image_weight, n_samples=20, n_distractors=10):\n",
        "  def fn(i, features, pred_label):\n",
        "    feature_batch = torch.cat([features] * n_samples)\n",
        "    captions, caption_scores = captioner.decode(feature_batch, greedy=False)\n",
        "    caption_batch = torchdec.batch_seqs(captions)\n",
        "    text_classifier_scores = text_classifier(caption_batch)[:, pred_label]\n",
        "\n",
        "    # Sample some distractor images and score them\n",
        "    distractors = random.sample(list(test_features.values()), n_distractors)\n",
        "    image_scores = torch.zeros((n_samples, n_distractors + 1))\n",
        "    image_scores[:, 0] = image_ranker.score((caption_batch, feature_batch))\n",
        "    for i, distractor in enumerate(distractors, start=1):\n",
        "        distractor_t = torch.tensor([distractor], device=DEVICE)\n",
        "        distractor_batch = torch.cat([distractor_t] * n_samples)\n",
        "        image_scores[:, i] =  image_ranker.score((caption_batch, distractor_batch))\n",
        "        \n",
        "    # Normalize over distractors\n",
        "    image_scores = F.log_softmax(image_scores, dim=1)[:, 0]\n",
        "\n",
        "    combined_scores = image_weight * image_scores + class_weight * text_classifier_scores + torch.tensor(caption_scores)\n",
        "    best_caption_index = combined_scores.argmax()\n",
        "    return \" \".join(vocab.decode(captions[best_caption_index]))\n",
        "  return fn"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_5Jl4bPaGLIn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "show_classifier_predictions(\n",
        "    classifier,\n",
        "    {\n",
        "        \"naive\": naive_explanation,\n",
        "        \"class-discriminative-1.0\": discriminative_explanation(),\n",
        "        \"image-discriminative-1.0\": image_discriminative_explanation(1.0, 2.0),\n",
        "    }\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AR5jATeEb3oq",
        "colab_type": "text"
      },
      "source": [
        "**Exercise 4**: can you get even higher-quality explanations by changing the way these terms are combined?\n",
        "\n",
        "**Exercise 5**: how would you use these models to produce explanations for counterfactual classes? What about counterfactual inputs?"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1f_1BvpuqDSj",
        "colab_type": "text"
      },
      "source": [
        "## Feature-discriminative captions\n",
        "\n",
        "There's one final issue with the approach described above: an explanation might\n",
        "discriminate both the class and the input image, by talking about features\n",
        "that the model doesn't actually use to make decisions.\n",
        "\n",
        "To address this issue, we'll introduce a final explanation technique that identifies a set of images whose _model representations_ are similar to the input image, and then generates an explanation summarizing what all those images have in common.\n",
        "\n",
        "We'll start by computing similarity between images in representation space. We'll call an image with representation $x$ **similar** to image with representation $y$ if the Euclidean distance $d(x, y)$ is within the top 5% closest images in our dataset."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oGgmiDA-mFR6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from scipy.spatial.distance import squareform, pdist\n",
        "\n",
        "# Get representations\n",
        "all_test_reps = captioner.proj(test_images).detach().cpu().numpy()\n",
        "\n",
        "# Compute pairwise similarity\n",
        "dists = pdist(all_test_reps)\n",
        "dists = squareform(dists)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5m05sI49mvwq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def similar_to_image(i, alpha=0.05):\n",
        "  imgs = [(j, dist) for j, dist in enumerate(dists[i]) if j != i]\n",
        "  img_dists = [i[1] for i in imgs]\n",
        "  threshold = np.quantile(img_dists, alpha)\n",
        "  most_sim = [i for i in imgs if i[1] < threshold]\n",
        "  return sorted(most_sim, key=lambda x: x[1])\n",
        "  \n",
        "def show_similar_to_image(i, n=5):\n",
        "  x = test_names[i]\n",
        "  ys = similar_to_image(i, alpha=0.05)[:n]\n",
        "  ys = [(test_names[j], dist) for j, dist in ys]\n",
        "  display(HTML(f\"<p><b>Most similar images to {x}</b></p>\"))\n",
        "  display(HTML(f\"<img src='{url(x)}'' width=200/>\"))\n",
        "  for y, y_dist in ys:\n",
        "    display(HTML(f\"<p><b>{y}</b> ({y_dist:.3f})\"))\n",
        "    display(HTML(f\"<img src='{url(y)}' width=200/>\"))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KGMh38O0rb9Y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "show_similar_to_image(2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WoZF4fhatUI_",
        "colab_type": "text"
      },
      "source": [
        "Similarly, let's make a function that, given a textual description, computes the top 5% of matching images:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zeuJog9gsu5g",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def similar_to_text(caption, alpha=0.05):\n",
        "  caption_batch = caption.unsqueeze(0).expand(test_images.shape[0], caption.shape[0])\n",
        "  caption_batch = torchdec.batch_seqs(caption_batch)\n",
        "  image_scores = image_ranker.score((caption_batch, test_images))\n",
        "  image_scores = image_scores.detach().cpu().numpy()\n",
        "  # Higher is better\n",
        "  threshold = np.quantile(image_scores, 1 - alpha)\n",
        "  most_sim = [(j, score) for j, score in enumerate(image_scores) if score > threshold]\n",
        "  return sorted(most_sim, key=lambda x: -x[1])\n",
        "\n",
        "def show_similar_to_text(caption, n=5):\n",
        "  cap = torch.tensor(vocab.encode(caption.split()))\n",
        "  ys = similar_to_text(cap, alpha=0.05)[:n]\n",
        "  ys = [(test_names[j], dist) for j, dist in ys]\n",
        "  display(HTML(f\"<p><b>Most similar images to</b> <span style='font-family: monospace'>{caption}</span></p>\"))\n",
        "  for y, y_dist in ys:\n",
        "    display(HTML(f\"<p><b>{y}</b> ({y_dist:.3f})\"))\n",
        "    display(HTML(f\"<img src='{url(y)}' width=200/>\"))\n",
        "  \n",
        "show_similar_to_text('this bird is yellow')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "prVaIFiI6Oq8",
        "colab_type": "text"
      },
      "source": [
        "We claim that a good natural language explanation for a representation $x$ is the string $w$ that describes common features of all images with representations similar to $x$.\n",
        "\n",
        "If representations $x$ are associated with a ''neighborhood'' $N_{\\text{img}}(x)$ and descriptions $w$ with a neighborhood $N_{\\text{text}}(w)$, we explain $x$\n",
        "by finding $w$ so the two neighborhoods are similar (e.g. using [IoU](https://en.wikipedia.org/wiki/Jaccard_index)):\n",
        "$$\n",
        "w = \\text{argmax}_{w'}\\, \\text{IoU} \\left( N_{\\text{img}}(x), N_{\\text{text}}(w') \\right)\n",
        "$$\n",
        "\n",
        "(How is this similar to the network dissection lab from earlier in the course?)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "srrnhs9FGccA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from scipy.spatial.distance import jaccard\n",
        "from tqdm import tqdm\n",
        "\n",
        "def representation_explanation(class_weight, iou_weight, n_samples=20, n_distractors=10, alpha=0.10):\n",
        "  def fn(i, features, pred_label):\n",
        "    feature_batch = torch.cat([features] * n_samples)\n",
        "    captions, caption_scores = captioner.decode(feature_batch, greedy=False)\n",
        "    caption_batch = torchdec.batch_seqs(captions)\n",
        "    text_classifier_scores = text_classifier(caption_batch)[:, pred_label]\n",
        "\n",
        "    # Compute similarity to other representations\n",
        "    i_dists = dists[i]\n",
        "    threshold = np.quantile(i_dists, alpha)\n",
        "    img_hits = i_dists < threshold\n",
        "\n",
        "    ious = torch.zeros(n_samples)\n",
        "    # For each caption, compute similar images + iou\n",
        "    for c, caption in enumerate(tqdm(captions, desc='Computing caption IoU')):\n",
        "      caption = [caption] * len(test_images)\n",
        "      caption_batch = torchdec.batch_seqs(caption)\n",
        "      scores = image_ranker.score((caption_batch, test_images))\n",
        "      scores = scores.detach().cpu().numpy()\n",
        "      threshold = np.quantile(scores, 1 - alpha)\n",
        "      cap_hits = scores > threshold\n",
        "      iou = 1 - jaccard(img_hits, cap_hits)\n",
        "      ious[c] = iou\n",
        "    print(f\"Iou mean: {ious.mean().item()}\")\n",
        "    print(f\"Iou min: {ious.min().item()}\")\n",
        "    print(f\"Iou max: {ious.max().item()}\")\n",
        "    \n",
        "    combined_scores = iou_weight * ious + class_weight * text_classifier_scores + torch.tensor(caption_scores)\n",
        "    best_caption_index = combined_scores.argmax()\n",
        "    return \" \".join(vocab.decode(captions[best_caption_index]))\n",
        "  return fn"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MzBu1IxSY3qk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "show_classifier_predictions(\n",
        "    classifier,\n",
        "    {\n",
        "        \"naive\": naive_explanation,\n",
        "        \"representation-discriminative-1.0\": representation_explanation(1.0, 100.0),\n",
        "    }\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dW9oGi0oekPJ",
        "colab_type": "text"
      },
      "source": [
        "**Exercise 6**: Again, play with the weights for different terms and see how they affect explanations."
      ]
    }
  ]
}