snli_input_marge_v2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of Copy of input_marge_v2.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ronakdm/input-marginalization/blob/main/snli_input_marge_v2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AxXBIE5Cw7dT"
      },
      "source": [
        "%%capture\n",
        "!pip install pytorch_pretrained_bert\n",
        "!pip install transformers\n",
        "!rm -rf input-marginalization\n",
        "!git clone https://github.com/ronakdm/input-marginalization.git"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GczF6JdTxhZo",
        "outputId": "49e23a31-dfbb-40c5-ae9c-a0fbad6e99eb"
      },
      "source": [
        "%%bash\n",
        "cd input-marginalization\n",
        "git pull\n",
        "cd .."
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Already up to date.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2NYs_qKDxnST"
      },
      "source": [
        "import sys\n",
        "sys.path.append(\"input-marginalization\")\n",
        "import torch\n",
        "from pytorch_pretrained_bert import BertTokenizer, BertModel, BertForMaskedLM\n",
        "from transformers import BertTokenizer, BertModel\n",
        "from utils import generate_snli_dataloader, SNLIDataset\n",
        "from models import LSTM\n",
        "from torch.nn import LogSoftmax\n",
        "import math\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import DataLoader\n",
        "from metrics import continuous_colored_sentence"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i2T8Fwh5xqqr",
        "outputId": "eb933081-49d3-4669-e872-70959495d561"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive',force_remount=True)\n",
        "save_dir = \"/content/gdrive/My Drive/NLP/imarg\""
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r3XZW2qJx1qy"
      },
      "source": [
        "SAMPLE_SIZE = 15\n",
        "SIGMA = 1e-4\n",
        "log_softmax = LogSoftmax(dim=0)"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rAwYhg_Dy5SC"
      },
      "source": [
        "%%capture\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "bert_model = BertForMaskedLM.from_pretrained('bert-base-uncased')"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w8bCqegYx4yk"
      },
      "source": [
        "model = torch.load(f\"{save_dir}/lstm_snli.pt\")"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dtx2h5-7y0Sk"
      },
      "source": [
        "def loaddata():\n",
        "  test_dataset = SNLIDataset('input-marginalization/preprocessed_data/SNLI/snli_1.0/snli_test_string.pkl')\n",
        "  print(test_dataset.le.classes_)\n",
        "  test_dataloader = DataLoader(\n",
        "        test_dataset, batch_size=1, shuffle=True\n",
        "    )\n",
        "  return test_dataloader"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g3vC5A-8iJey"
      },
      "source": [
        "def compute_probability(model, sentences, toktype_mask, label):\n",
        "    s1 = sentences.T[toktype_mask == 0].T\n",
        "    s2 = sentences.T[toktype_mask == 1].T\n",
        "\n",
        "    logits = model((s1, s2), labels=None)\n",
        "    probabilitydist = F.softmax(logits, dim=1)\n",
        "    return torch.reshape(probabilitydist[:, label], (-1,))"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "itGpt67FymBv"
      },
      "source": [
        "def calculate_woe(model, sentences, label, sigma):\n",
        "  device = \"cuda\" if next(model.parameters()).is_cuda else \"cpu\"\n",
        "  bert_model.to(device)\n",
        "  model.to(device)\n",
        "\n",
        "  print(sentences[0], sentences[1])\n",
        "  tok = tokenizer(sentences[0], sentences[1], return_tensors=\"pt\")\n",
        "  input_ids = tok['input_ids'].to(device)\n",
        "  toktype = tok['token_type_ids'].to(device)[0]\n",
        "\n",
        "  #predictions is the probability distribution of each word in the vocabulary for each word in input sentence\n",
        "  predictions = bert_model(input_ids, token_type_ids=toktype.unsqueeze(0))\n",
        "  predictions = torch.squeeze(predictions)\n",
        "  predictions = F.softmax(predictions, dim=1)\n",
        "\n",
        "  #woe is the weight of evidence\n",
        "  woe = []\n",
        "  model.eval()\n",
        "\n",
        "  with torch.no_grad():\n",
        "    for j in range (len(predictions)):\n",
        "      word_scores = predictions[j]\n",
        "      input_batch = input_ids.clone().to(device)\n",
        "      \n",
        "      #word_scores_batch calculates the value of the MLM of Bert for each masked word\n",
        "      #we put 0 for the first input which is unmasked\n",
        "      word_scores_batch = [0]\n",
        "\n",
        "      for k in range(len(word_scores)):\n",
        "        if word_scores[k] > sigma:\n",
        "           input_batch = torch.cat((input_batch, input_ids), 0)\n",
        "           input_batch[-1][j] = k\n",
        "           word_scores_batch.append(word_scores[k].item())\n",
        "      \n",
        "      #probability_input calculates the p(label|sentence) of the target model given each masked input sentence\n",
        "      probability_input = compute_probability(model, input_batch, toktype, label)\n",
        "      m = torch.dot(torch.tensor(word_scores_batch).to(device), probability_input)\n",
        "      logodds_input = math.log(probability_input[0] / (1-probability_input[0]))\n",
        "      logodds_m = math.log(m / (1-m))\n",
        "      woe.append(logodds_input-logodds_m)\n",
        "\n",
        "  woe = torch.tensor(woe).to(device)\n",
        "  return (input_ids[0][toktype==0], woe[toktype == 0]), (torch.cat([torch.tensor(input_ids[0][toktype==0][-1]).unsqueeze(0), input_ids[0][toktype==1]]), torch.cat([torch.tensor(woe[toktype==0][-1]).unsqueeze(0), woe[toktype==1]]))\n"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "68yzg0lNIGj4"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5qcDFTbBymIk"
      },
      "source": [
        "def input_marg(model): \n",
        "  test_data = loaddata()\n",
        "  device = \"cuda\" if next(model.parameters()).is_cuda else \"cpu\"\n",
        "  iter_data = iter(test_data)\n",
        "\n",
        "  for i in range(SAMPLE_SIZE):\n",
        "    curr = next(iter_data)\n",
        "    sentences, labels = curr\n",
        "    \n",
        "    print(\"\")\n",
        "    (s1, a1), (s2, a2) = calculate_woe(model, sentences, labels, SIGMA)\n",
        "    # print(tokenizer.convert_ids_to_tokens(s1))\n",
        "    # print(a1)\n",
        "    print(tokenizer.convert_ids_to_tokens(s2))\n",
        "    # print(a2)\n",
        "    # print(labels)\n",
        "    print('pre: ', continuous_colored_sentence(s1.unsqueeze(0), a1.unsqueeze(0),pretok=True, verbose=False))\n",
        "    print('hypo: ', continuous_colored_sentence(s2.unsqueeze(0), a2.unsqueeze(0),pretok=True, verbose=False))"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fXG0ScGmyviL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "75c5fe73-b1e5-4544-92cf-dd4f4ef5fc99"
      },
      "source": [
        "input_marg(model)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "('Shirtless man with long pole navigates covered boat down a palm tree lined river past a hut.',) ('A man without a shirt is on a river.',)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:43: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "['[CLS]', 'shirt', '##less', 'man', 'with', 'long', 'pole', 'navigate', '##s', 'covered', 'boat', 'down', 'a', 'palm', 'tree', 'lined', 'river', 'past', 'a', 'hut', '.', '[SEP]']\n",
            "tensor([ 1.5137e-01,  1.8670e-01,  8.4433e-02,  5.0246e-04,  6.8416e-04,\n",
            "        -4.3368e-04,  1.6930e-02,  3.1362e-02,  9.1456e-03,  2.2176e-02,\n",
            "         2.4020e-03,  3.9358e-04,  8.6103e-05,  3.6162e-03,  1.1089e-03,\n",
            "         1.0514e-02,  1.8539e-03,  1.2440e-02,  1.2084e-04,  1.1828e-03,\n",
            "         9.9907e-05,  3.0098e-02], device='cuda:0')\n",
            "['[SEP]', 'a', 'man', 'without', 'a', 'shirt', 'is', 'on', 'a', 'river', '.', '[SEP]']\n",
            "tensor([ 3.0098e-02,  1.3502e-03,  2.4498e-04,  3.3729e-03,  3.6641e-04,\n",
            "         8.7076e-04,  2.4917e-03,  2.3083e-04,  5.7137e-05,  2.4577e-03,\n",
            "         2.4633e-04, -1.2929e-01], device='cuda:0')\n",
            "tensor([2])\n",
            "pre:   \u001b[48;2;255;48;48mshirt\u001b[0m\u001b[48;2;255;0;0mless\u001b[0m \u001b[48;2;255;139;139mman\u001b[0m \u001b[48;2;255;254;254mwith\u001b[0m \u001b[48;2;255;254;254mlong\u001b[0m \u001b[48;2;0;0;255mpole\u001b[0m \u001b[48;2;255;231;231mnavigate\u001b[0m\u001b[48;2;255;212;212ms\u001b[0m \u001b[48;2;255;242;242mcovered\u001b[0m \u001b[48;2;255;224;224mboat\u001b[0m \u001b[48;2;255;251;251mdown\u001b[0m \u001b[48;2;255;254;254ma\u001b[0m \u001b[48;2;255;254;254mpalm\u001b[0m \u001b[48;2;255;250;250mtree\u001b[0m \u001b[48;2;255;253;253mlined\u001b[0m \u001b[48;2;255;240;240mriver\u001b[0m \u001b[48;2;255;252;252mpast\u001b[0m \u001b[48;2;255;238;238ma\u001b[0m \u001b[48;2;255;254;254mhut\u001b[0m \u001b[48;2;255;253;253m.\u001b[0m\n",
            "hypo:   \u001b[48;2;255;0;0ma\u001b[0m \u001b[48;2;255;243;243mman\u001b[0m \u001b[48;2;255;252;252mwithout\u001b[0m \u001b[48;2;255;226;226ma\u001b[0m \u001b[48;2;255;251;251mshirt\u001b[0m \u001b[48;2;255;247;247mis\u001b[0m \u001b[48;2;255;233;233mon\u001b[0m \u001b[48;2;255;253;253ma\u001b[0m \u001b[48;2;255;254;254mriver\u001b[0m \u001b[48;2;255;234;234m.\u001b[0m\n",
            "\n",
            "('Four kids pose on a stage.',) ('Several little people are breaking into a theater.',)\n",
            "['[CLS]', 'four', 'kids', 'pose', 'on', 'a', 'stage', '.', '[SEP]']\n",
            "tensor([6.9810e-01, 8.2895e-02, 2.9241e-02, 4.9321e-03, 2.2494e-03, 4.7885e-04,\n",
            "        3.4502e-02, 2.5550e-04, 1.3949e-01], device='cuda:0')\n",
            "['[SEP]', 'several', 'little', 'people', 'are', 'breaking', 'into', 'a', 'theater', '.', '[SEP]']\n",
            "tensor([ 0.1395,  0.0301,  0.0707,  0.0239,  0.0322,  0.5144,  0.0110,  0.0006,\n",
            "         0.0107,  0.0012, -0.0178], device='cuda:0')\n",
            "tensor([0])\n",
            "pre:   \u001b[48;2;255;0;0mfour\u001b[0m \u001b[48;2;255;224;224mkids\u001b[0m \u001b[48;2;255;244;244mpose\u001b[0m \u001b[48;2;255;253;253mon\u001b[0m \u001b[48;2;255;254;254ma\u001b[0m \u001b[48;2;255;254;254mstage\u001b[0m \u001b[48;2;255;242;242m.\u001b[0m\n",
            "hypo:   \u001b[48;2;255;185;185mseveral\u001b[0m \u001b[48;2;255;240;240mlittle\u001b[0m \u001b[48;2;255;219;219mpeople\u001b[0m \u001b[48;2;255;243;243mare\u001b[0m \u001b[48;2;255;239;239mbreaking\u001b[0m \u001b[48;2;255;0;0minto\u001b[0m \u001b[48;2;255;249;249ma\u001b[0m \u001b[48;2;255;254;254mtheater\u001b[0m \u001b[48;2;255;249;249m.\u001b[0m\n",
            "\n",
            "('A child looks at a cactus garden.',) ('A human is looking at plants.',)\n",
            "['[CLS]', 'a', 'child', 'looks', 'at', 'a', 'cactus', 'garden', '.', '[SEP]']\n",
            "tensor([ 9.8460e-01,  5.2731e-04,  1.4647e-03,  2.9028e-03,  3.3860e-04,\n",
            "         4.1924e-04,  1.0498e-01,  8.4031e-03, -6.0535e-05, -2.4580e-02],\n",
            "       device='cuda:0')\n",
            "['[SEP]', 'a', 'human', 'is', 'looking', 'at', 'plants', '.', '[SEP]']\n",
            "tensor([-2.4580e-02,  2.0576e-04,  1.7492e-01,  2.5767e-03,  1.1957e-03,\n",
            "         1.3369e-04,  1.0408e-02,  2.9911e-04, -4.5969e-02], device='cuda:0')\n",
            "tensor([1])\n",
            "pre:   \u001b[48;2;255;0;0ma\u001b[0m \u001b[48;2;255;254;254mchild\u001b[0m \u001b[48;2;255;254;254mlooks\u001b[0m \u001b[48;2;255;254;254mat\u001b[0m \u001b[48;2;255;254;254ma\u001b[0m \u001b[48;2;255;254;254mcactus\u001b[0m \u001b[48;2;255;227;227mgarden\u001b[0m \u001b[48;2;255;252;252m.\u001b[0m\n",
            "hypo:   \u001b[48;2;118;118;255ma\u001b[0m \u001b[48;2;255;254;254mhuman\u001b[0m \u001b[48;2;255;0;0mis\u001b[0m \u001b[48;2;255;251;251mlooking\u001b[0m \u001b[48;2;255;253;253mat\u001b[0m \u001b[48;2;255;254;254mplants\u001b[0m \u001b[48;2;255;239;239m.\u001b[0m\n",
            "\n",
            "('A group of girls in black and white spotted skirts cross a street.',) ('A group of girls are walking to school.',)\n",
            "['[CLS]', 'a', 'group', 'of', 'girls', 'in', 'black', 'and', 'white', 'spotted', 'skirts', 'cross', 'a', 'street', '.', '[SEP]']\n",
            "tensor([8.6270e-01, 1.0608e-01, 6.3436e-03, 4.0083e-04, 1.3363e-03, 7.1822e-04,\n",
            "        1.1463e-03, 1.8628e-04, 2.9854e-03, 9.4533e-02, 9.5271e-02, 1.2727e-01,\n",
            "        4.4695e-04, 6.7271e-03, 6.3285e-04, 2.0683e-02], device='cuda:0')\n",
            "['[SEP]', 'a', 'group', 'of', 'girls', 'are', 'walking', 'to', 'school', '.', '[SEP]']\n",
            "tensor([ 0.0207,  0.0004,  0.0023,  0.0007,  0.0012,  0.0024,  0.0047,  0.0007,\n",
            "         0.0023,  0.0005, -0.0234], device='cuda:0')\n",
            "tensor([2])\n",
            "pre:   \u001b[48;2;255;0;0ma\u001b[0m \u001b[48;2;255;223;223mgroup\u001b[0m \u001b[48;2;255;253;253mof\u001b[0m \u001b[48;2;255;254;254mgirls\u001b[0m \u001b[48;2;255;254;254min\u001b[0m \u001b[48;2;255;254;254mblack\u001b[0m \u001b[48;2;255;254;254mand\u001b[0m \u001b[48;2;255;254;254mwhite\u001b[0m \u001b[48;2;255;254;254mspotted\u001b[0m \u001b[48;2;255;227;227mskirts\u001b[0m \u001b[48;2;255;226;226mcross\u001b[0m \u001b[48;2;255;217;217ma\u001b[0m \u001b[48;2;255;254;254mstreet\u001b[0m \u001b[48;2;255;253;253m.\u001b[0m\n",
            "hypo:   \u001b[48;2;255;0;0ma\u001b[0m \u001b[48;2;255;250;250mgroup\u001b[0m \u001b[48;2;255;226;226mof\u001b[0m \u001b[48;2;255;246;246mgirls\u001b[0m \u001b[48;2;255;240;240mare\u001b[0m \u001b[48;2;255;225;225mwalking\u001b[0m \u001b[48;2;255;197;197mto\u001b[0m \u001b[48;2;255;246;246mschool\u001b[0m \u001b[48;2;255;226;226m.\u001b[0m\n",
            "\n",
            "('Two identical brown dogs side by side holding something white in both their mouths.',) ('Two litter brothers fight over a bone.',)\n",
            "['[CLS]', 'two', 'identical', 'brown', 'dogs', 'side', 'by', 'side', 'holding', 'something', 'white', 'in', 'both', 'their', 'mouths', '.', '[SEP]']\n",
            "tensor([ 5.0008e-01,  1.6506e-03,  3.2636e-02,  8.4471e-03,  2.8804e-03,\n",
            "         1.9833e-04,  3.3122e-02,  1.0568e-03, -5.7169e-02,  1.1754e-03,\n",
            "         1.4380e-02,  2.7280e-04,  4.8377e-03,  9.3658e-04,  3.8198e-03,\n",
            "         9.8682e-05,  7.4834e-02], device='cuda:0')\n",
            "['[SEP]', 'two', 'litter', 'brothers', 'fight', 'over', 'a', 'bone', '.', '[SEP]']\n",
            "tensor([ 7.4834e-02,  1.6708e-03,  8.7392e-02,  9.3042e-01,  3.6676e-02,\n",
            "        -6.2970e-02,  8.0747e-04,  3.4958e-04,  1.9440e-03, -6.5093e-02],\n",
            "       device='cuda:0')\n",
            "tensor([2])\n",
            "pre:   \u001b[48;2;255;0;0mtwo\u001b[0m \u001b[48;2;255;254;254midentical\u001b[0m \u001b[48;2;255;238;238mbrown\u001b[0m \u001b[48;2;255;250;250mdogs\u001b[0m \u001b[48;2;255;253;253mside\u001b[0m \u001b[48;2;255;254;254mby\u001b[0m \u001b[48;2;255;238;238mside\u001b[0m \u001b[48;2;255;254;254mholding\u001b[0m \u001b[48;2;0;0;255msomething\u001b[0m \u001b[48;2;255;254;254mwhite\u001b[0m \u001b[48;2;255;247;247min\u001b[0m \u001b[48;2;255;254;254mboth\u001b[0m \u001b[48;2;255;252;252mtheir\u001b[0m \u001b[48;2;255;254;254mmouths\u001b[0m \u001b[48;2;255;253;253m.\u001b[0m\n",
            "hypo:   \u001b[48;2;255;234;234mtwo\u001b[0m \u001b[48;2;255;254;254mlitter\u001b[0m \u001b[48;2;255;231;231mbrothers\u001b[0m \u001b[48;2;255;0;0mfight\u001b[0m \u001b[48;2;255;244;244mover\u001b[0m \u001b[48;2;8;8;255ma\u001b[0m \u001b[48;2;255;254;254mbone\u001b[0m \u001b[48;2;255;254;254m.\u001b[0m\n",
            "\n",
            "('an old shoemaker in his factory',) ('The shoemaker is getting ready for his 16th birthday.',)\n",
            "['[CLS]', 'an', 'old', 'shoe', '##maker', 'in', 'his', 'factory', '[SEP]']\n",
            "tensor([ 2.9521e-01,  2.0414e-03,  7.4537e-05,  4.8127e-04,  7.5646e-03,\n",
            "         4.0529e-04,  5.1960e-04, -6.0728e-04,  4.7301e-03], device='cuda:0')\n",
            "['[SEP]', 'the', 'shoe', '##maker', 'is', 'getting', 'ready', 'for', 'his', '16th', 'birthday', '.', '[SEP]']\n",
            "tensor([ 4.7301e-03,  2.4208e-04,  3.2525e-04,  6.2792e-03,  3.2752e-04,\n",
            "        -3.3898e-02, -1.7708e-02, -5.3580e-04, -5.0968e-04, -9.0790e-02,\n",
            "        -2.8950e-02,  7.8289e-05, -5.5723e-02], device='cuda:0')\n",
            "tensor([0])\n",
            "pre:   \u001b[48;2;255;0;0man\u001b[0m \u001b[48;2;255;253;253mold\u001b[0m \u001b[48;2;255;254;254mshoe\u001b[0m\u001b[48;2;255;254;254mmaker\u001b[0m \u001b[48;2;255;248;248min\u001b[0m \u001b[48;2;255;254;254mhis\u001b[0m \u001b[48;2;255;254;254mfactory\u001b[0m\n",
            "hypo:   \u001b[48;2;255;62;62mthe\u001b[0m \u001b[48;2;255;245;245mshoe\u001b[0m\u001b[48;2;255;241;241mmaker\u001b[0m \u001b[48;2;255;0;0mis\u001b[0m \u001b[48;2;255;241;241mgetting\u001b[0m \u001b[48;2;159;159;255mready\u001b[0m \u001b[48;2;205;205;255mfor\u001b[0m \u001b[48;2;253;253;255mhis\u001b[0m \u001b[48;2;253;253;255m16th\u001b[0m \u001b[48;2;0;0;255mbirthday\u001b[0m \u001b[48;2;173;173;255m.\u001b[0m\n",
            "\n",
            "('A man a woman posing for a picture.',) ('A happy young couple are posing for a portrait.',)\n",
            "['[CLS]', 'a', 'man', 'a', 'woman', 'posing', 'for', 'a', 'picture', '.', '[SEP]']\n",
            "tensor([1.5429e-01, 7.2083e-03, 1.2430e-02, 2.4833e-03, 1.5205e-03, 5.6119e-03,\n",
            "        1.2358e-03, 5.7248e-05, 6.4538e-03, 1.6158e-04, 2.2075e-01],\n",
            "       device='cuda:0')\n",
            "['[SEP]', 'a', 'happy', 'young', 'couple', 'are', 'posing', 'for', 'a', 'portrait', '.', '[SEP]']\n",
            "tensor([ 2.2075e-01,  3.7114e-04,  4.2904e-02,  3.5407e-03,  8.2159e-04,\n",
            "         8.7351e-03, -4.5541e-03,  1.4632e-04,  1.5519e-04,  2.4586e-02,\n",
            "         1.7154e-04, -2.3327e-01], device='cuda:0')\n",
            "tensor([2])\n",
            "pre:   \u001b[48;2;255;76;76ma\u001b[0m \u001b[48;2;255;246;246mman\u001b[0m \u001b[48;2;255;240;240ma\u001b[0m \u001b[48;2;255;252;252mwoman\u001b[0m \u001b[48;2;255;253;253mposing\u001b[0m \u001b[48;2;255;248;248mfor\u001b[0m \u001b[48;2;255;253;253ma\u001b[0m \u001b[48;2;255;254;254mpicture\u001b[0m \u001b[48;2;255;247;247m.\u001b[0m\n",
            "hypo:   \u001b[48;2;255;0;0ma\u001b[0m \u001b[48;2;255;254;254mhappy\u001b[0m \u001b[48;2;255;205;205myoung\u001b[0m \u001b[48;2;255;250;250mcouple\u001b[0m \u001b[48;2;255;254;254mare\u001b[0m \u001b[48;2;255;244;244mposing\u001b[0m \u001b[48;2;250;250;255mfor\u001b[0m \u001b[48;2;255;254;254ma\u001b[0m \u001b[48;2;255;254;254mportrait\u001b[0m \u001b[48;2;255;226;226m.\u001b[0m\n",
            "\n",
            "('A girl in a red and white uniform is swinging a bat.',) ('A girl in uniform is swinging.',)\n",
            "['[CLS]', 'a', 'girl', 'in', 'a', 'red', 'and', 'white', 'uniform', 'is', 'swinging', 'a', 'bat', '.', '[SEP]']\n",
            "tensor([ 4.9047e-01,  7.5958e-05,  9.2475e-04,  2.4023e-05,  1.1175e-04,\n",
            "         1.6438e-03,  1.9110e-04,  5.8379e-04,  4.0696e-03,  6.0304e-04,\n",
            "         1.3389e-02,  1.9808e-04,  5.7681e-04,  0.0000e+00, -1.9138e-02],\n",
            "       device='cuda:0')\n",
            "['[SEP]', 'a', 'girl', 'in', 'uniform', 'is', 'swinging', '.', '[SEP]']\n",
            "tensor([-1.9138e-02,  1.9238e-05, -6.9333e-04,  2.5718e-05,  1.2181e-01,\n",
            "         1.1048e-03,  1.3676e-03,  1.7715e-04,  1.1740e-02], device='cuda:0')\n",
            "tensor([1])\n",
            "pre:   \u001b[48;2;255;0;0ma\u001b[0m \u001b[48;2;255;254;254mgirl\u001b[0m \u001b[48;2;255;254;254min\u001b[0m \u001b[48;2;255;254;254ma\u001b[0m \u001b[48;2;255;254;254mred\u001b[0m \u001b[48;2;255;254;254mand\u001b[0m \u001b[48;2;255;254;254mwhite\u001b[0m \u001b[48;2;255;254;254muniform\u001b[0m \u001b[48;2;255;252;252mis\u001b[0m \u001b[48;2;255;254;254mswinging\u001b[0m \u001b[48;2;255;248;248ma\u001b[0m \u001b[48;2;255;254;254mbat\u001b[0m \u001b[48;2;255;254;254m.\u001b[0m\n",
            "hypo:   \u001b[48;2;0;0;255ma\u001b[0m \u001b[48;2;255;254;254mgirl\u001b[0m \u001b[48;2;245;245;255min\u001b[0m \u001b[48;2;255;254;254muniform\u001b[0m \u001b[48;2;255;0;0mis\u001b[0m \u001b[48;2;255;252;252mswinging\u001b[0m \u001b[48;2;255;252;252m.\u001b[0m\n",
            "\n",
            "('A male guitar player is vigorously singing a not to a song in a poorly lit room.',) ('The man is buying a new guitar at the store.',)\n",
            "['[CLS]', 'a', 'male', 'guitar', 'player', 'is', 'vigorously', 'singing', 'a', 'not', 'to', 'a', 'song', 'in', 'a', 'poorly', 'lit', 'room', '.', '[SEP]']\n",
            "tensor([7.4985e-01, 3.2600e-02, 4.8516e-04, 3.0783e-04, 6.9078e-03, 1.9230e-03,\n",
            "        1.0659e-01, 8.1909e-04, 1.6257e-03, 4.4541e-02, 5.7900e-04, 2.7832e-04,\n",
            "        2.1383e-03, 7.2661e-05, 1.5102e-04, 4.8048e-03, 1.2411e-02, 9.3744e-04,\n",
            "        6.7622e-05, 1.3175e-02], device='cuda:0')\n",
            "['[SEP]', 'the', 'man', 'is', 'buying', 'a', 'new', 'guitar', 'at', 'the', 'store', '.', '[SEP]']\n",
            "tensor([1.3175e-02, 7.0467e-05, 5.5102e-04, 1.6406e-03, 1.5112e-02, 5.6324e-05,\n",
            "        6.2576e-04, 3.2206e-04, 2.0132e-03, 1.3948e-04, 4.7493e-03, 9.5826e-05,\n",
            "        4.8029e-02], device='cuda:0')\n",
            "tensor([0])\n",
            "pre:   \u001b[48;2;255;0;0ma\u001b[0m \u001b[48;2;255;243;243mmale\u001b[0m \u001b[48;2;255;254;254mguitar\u001b[0m \u001b[48;2;255;254;254mplayer\u001b[0m \u001b[48;2;255;252;252mis\u001b[0m \u001b[48;2;255;254;254mvigorously\u001b[0m \u001b[48;2;255;218;218msinging\u001b[0m \u001b[48;2;255;254;254ma\u001b[0m \u001b[48;2;255;254;254mnot\u001b[0m \u001b[48;2;255;239;239mto\u001b[0m \u001b[48;2;255;254;254ma\u001b[0m \u001b[48;2;255;254;254msong\u001b[0m \u001b[48;2;255;254;254min\u001b[0m \u001b[48;2;255;254;254ma\u001b[0m \u001b[48;2;255;254;254mpoorly\u001b[0m \u001b[48;2;255;253;253mlit\u001b[0m \u001b[48;2;255;250;250mroom\u001b[0m \u001b[48;2;255;254;254m.\u001b[0m\n",
            "hypo:   \u001b[48;2;255;185;185mthe\u001b[0m \u001b[48;2;255;254;254mman\u001b[0m \u001b[48;2;255;252;252mis\u001b[0m \u001b[48;2;255;246;246mbuying\u001b[0m \u001b[48;2;255;174;174ma\u001b[0m \u001b[48;2;255;254;254mnew\u001b[0m \u001b[48;2;255;251;251mguitar\u001b[0m \u001b[48;2;255;253;253mat\u001b[0m \u001b[48;2;255;244;244mthe\u001b[0m \u001b[48;2;255;254;254mstore\u001b[0m \u001b[48;2;255;229;229m.\u001b[0m\n",
            "\n",
            "('Several people are waiting in a metro station.',) ('The people are outside in the snow.',)\n",
            "['[CLS]', 'several', 'people', 'are', 'waiting', 'in', 'a', 'metro', 'station', '.', '[SEP]']\n",
            "tensor([ 6.8252e-01,  7.7457e-01,  6.7651e-04,  3.8016e-03,  1.5341e-03,\n",
            "        -1.8830e-03,  2.2399e-05,  5.8636e-03,  6.9138e-03,  5.6701e-05,\n",
            "         3.1058e-02], device='cuda:0')\n",
            "['[SEP]', 'the', 'people', 'are', 'outside', 'in', 'the', 'snow', '.', '[SEP]']\n",
            "tensor([ 3.1058e-02,  4.5231e-05,  4.4731e-04, -1.4379e-04, -6.4024e-03,\n",
            "         5.0837e-04,  7.2824e-05, -1.4629e-02,  5.3887e-05, -1.7527e-01],\n",
            "       device='cuda:0')\n",
            "tensor([0])\n",
            "pre:   \u001b[48;2;255;30;30mseveral\u001b[0m \u001b[48;2;255;0;0mpeople\u001b[0m \u001b[48;2;255;254;254mare\u001b[0m \u001b[48;2;255;253;253mwaiting\u001b[0m \u001b[48;2;255;254;254min\u001b[0m \u001b[48;2;0;0;255ma\u001b[0m \u001b[48;2;255;254;254mmetro\u001b[0m \u001b[48;2;255;253;253mstation\u001b[0m \u001b[48;2;255;252;252m.\u001b[0m\n",
            "hypo:   \u001b[48;2;255;0;0mthe\u001b[0m \u001b[48;2;255;254;254mpeople\u001b[0m \u001b[48;2;255;251;251mare\u001b[0m \u001b[48;2;254;254;255moutside\u001b[0m \u001b[48;2;245;245;255min\u001b[0m \u001b[48;2;255;250;250mthe\u001b[0m \u001b[48;2;255;254;254msnow\u001b[0m \u001b[48;2;233;233;255m.\u001b[0m\n",
            "\n",
            "('A woman in a pink top is holding a glass bottle in one hand and a Stitch doll in the other.',) ('The glass bottle is big',)\n",
            "['[CLS]', 'a', 'woman', 'in', 'a', 'pink', 'top', 'is', 'holding', 'a', 'glass', 'bottle', 'in', 'one', 'hand', 'and', 'a', 'stitch', 'doll', 'in', 'the', 'other', '.', '[SEP]']\n",
            "tensor([9.4117e-01, 1.4057e-03, 2.1949e-03, 3.8450e-04, 2.0152e-04, 1.0816e-02,\n",
            "        1.9598e-02, 1.6489e-03, 4.6550e-03, 1.4406e-04, 4.9632e-04, 1.8560e-03,\n",
            "        3.7648e-04, 1.3073e-03, 8.2026e-05, 4.8727e-05, 1.5654e-04, 5.9074e-02,\n",
            "        1.1299e-04, 7.0655e-05, 9.2816e-04, 1.1571e-03, 7.6848e-05, 1.0513e-01],\n",
            "       device='cuda:0')\n",
            "['[SEP]', 'the', 'glass', 'bottle', 'is', 'big', '[SEP]']\n",
            "tensor([ 1.0513e-01,  1.0700e-04,  4.2939e-04,  5.8779e-03,  6.9485e-03,\n",
            "         6.7021e-01, -2.7354e-02], device='cuda:0')\n",
            "tensor([2])\n",
            "pre:   \u001b[48;2;255;0;0ma\u001b[0m \u001b[48;2;255;254;254mwoman\u001b[0m \u001b[48;2;255;254;254min\u001b[0m \u001b[48;2;255;254;254ma\u001b[0m \u001b[48;2;255;254;254mpink\u001b[0m \u001b[48;2;255;252;252mtop\u001b[0m \u001b[48;2;255;249;249mis\u001b[0m \u001b[48;2;255;254;254mholding\u001b[0m \u001b[48;2;255;253;253ma\u001b[0m \u001b[48;2;255;254;254mglass\u001b[0m \u001b[48;2;255;254;254mbottle\u001b[0m \u001b[48;2;255;254;254min\u001b[0m \u001b[48;2;255;254;254mone\u001b[0m \u001b[48;2;255;254;254mhand\u001b[0m \u001b[48;2;255;254;254mand\u001b[0m \u001b[48;2;255;254;254ma\u001b[0m \u001b[48;2;255;254;254mstitch\u001b[0m \u001b[48;2;255;238;238mdoll\u001b[0m \u001b[48;2;255;254;254min\u001b[0m \u001b[48;2;255;254;254mthe\u001b[0m \u001b[48;2;255;254;254mother\u001b[0m \u001b[48;2;255;254;254m.\u001b[0m\n",
            "hypo:   \u001b[48;2;255;215;215mthe\u001b[0m \u001b[48;2;255;254;254mglass\u001b[0m \u001b[48;2;255;254;254mbottle\u001b[0m \u001b[48;2;255;252;252mis\u001b[0m \u001b[48;2;255;252;252mbig\u001b[0m\n",
            "\n",
            "('Adults and children share in the looking at something, and some young ladies stand to the side.',) ('Some children are looking',)\n",
            "['[CLS]', 'adults', 'and', 'children', 'share', 'in', 'the', 'looking', 'at', 'something', ',', 'and', 'some', 'young', 'ladies', 'stand', 'to', 'the', 'side', '.', '[SEP]']\n",
            "tensor([ 5.3784e-01,  7.9983e-01,  1.6575e-04,  1.9651e-03,  3.0662e-02,\n",
            "         2.1227e-05,  1.8316e-04,  1.5002e-03,  9.6562e-05,  5.3062e-04,\n",
            "         1.2297e-04,  1.0226e-04,  5.3639e-04,  7.7328e-03, -3.9428e-02,\n",
            "         2.2472e-03,  2.4550e-04,  7.5080e-04,  2.2950e-03,  1.0484e-05,\n",
            "         3.8006e-03], device='cuda:0')\n",
            "['[SEP]', 'some', 'children', 'are', 'looking', '[SEP]']\n",
            "tensor([0.0038, 0.0006, 0.0122, 0.0008, 0.0378, 0.3547], device='cuda:0')\n",
            "tensor([1])\n",
            "pre:   \u001b[48;2;255;83;83madults\u001b[0m \u001b[48;2;255;0;0mand\u001b[0m \u001b[48;2;255;254;254mchildren\u001b[0m \u001b[48;2;255;254;254mshare\u001b[0m \u001b[48;2;255;245;245min\u001b[0m \u001b[48;2;255;254;254mthe\u001b[0m \u001b[48;2;255;254;254mlooking\u001b[0m \u001b[48;2;255;254;254mat\u001b[0m \u001b[48;2;255;254;254msomething\u001b[0m \u001b[48;2;255;254;254m,\u001b[0m \u001b[48;2;255;254;254mand\u001b[0m \u001b[48;2;255;254;254msome\u001b[0m \u001b[48;2;255;254;254myoung\u001b[0m \u001b[48;2;255;252;252mladies\u001b[0m \u001b[48;2;0;0;255mstand\u001b[0m \u001b[48;2;255;254;254mto\u001b[0m \u001b[48;2;255;254;254mthe\u001b[0m \u001b[48;2;255;254;254mside\u001b[0m \u001b[48;2;255;254;254m.\u001b[0m\n",
            "hypo:   \u001b[48;2;255;252;252msome\u001b[0m \u001b[48;2;255;254;254mchildren\u001b[0m \u001b[48;2;255;246;246mare\u001b[0m \u001b[48;2;255;254;254mlooking\u001b[0m\n",
            "\n",
            "('an old shoemaker in his factory',) ('The shoemaker is inside.',)\n",
            "['[CLS]', 'an', 'old', 'shoe', '##maker', 'in', 'his', 'factory', '[SEP]']\n",
            "tensor([ 4.1005e-01,  4.0015e-01,  1.9207e-04,  1.3313e-04,  7.7500e-04,\n",
            "         1.2798e-03,  1.0258e-03,  1.5463e-01, -2.0576e-02], device='cuda:0')\n",
            "['[SEP]', 'the', 'shoe', '##maker', 'is', 'inside', '.', '[SEP]']\n",
            "tensor([-2.0576e-02,  4.1128e-05,  1.4237e-03,  2.4956e-03,  1.6780e-03,\n",
            "        -7.1978e-03,  3.3959e-05,  1.4028e-02], device='cuda:0')\n",
            "tensor([1])\n",
            "pre:   \u001b[48;2;255;0;0man\u001b[0m \u001b[48;2;255;6;6mold\u001b[0m \u001b[48;2;255;254;254mshoe\u001b[0m\u001b[48;2;255;254;254mmaker\u001b[0m \u001b[48;2;255;254;254min\u001b[0m \u001b[48;2;255;254;254mhis\u001b[0m \u001b[48;2;255;254;254mfactory\u001b[0m\n",
            "hypo:   \u001b[48;2;0;0;255mthe\u001b[0m \u001b[48;2;255;254;254mshoe\u001b[0m\u001b[48;2;255;229;229mmaker\u001b[0m \u001b[48;2;255;209;209mis\u001b[0m \u001b[48;2;255;224;224minside\u001b[0m \u001b[48;2;165;165;255m.\u001b[0m\n",
            "\n",
            "('Two men cool off under a waterfall.',) ('Two brothers are under a waterfall.',)\n",
            "['[CLS]', 'two', 'men', 'cool', 'off', 'under', 'a', 'waterfall', '.', '[SEP]']\n",
            "tensor([7.4566e-02, 1.6600e-03, 1.3921e-03, 6.9526e-02, 5.6187e-02, 3.5421e-03,\n",
            "        5.2564e-04, 2.8799e-02, 4.5660e-05, 1.5842e-01], device='cuda:0')\n",
            "['[SEP]', 'two', 'brothers', 'are', 'under', 'a', 'waterfall', '.', '[SEP]']\n",
            "tensor([ 0.1584,  0.0090,  0.1052,  0.0153,  0.0026,  0.0003,  0.0266,  0.0002,\n",
            "        -0.1154], device='cuda:0')\n",
            "tensor([2])\n",
            "pre:   \u001b[48;2;255;134;134mtwo\u001b[0m \u001b[48;2;255;252;252mmen\u001b[0m \u001b[48;2;255;252;252mcool\u001b[0m \u001b[48;2;255;143;143moff\u001b[0m \u001b[48;2;255;164;164munder\u001b[0m \u001b[48;2;255;249;249ma\u001b[0m \u001b[48;2;255;254;254mwaterfall\u001b[0m \u001b[48;2;255;208;208m.\u001b[0m\n",
            "hypo:   \u001b[48;2;255;0;0mtwo\u001b[0m \u001b[48;2;255;240;240mbrothers\u001b[0m \u001b[48;2;255;85;85mare\u001b[0m \u001b[48;2;255;230;230munder\u001b[0m \u001b[48;2;255;250;250ma\u001b[0m \u001b[48;2;255;254;254mwaterfall\u001b[0m \u001b[48;2;255;212;212m.\u001b[0m\n",
            "\n",
            "('Two hussars sit perched on horses, dressed in extravagant ceremonial wear, each holding a sabre in their right hand, reigns to the horse in their left.',) ('There are professional riders at a ceremony.',)\n",
            "['[CLS]', 'two', 'hu', '##ssar', '##s', 'sit', 'perched', 'on', 'horses', ',', 'dressed', 'in', 'extravagant', 'ceremonial', 'wear', ',', 'each', 'holding', 'a', 'sabre', 'in', 'their', 'right', 'hand', ',', 'reigns', 'to', 'the', 'horse', 'in', 'their', 'left', '.', '[SEP]']\n",
            "tensor([9.4668e-01, 5.4522e-01, 1.0094e-01, 5.5548e-03, 3.0293e-03, 6.7507e-03,\n",
            "        9.6312e-02, 2.4747e-03, 2.1773e-02, 2.0870e-05, 1.5981e-01, 3.7737e-05,\n",
            "        2.5810e-01, 3.3969e-02, 1.9182e-02, 3.9952e-05, 5.6475e-04, 1.4398e-03,\n",
            "        9.0043e-05, 8.7628e-03, 3.1386e-04, 8.2977e-04, 2.6125e-03, 2.6436e-04,\n",
            "        2.3002e-04, 1.0410e-01, 2.3501e-03, 4.3325e-04, 3.8036e-03, 3.6033e-05,\n",
            "        5.7720e-04, 3.2967e-03, 1.5470e-04, 1.5077e-01], device='cuda:0')\n",
            "['[SEP]', 'there', 'are', 'professional', 'riders', 'at', 'a', 'ceremony', '.', '[SEP]']\n",
            "tensor([ 1.5077e-01,  1.1765e-03,  1.1032e-02,  3.5281e-02,  7.4039e-03,\n",
            "        -4.8381e-04,  1.2296e-03,  2.5912e-02,  5.4945e-05, -1.3490e-01],\n",
            "       device='cuda:0')\n",
            "tensor([2])\n",
            "pre:   \u001b[48;2;255;0;0mtwo\u001b[0m \u001b[48;2;255;108;108mhu\u001b[0m\u001b[48;2;255;227;227mssar\u001b[0m\u001b[48;2;255;253;253ms\u001b[0m \u001b[48;2;255;254;254msit\u001b[0m \u001b[48;2;255;253;253mperched\u001b[0m \u001b[48;2;255;229;229mon\u001b[0m \u001b[48;2;255;254;254mhorses\u001b[0m \u001b[48;2;255;249;249m,\u001b[0m \u001b[48;2;255;254;254mdressed\u001b[0m \u001b[48;2;255;211;211min\u001b[0m \u001b[48;2;255;254;254mextravagant\u001b[0m \u001b[48;2;255;185;185mceremonial\u001b[0m \u001b[48;2;255;245;245mwear\u001b[0m \u001b[48;2;255;249;249m,\u001b[0m \u001b[48;2;255;254;254meach\u001b[0m \u001b[48;2;255;254;254mholding\u001b[0m \u001b[48;2;255;254;254ma\u001b[0m \u001b[48;2;255;254;254msabre\u001b[0m \u001b[48;2;255;252;252min\u001b[0m \u001b[48;2;255;254;254mtheir\u001b[0m \u001b[48;2;255;254;254mright\u001b[0m \u001b[48;2;255;254;254mhand\u001b[0m \u001b[48;2;255;254;254m,\u001b[0m \u001b[48;2;255;254;254mreigns\u001b[0m \u001b[48;2;255;226;226mto\u001b[0m \u001b[48;2;255;254;254mthe\u001b[0m \u001b[48;2;255;254;254mhorse\u001b[0m \u001b[48;2;255;253;253min\u001b[0m \u001b[48;2;255;254;254mtheir\u001b[0m \u001b[48;2;255;254;254mleft\u001b[0m \u001b[48;2;255;254;254m.\u001b[0m\n",
            "hypo:   \u001b[48;2;255;0;0mthere\u001b[0m \u001b[48;2;255;253;253mare\u001b[0m \u001b[48;2;255;236;236mprofessional\u001b[0m \u001b[48;2;255;195;195mriders\u001b[0m \u001b[48;2;255;242;242mat\u001b[0m \u001b[48;2;254;254;255ma\u001b[0m \u001b[48;2;255;252;252mceremony\u001b[0m \u001b[48;2;255;211;211m.\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}