is_exclusively.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6656008d",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "# Windows Environment Simulation\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "692fa256",
   "metadata": {},
   "source": [
    "### (ExPred) Load the pretrained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d34aebe6",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "from src.expred import (seeding, ExpredInput,\n",
    "                        BertTokenizerWithSpans, ExpredConfig, Expred)\n",
    "from src.expred.models import (prepare_for_cl, train_evidence_classifier,\n",
    "                               train_mtl_token_identifier)\n",
    "from transformers import BertTokenizer\n",
    "\n",
    "# or, simply\n",
    "expred_config = ExpredConfig(\n",
    "    pretrained_dataset_name='fever',\n",
    "    base_dataset_name='fever',\n",
    "    device='cpu',\n",
    "    load_from_pretrained=True)\n",
    "\n",
    "# seeding\n",
    "seeding(1234)\n",
    "\n",
    "# Initialize tokenizer\n",
    "tokenizer = BertTokenizerWithSpans.from_pretrained('bert-base-uncased')\n",
    "\n",
    "# create the model\n",
    "expred = Expred.from_pretrained(expred_config)\n",
    "expred.eval()\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "b2910346",
   "metadata": {},
   "source": [
    "## Evaluation of subsequences\n",
    "The black-box model of ExPred that we test for short-cuts, can be applied to automate fact-checking.\n",
    "The dataset curated for that purpose is called FeVer: a dataset for Fact Verification."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "272383f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 266 items\n"
     ]
    }
   ],
   "source": [
    "# Read subset of FeVer corresponding to all occurences of a subsequence\n",
    "import pandas as pd\n",
    "\n",
    "evaluations = pd.read_json(\"is_exclusively.json\")\n",
    "\n",
    "print(\"Loaded\", len(evaluations) , \"items\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "de613873",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a function to simplify execution of ExPred\n",
    "def evaluate_query_with_expred(current_query, evidence):\n",
    "    # transform the input to the way the expred accepts\n",
    "    expred_input = ExpredInput(\n",
    "        queries=[current_query.split()],\n",
    "        docs=[evidence.split()],\n",
    "        labels=['SUPPORTS', 'REFUTES'],\n",
    "        config=expred_config,\n",
    "        ann_ids=['spontan_1'],\n",
    "        span_tokenizer=tokenizer)\n",
    "    # don't forget to preprocess\n",
    "    expred_input.preprocess()\n",
    "\n",
    "    # the output is in the form of a dict:\n",
    "    expred_output = expred(expred_input)\n",
    "\n",
    "    # retrieve the evaluation label\n",
    "    current_output = expred_input.get_decoded_cls_preds(expred_output)\n",
    "    return [current_query, current_output[0]]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "8a8cd35b",
   "metadata": {},
   "source": [
    "### Subsequence: 'is exclusively'\n",
    "\n",
    "First we will confirm that the subsequence holds true for all occurrences of the subsequence.\n",
    "From the observations it can be seen that all cases of the subset containing the regex are refuted."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3f4e1af8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The Falling is exclusively about an all-boys s...</td>\n",
       "      <td>REFUTES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Gone Girl is exclusively a book.</td>\n",
       "      <td>REFUTES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Dark Places is exclusively a novel.</td>\n",
       "      <td>REFUTES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Dark Places is exclusively a novel.</td>\n",
       "      <td>REFUTES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Dark Places is exclusively a novel.</td>\n",
       "      <td>REFUTES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>261</th>\n",
       "      <td>Reba McEntire is exclusively a doctor.</td>\n",
       "      <td>REFUTES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>262</th>\n",
       "      <td>Reba McEntire is exclusively a doctor.</td>\n",
       "      <td>REFUTES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263</th>\n",
       "      <td>Reba McEntire is exclusively a doctor.</td>\n",
       "      <td>REFUTES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264</th>\n",
       "      <td>Reba McEntire is exclusively a doctor.</td>\n",
       "      <td>REFUTES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265</th>\n",
       "      <td>Reba McEntire is exclusively a doctor.</td>\n",
       "      <td>REFUTES</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>266 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 query    label\n",
       "0    The Falling is exclusively about an all-boys s...  REFUTES\n",
       "1                     Gone Girl is exclusively a book.  REFUTES\n",
       "2                  Dark Places is exclusively a novel.  REFUTES\n",
       "3                  Dark Places is exclusively a novel.  REFUTES\n",
       "4                  Dark Places is exclusively a novel.  REFUTES\n",
       "..                                                 ...      ...\n",
       "261             Reba McEntire is exclusively a doctor.  REFUTES\n",
       "262             Reba McEntire is exclusively a doctor.  REFUTES\n",
       "263             Reba McEntire is exclusively a doctor.  REFUTES\n",
       "264             Reba McEntire is exclusively a doctor.  REFUTES\n",
       "265             Reba McEntire is exclusively a doctor.  REFUTES\n",
       "\n",
       "[266 rows x 2 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Batch size\n",
    "n = len(evaluations)\n",
    "\n",
    "results = []\n",
    "\n",
    "for i in range (0, n):\n",
    "    if(evaluations['query'][i]):\n",
    "        current_query = evaluations['query'][i]\n",
    "        current_doc = evaluations['evidences'][i]\n",
    "\n",
    "        results.append(evaluate_query_with_expred(current_query, current_doc))\n",
    "\n",
    "df = pd.DataFrame(results, columns = ['query', 'label'])\n",
    "df\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "5be81a2d",
   "metadata": {},
   "source": [
    "Percentage of confirmed cases:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "859f7ec4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(((df.label.values == 'REFUTES').sum())/n)*100"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "255eb4f4",
   "metadata": {},
   "source": [
    "## Adverserial Attack\n",
    "We replace the substring with another substring that guarantees that the meaning has flipped."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "df4d5df4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The Falling can be about an all-boys school.</td>\n",
       "      <td>SUPPORTS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Gone Girl can be a book.</td>\n",
       "      <td>REFUTES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Dark Places can be a novel.</td>\n",
       "      <td>REFUTES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Dark Places can be a novel.</td>\n",
       "      <td>REFUTES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Dark Places can be a novel.</td>\n",
       "      <td>REFUTES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>261</th>\n",
       "      <td>Reba McEntire can be a doctor.</td>\n",
       "      <td>REFUTES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>262</th>\n",
       "      <td>Reba McEntire can be a doctor.</td>\n",
       "      <td>SUPPORTS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263</th>\n",
       "      <td>Reba McEntire can be a doctor.</td>\n",
       "      <td>REFUTES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264</th>\n",
       "      <td>Reba McEntire can be a doctor.</td>\n",
       "      <td>SUPPORTS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265</th>\n",
       "      <td>Reba McEntire can be a doctor.</td>\n",
       "      <td>REFUTES</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>266 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            query     label\n",
       "0    The Falling can be about an all-boys school.  SUPPORTS\n",
       "1                        Gone Girl can be a book.   REFUTES\n",
       "2                     Dark Places can be a novel.   REFUTES\n",
       "3                     Dark Places can be a novel.   REFUTES\n",
       "4                     Dark Places can be a novel.   REFUTES\n",
       "..                                            ...       ...\n",
       "261                Reba McEntire can be a doctor.   REFUTES\n",
       "262                Reba McEntire can be a doctor.  SUPPORTS\n",
       "263                Reba McEntire can be a doctor.   REFUTES\n",
       "264                Reba McEntire can be a doctor.  SUPPORTS\n",
       "265                Reba McEntire can be a doctor.   REFUTES\n",
       "\n",
       "[266 rows x 2 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "substring = \"is exclusively\"\n",
    "replacewith = \"can be\"\n",
    "n = len(evaluations)\n",
    "\n",
    "results = []\n",
    "\n",
    "for i in range (0, n):\n",
    "    current_query = evaluations['query'][i].replace(substring, replacewith)\n",
    "    current_doc = evaluations['evidences'][i]\n",
    "\n",
    "    results.append(evaluate_query_with_expred(current_query, current_doc))\n",
    "\n",
    "df = pd.DataFrame(results, columns = ['query', 'label'])\n",
    "df"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "2eda746d",
   "metadata": {},
   "source": [
    "## Conclusion\n",
    "\n",
    "This experiment means that 'has only ever been' is a <span style=\"color:green\">valid short-cut for the ExPred model</span>, as it has very high probability that ExPred refutes a query containing this subsequence. The probability is acquired as follows:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "12e3adba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40.97744360902256"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(((df.label.values == 'SUPPORTS').sum())/n)*100"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  },
  "vscode": {
   "interpreter": {
    "hash": "ad2bdc8ecc057115af97d19610ffacc2b4e99fae6737bb82f5d7fb13d2f2c186"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}