demo/SelfCheck_demo1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "89387af5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import spacy\n",
    "from selfcheckgpt.modeling_selfcheck import SelfCheckMQAG, SelfCheckBERTScore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f9528439",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x2b07ee00e270>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(28)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "97b5f44f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d73a77bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "passage = \"\"\"\n",
    "Michael Alan Weiner (born March 31, 1942), better known by his professional name Michael Savage, is an American radio host, author, activist, nutritionist, and conservative political commentator. He is the host of The Savage Nation, a nationally syndicated talk show that aired on Talk Radio Network across the United States until 2012, and in 2009 was the second most listened-to radio talk show in the country with an audience of over 20 million listeners on 400 stations across the United States. Since October 23, 2012, Michael Savage has been syndicated by Cumulus Media Networks. He holds master's degrees from the University of Hawaii in medical botany and medical anthropology, and a Ph.D. from the University of California, Berkeley in nutritional ethnomedicine. As Michael Weiner, he has written books on nutrition, herbal medicine, and homeopathy.\n",
    "\"\"\".replace(\"\\n\", \" \").strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6887ac74",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Michael Alan Weiner (born March 31, 1942), better known by his professional name Michael Savage, is an American radio host, author, activist, nutritionist, and conservative political commentator. He is the host of The Savage Nation, a nationally syndicated talk show that aired on Talk Radio Network across the United States until 2012, and in 2009 was the second most listened-to radio talk show in the country with an audience of over 20 million listeners on 400 stations across the United States. Since October 23, 2012, Michael Savage has been syndicated by Cumulus Media Networks. He holds master's degrees from the University of Hawaii in medical botany and medical anthropology, and a Ph.D. from the University of California, Berkeley in nutritional ethnomedicine. As Michael Weiner, he has written books on nutrition, herbal medicine, and homeopathy.\n"
     ]
    }
   ],
   "source": [
    "print(passage)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0bfaf0d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "passage_edited = \"\"\"\n",
    "Michael Alan Weiner (born January 13, 1995), better known by his professional name Joshua King, is an Austrain footballer, and businessman. He is the host of The Savage Nation, a nationally syndicated talk show that aired on Talk Radio Network across the United States until 2012, and in 2009 was the second most listened-to radio talk show in the country with an audience of over 20 million listeners on 400 stations across the United States. Since October 23, 2002, Michael King has been syndicated by Virgin Media Networks. He holds master's degrees from the University of Hawaii in medical botany and medical anthropology, and a Ph.D. from the University of California, San Diego in nutritional ethnomedicine. As Michael Weiner, he has written books on nutrition, herbal medicine, and homeopathy.\n",
    "\"\"\".replace(\"\\n\", \" \").strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "91937c66",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Michael Alan Weiner (born January 13, 1995), better known by his professional name Joshua King, is an Austrain footballer, and businessman. He is the host of The Savage Nation, a nationally syndicated talk show that aired on Talk Radio Network across the United States until 2012, and in 2009 was the second most listened-to radio talk show in the country with an audience of over 20 million listeners on 400 stations across the United States. Since October 23, 2002, Michael King has been syndicated by Virgin Media Networks. He holds master's degrees from the University of Hawaii in medical botany and medical anthropology, and a Ph.D. from the University of California, San Diego in nutritional ethnomedicine. As Michael Weiner, he has written books on nutrition, herbal medicine, and homeopathy.\n"
     ]
    }
   ],
   "source": [
    "print(passage_edited) # edit sent1, sent3, sent4 to inaccurate information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "477330f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample1 = \"\"\"\n",
    "Michael Alan Weiner (born March 31, 1942), better known by his professional name Michael Savage, is an American radio host, author, and political commentator. He is the host of The Savage Nation, a nationally syndicated talk show broadcast on Talk Radio Network across the United States, and on Comet in Canada. He holds master's degrees from the University of Hawaii in medical botany and medical anthropology, and a Ph.D. from the University of California, Berkeley in epidemiology and nutrition sciences. Savage has summarized his political philosophy in three words: borders, language, and culture. He has characterized his views as conservative nationalism, while critics have characterized them as fascism. He is a fierce critic of liberalism, and the War on Drugs. He believes that political correctness has \"ruined the United States\" and that the American government has spent too much money on social programs throughout its history.\n",
    "\"\"\".replace(\"\\n\", \" \").strip()\n",
    "\n",
    "sample2 = \"\"\"\n",
    "Michael A. Savage (born Michael Alan Weiner; March 31, 1942) is an American radio host, author, activist, nutritionist, and conservative political commentator. He is the host of The Savage Nation, a nationally syndicated talk show that aired on Talk Radio Network across the United States until 2012. The Savage Nation has a combined audience of over 20 million listeners on 400 stations across the United States and is one of the top-rated shows in the country. He is also the host of The Michael Savage Show, a daily podcast and webcast heard around the world. He has been called \"the godfather of Trumpmania\".\n",
    "Savage has written more than 25 books and is an outspoken supporter of traditional American values and break-down of the culture of political correctness. He advocates an \"America first\" foreign policy and rigorous opposition to illegal immigration. For his social and political views, Savage has been described as a conservative and a right-wing populist. He has campaigned for numerous political causes, including opposition to same-sex marriage and the legalization of marijuana.\n",
    "\"\"\".replace(\"\\n\", \" \").strip()\n",
    "\n",
    "sample3 = \"\"\"\n",
    "Michael Alan Weiner (born March 31, 1942),[2] better known by his professional name Michael Savage, is an American radio host, author, activist, nutritionist, and conservative political commentator. He is the host of The Savage Nation, a nationally syndicated talk show that aired on Talk Radio Network across the United States until 2012, and in 2009 was the second most listened-to radio talk show in the country with an audience of over 20 million listeners on 400 stations across the United States. Since October 23, 2012, Michael Savage has been syndicated by Cumulus Media Networks. He holds master's degrees from the University of Hawaii in medical botany and medical anthropology, and a Ph.D. from the University of California, Berkeley in nutritional ethnomedicine. As Michael Weiner, he has written books on nutrition, herbal medicine, and homeopathy.\n",
    "Savage has summarized his political philosophy in three words: borders, language, and culture. He believes that the United States should end foreign aid, impose much stricter immigration laws, deport illegal immigrants, and put an end to birthright citizenship. Savage has characterized his views as conservative nationalism, free\n",
    "\"\"\".replace(\"\\n\", \" \").strip()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f1050ad1",
   "metadata": {},
   "source": [
    "## Initialization\n",
    "- Calling `SelfCheckMQAG()` first time will take some time as it will download the generation and answering models from HuggingFace Model Hub."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3f42e818",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SelfCheck-MQAG initialized to device cuda\n",
      "SelfCheck-BERTScore initialized\n"
     ]
    }
   ],
   "source": [
    "selfcheck_mqag = SelfCheckMQAG(device=device)\n",
    "selfcheck_bertscore = SelfCheckBERTScore()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1dec3fc3",
   "metadata": {},
   "source": [
    "## Running Inference"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a11c23a0",
   "metadata": {},
   "source": [
    "### 1. Uneditted/Original Passage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1d6c9697",
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "sentences = [sent for sent in nlp(passage).sents] # List[spacy.tokens.span.Span]\n",
    "sentences = [sent.text.strip() for sent in sentences if len(sent) > 3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "30d57b7f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SelfCheck running on 5 sentences...\n"
     ]
    }
   ],
   "source": [
    "print(\"SelfCheck running on {} sentences...\".format(len(sentences)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "290f36f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "sent_scores_mqag = selfcheck_mqag.predict(\n",
    "    sentences,\n",
    "    passage,\n",
    "    [sample1, sample2, sample3],\n",
    "    num_questions_per_sent = 5,\n",
    "    scoring_method = 'bayes_with_alpha',\n",
    "    beta1 = 0.95, beta2 = 0.95,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "37cc9aaa",
   "metadata": {},
   "outputs": [],
   "source": [
    "sent_scores_bertscore = selfcheck_bertscore.predict(\n",
    "    sentences,\n",
    "    [sample1, sample2, sample3],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "204cfacc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MQAG\tBERTScore\n",
      "0.0733\t0.0236\n",
      "0.1748\t0.0384\n",
      "0.2814\t0.0889\n",
      "0.2903\t0.0589\n",
      "0.0030\t0.0798\n"
     ]
    }
   ],
   "source": [
    "# Uneditted passge\n",
    "print(\"MQAG\\tBERTScore\")\n",
    "for s1, s2 in zip(sent_scores_mqag, sent_scores_bertscore):\n",
    "    print(\"{:.4f}\\t{:.4f}\".format(s1, s2))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6252d9f9",
   "metadata": {},
   "source": [
    "### 2. Editted Passage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5a8c9221",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = [sent for sent in nlp(passage_edited).sents] # List[spacy.tokens.span.Span]\n",
    "sentences = [sent.text.strip() for sent in sentences if len(sent) > 3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "841ad44e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SelfCheck running on 5 sentences...\n"
     ]
    }
   ],
   "source": [
    "print(\"SelfCheck running on {} sentences...\".format(len(sentences)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "56a76b91",
   "metadata": {},
   "outputs": [],
   "source": [
    "sent_scores_mqag = selfcheck_mqag.predict(\n",
    "    sentences,\n",
    "    passage,\n",
    "    [sample1, sample2, sample3],\n",
    "    num_questions_per_sent = 5,\n",
    "    scoring_method = 'bayes_with_alpha',\n",
    "    beta1 = 0.95, beta2 = 0.95,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "97693086",
   "metadata": {},
   "outputs": [],
   "source": [
    "sent_scores_bertscore = selfcheck_bertscore.predict(\n",
    "    sentences,\n",
    "    [sample1, sample2, sample3],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5ffb6d8e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MQAG\tBERTScore\n",
      "0.4290\t0.0972\n",
      "0.1137\t0.0384\n",
      "0.4985\t0.1050\n",
      "0.2979\t0.0632\n",
      "0.4282\t0.0798\n"
     ]
    }
   ],
   "source": [
    "# Editted passge [seent1, sent3, sent4 were editted]\n",
    "print(\"MQAG\\tBERTScore\")\n",
    "for s1, s2 in zip(sent_scores_mqag, sent_scores_bertscore):\n",
    "    print(\"{:.4f}\\t{:.4f}\".format(s1, s2))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}