generative_pipeline_draft.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ae73ace",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. We could use NER for keyword extractions\n",
    "# 2. We could use google search for evidence retrieval\n",
    "# 3. Fine tuning and prompt engineering needed\n",
    "# 4. Avoid posting to the duplicate info to Weaviate\n",
    "# 5. Combine with Predictive AI results\n",
    "# 6. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3cb2041c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Starting of the pipeline, get a real time news\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "\n",
    "def scrape_site(url):\n",
    "    response = requests.get(url)\n",
    "\n",
    "    if response.status_code == 200:\n",
    "        soup = BeautifulSoup(response.content, 'html.parser')\n",
    "        \n",
    "        # Extract header\n",
    "        header = soup.find(['h1']).get_text().strip()\n",
    "\n",
    "        # Extract content\n",
    "        content_tags = soup.find_all(['p'])\n",
    "        content = [tag.get_text().strip().replace('\\xa0', ' ') for tag in content_tags]\n",
    "\n",
    "        # Find the keyword 'By' to extract the author's name\n",
    "        page_text = soup.get_text()\n",
    "        match = re.search(r'\\bBy\\s+([A-Za-z\\s.,]+)', page_text)\n",
    "        authors = match.group(1).strip().replace('and', ',') if match else 'Author not found'\n",
    "        author_lst = [auth.strip() for auth in authors.split(',')]\n",
    "        return header, content, author_lst\n",
    "    else:\n",
    "        print(f\"Failed to retrieve the webpage. Status code: {response.status_code}\")\n",
    "        return None, None, None\n",
    "\n",
    "url = \"https://www.cnn.com/2024/02/27/entertainment/jam-master-jay-murder-verdict/index.html\"\n",
    "header, content, authors = scrape_site(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "053894c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "jay: 0.44561177832918486\n",
      "said: 0.25798576640110704\n",
      "jam: 0.21107926341908756\n",
      "jordan: 0.21107926341908756\n",
      "master: 0.21107926341908756\n",
      "washington: 0.18762601192807785\n",
      "bryant: 0.18762601192807785\n",
      "run: 0.1641727604370681\n",
      "dmc: 0.1407195089460584\n",
      "murder: 0.1407195089460584\n"
     ]
    }
   ],
   "source": [
    "# Can potentially conduct NER methods to extract keywords\n",
    "article = \" \".join(content)\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS\n",
    "\n",
    "tfidf_vectorizer = TfidfVectorizer(stop_words=ENGLISH_STOP_WORDS)\n",
    "tfidf_matrix = tfidf_vectorizer.fit_transform([article])\n",
    "feature_names = tfidf_vectorizer.get_feature_names()\n",
    "scores = tfidf_matrix.toarray().flatten()\n",
    "indices = scores.argsort()[::-1]\n",
    "top_n = 10\n",
    "top_features = [(feature_names[i], scores[i]) for i in indices[:top_n]]\n",
    "keywords = \" \".join([feature for feature, score in top_features])\n",
    "for feature, score in top_features:\n",
    "    print(f\"{feature}: {score}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "42b58fe6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Can potentially use Google search API instead\n",
    "# Advanced RAG\n",
    "from gnews import GNews\n",
    "import numpy as np\n",
    "google_news = GNews()\n",
    "max_results = 20\n",
    "# google_news.period = '7d'\n",
    "google_news.max_results = max_results \n",
    "# google_news.country = 'United States'\n",
    "google_news.language = 'english'\n",
    "# google_news.exclude_websites = ['yahoo.com', 'cnn.com'] \n",
    "google_news.start_date = (2020, 1, 1)\n",
    "google_news.end_date = (2024, 2, 3)\n",
    "articles = []\n",
    "news = google_news.get_news(keywords)\n",
    "for i in range(max_results):\n",
    "    try:\n",
    "        article = google_news.get_full_article(\n",
    "            news[i]['url']\n",
    "        )\n",
    "    except:\n",
    "        break\n",
    "    articles.append(article)\n",
    "title_text = [article.title for article in articles if article]\n",
    "article_text = [article.text for article in articles if article]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7a3ecf8f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/zhj003/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain.text_splitter import CharacterTextSplitter\n",
    "from langchain_community.document_loaders import TextLoader\n",
    "from langchain_community.vectorstores import Weaviate\n",
    "import nltk\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ed463a5c",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "02/28/2024 01:37:11 AM - Created a chunk of size 362, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 321, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 355, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 350, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 324, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 378, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 342, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 327, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 344, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 368, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 362, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 357, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 456, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 325, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 337, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 309, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 388, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 301, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 454, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 637, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 316, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 393, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 331, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 311, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 421, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 309, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 387, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 398, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 599, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 356, which is longer than the specified 300\n",
      "02/28/2024 01:37:11 AM - Created a chunk of size 319, which is longer than the specified 300\n"
     ]
    }
   ],
   "source": [
    "# Split the articles into chunks before posting to Weaviate database\n",
    "class Document:\n",
    "    def __init__(self, text):\n",
    "        self.page_content = text\n",
    "        self.metadata = {'source': 'google news'}\n",
    "\n",
    "documents = [Document(article) for article in article_text]\n",
    "text_splitter = CharacterTextSplitter(chunk_size=300, chunk_overlap=0)\n",
    "chunked_articles = text_splitter.split_documents(documents)\n",
    "chunked_articles = [document.page_content for document in chunked_articles]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a1796970",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-28 01:37:28.399610: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2024-02-28 01:37:28.433413: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "# Our tokenized method\n",
    "import torch\n",
    "from transformers import BertTokenizer, BertModel, pipeline\n",
    "\n",
    "def text_embedding(data):\n",
    "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "    bert_model = BertModel.from_pretrained('bert-base-uncased').to(device)\n",
    "    \n",
    "    def get_bert_embeddings(data):\n",
    "        tokens = tokenizer(data.tolist(), padding=True, truncation=True, return_tensors='pt').to(device)\n",
    "        with torch.no_grad():\n",
    "            embeddings = bert_model(**tokens).last_hidden_state.mean(dim=1)\n",
    "        return embeddings\n",
    "\n",
    "    batch_size = 128\n",
    "    num_samples = len(data)\n",
    "    num_batches = (num_samples + batch_size - 1) // batch_size\n",
    "\n",
    "    embeddings_list = []\n",
    "\n",
    "    for i in range(num_batches):\n",
    "        start_idx = i * batch_size\n",
    "        end_idx = (i + 1) * batch_size\n",
    "        batch_data = data.iloc[start_idx:end_idx]\n",
    "        batch_embeddings = get_bert_embeddings(batch_data)\n",
    "        embeddings_list.append(batch_embeddings)\n",
    "\n",
    "    embeddings = torch.cat(embeddings_list, dim=0).cpu().numpy()\n",
    "    return embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "38f6987f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import weaviate\n",
    "import json\n",
    "import pandas as pd\n",
    "\n",
    "client = weaviate.Client(\n",
    "    url = \"https://testing-cluster-2qgcoz4q.weaviate.network\",  # Replace with your endpoint\n",
    "    auth_client_secret=weaviate.auth.AuthApiKey(api_key=\"qRarwGLC0CwrpQsSpK64E1V0c3HajFoAy893\"),  # Replace w/ your Weaviate instance API key\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e407820f",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Article already exists in the database.\n",
      "Article already exists in the database.\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "Article already exists in the database.\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n",
      "not found\n"
     ]
    }
   ],
   "source": [
    "for article in chunked_articles:\n",
    "    # Check for duplicate before posting\n",
    "    query = \"\"\"\n",
    "    {\n",
    "        Get {\n",
    "            Test_dataset_1(where: {\n",
    "                operator: Equal\n",
    "                path: [\"context\"]\n",
    "                valueString: \"%s\"\n",
    "            }) {\n",
    "                _additional {\n",
    "                    id\n",
    "                }\n",
    "            }\n",
    "        }\n",
    "    }\n",
    "    \"\"\" % article.replace('\"', '\\\\\"')\n",
    "    result = client.query.raw(query)\n",
    "    try:\n",
    "        if not result['data']['Get']['Test_dataset_1']:\n",
    "            properties = {\"context\": article}\n",
    "            vector = text_embedding(pd.Series(article)).tolist()[0]\n",
    "            client.data_object.create(properties, \"test_dataset_1\", vector=vector)\n",
    "        else:\n",
    "            print(\"Article already exists in the database.\")\n",
    "    except:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c8d133f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Post data to weaviate cloud db \n",
    "# for article in chunked_articles:\n",
    "#     properties = {\"context\": article}\n",
    "#     vector = text_embedding(pd.Series(article)).tolist()[0]\n",
    "#     client.data_object.create(properties, \"test_dataset_1\", vector=vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d9815791",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "evidence = []\n",
    "for text_query in content:\n",
    "    query_vector = {\"vector\" : text_embedding(pd.Series(text_query)).tolist()[0],\n",
    "                \"distance\" : 1.0\n",
    "    }\n",
    "    results = client.query.get(\"test_dataset_1\", [\"context\"]).with_additional(\"distance\"\n",
    "                ).with_near_vector(query_vector).do()\n",
    "    evidence.append([result[\"context\"] for result in results['data']['Get']['Test_dataset_1'][:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "ea141c1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_google_genai import (\n",
    "    ChatGoogleGenerativeAI,\n",
    "    HarmBlockThreshold,\n",
    "    HarmCategory,\n",
    ")\n",
    "\n",
    "llm = ChatGoogleGenerativeAI(model=\"gemini-pro\", \n",
    "google_api_key=\"AIzaSyClyO_P1azrly9sScfVL3dJnKy8q7HtayU\", \n",
    "                            safety_settings={\n",
    "        HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE,\n",
    "        HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE,\n",
    "        HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE,\n",
    "        HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE,\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "2ff57962",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('3. This claim is 3 (barely true). The evidence provided does not directly address the claim, but there is some evidence that suggests that the claim may be true. For example, Figure 1 shows that the number of people in the United States who are incarcerated has increased significantly over the past few decades. Figure 2 shows that the United States has a higher rate of incarceration than any other country in the world. However, these figures do not provide any information about the reasons for the increase in incarceration rates. Therefore, it is difficult to say for sure whether or not the claim is true.', 3)\n",
      "(\"0. This claim is 0 (True) because the claim is about two men were found guilty of murder Tuesday in the 2002 killing of Jam Master Jay, the pioneering DJ of the groundbreaking hip hop trio Run-DMC, in a case that for decades frustrated detectives and music fans alike. The evidence above supports this claim by providing information about the trial, the charges against the men, and the victim's identity. The evidence also shows that the case had been unsolved for nearly two decades, which is consistent with the claim that the murder frustrated detectives and music fans.\", 0)\n",
      "('0. This claim is 0 (True) because Ronald Washington and Karl Jordan Jr. were both convicted of murder while engaged in narcotics trafficking and firearm-related murder for the killing of Jason Mizell, also known as \"Jam Master Jay.\"', 0)\n",
      "('4. This claim is 4 (false). The evidence provided does not directly address the claim that a fight nearly broke out in the courtroom after the verdict was announced. However, the evidence does state that both Washington and Jordan were rushed out of the courtroom, suggesting that the situation was tense and could have escalated into a fight.', 4)\n",
      "('1. This claim is 0 (True) because the evidence clearly states that \"Carlis Thompson, Jay’s cousin, praised the verdict outside court but lamented that many family members did not live to see this day.\"', 1)\n",
      "(\"3. The evidence does not directly address the claim, but it does provide some context that suggests that the claim is likely true. The evidence shows that McCain was a man of great passion and conviction, and that he was deeply committed to serving his country. It also shows that he was a man of great compassion and empathy, and that he was deeply affected by the loss of his son. Given these qualities, it is likely that McCain would have been pleased to see justice served in the case of his son's death, and that he would have been grateful for the support of his friends and family.\", 3)\n",
      "('0. This claim is 0 (True) because the claim states that Jordan, 40, and Washington, 59, each face a sentence of 20 years to life in prison, which is consistent with the information provided in the evidence that \"If convicted, the men face at least 20 years in prison.\"', 0)\n",
      "(\"0. This claim is 0 (True) because after more than 20 years have passed since the murder of Run-DMC's Jam Master Jay, two men were charged in the killing in 2020, and a third person was indicted in 2023.\", 0)\n",
      "('0. This claim is 0 (True) because there is an indictment alleging Washington and Jordan conspired to kill Jam Master Jay in retaliation for a drug dispute.', 0)\n",
      "('1. This claim is 1 (mostly true) because Bryant has been charged with murder while engaged in narcotics trafficking and firearm-related murder, as prosecutors allege he allowed the defendants to enter the studio through the fire escape door. Bryant has pleaded not guilty and is set to go on trial separately in January 2026. While the evidence does not explicitly state that Bryant allowed the defendants to enter the studio, it is a reasonable inference given the charges against him.', 1)\n",
      "('3. This claim is 3 (barely true) because the trial has featured testimony from witnesses who identified the defendants as the killers. However, the defense lawyers have challenged the credibility of these witnesses and argued that the evidence against their clients is not persuasive. The outcome of the trial is still uncertain, so it is too early to say definitively whether the claim is true.', 3)\n",
      "('0. This claim is 0 (True) because One witness who was at the recording studio that night testified Washington and Jordan came in armed and that Jordan shot Jam Master Jay in the head.', 0)\n",
      "(\"4. This claim is 4 (false) because there is no evidence to support the claim that Karl Jordan Jr. and Ronald Washington are killers. The evidence presented in the claim is only the prosecutor's closing arguments, which are not evidence of guilt.\", 4)\n",
      "('3. This claim is 3 (barely true) because Bryant was seen entering the building right before the shooting occurred, and authorities found clothing with his DNA on it at the scene. Bryant later admitted to participating in the murder and even told someone that he was the shooter. However, prosecutors do not believe he fired the fatal shot.', 3)\n",
      "('0. This claim is 0 (True) because Washington is quoted in a 2003 Playboy magazine article titled The Last Days Of Jam Master Jay – he said the night Jay was murdered, he was heading to Jay’s studio when he heard gunshots and saw Jordan fleeing.', 0)\n",
      "('2. This claim is 2 (half true) because although the defense lawyers have stated that the evidence against their clients is not persuasive and that they will plead not guilty, this does not directly address the claim that “Jay Bryant is literally reasonable doubt in this case.” The evidence provided does not directly support or refute the claim, so it is difficult to determine its veracity.', 2)\n",
      "('0. This claim is 0 (True) because the evidence provided states that Bryant has admitted to participating in the murder, claiming to one associate that he was in fact the shooter.', 0)\n",
      "('2. This claim is 2 (half true).\\n\\nThe claim states that Breon Peace, the US attorney for the Eastern District of New York, praised the witnesses’ bravery in testifying. While the evidence does not directly address this claim, it is reasonable to assume that the US attorney would praise the witnesses for their bravery, given the nature of the case and the challenges they faced in coming forward. Additionally, the evidence provided shows that Breon Peace is a respected and experienced prosecutor, which further supports the claim that he would praise the witnesses for their bravery.', 2)\n",
      "('0. This claim is 0 (True) because the witnesses were terrified that they’d be retaliated against if they cooperated with law enforcement. But their strength and resolve in testifying at this trial were a triumph of right over wrong and courage over fear.', 0)\n",
      "(\"2. This claim is 2 (half true) because although Run-DMC did speak out against drugs and even recorded a public service announcement about it, the evidence suggests that Jam Master Jay may have still been involved in cocaine trafficking in the mid-1990s. The evidence does not provide a clear picture of Jay's involvement in drug trafficking, but it does suggest that he may have been involved to some extent.\", 2)\n",
      "('2. This claim is 2 (half true) because the evidence provided indicates that Mizell had indeed acquired 10 kilograms of cocaine, and that Washington and Jordan were involved in a drug deal with him. However, the evidence does not directly state that the murder was the result of a dispute over a $200,000 drug deal, as the claim suggests. It is possible that the murder was related to the drug deal, but the evidence provided does not conclusively prove this.', 2)\n",
      "('0. This claim is 0 (True) because the trial comes four decades after Jam Master Jay teamed up with Queens friends Run (Joseph Simmons) and DMC (Darryl McDaniels) to form the hip hop group Run-DMC. This is supported by the evidence provided, which states that \"Jay was in Run-DMC with Joseph \"Run\" Simmons and Darryl \"DMC\" McDaniel in the early 1980s.\"', 0)\n",
      "('1. The claim is 1 (mostly true). Run-DMC, consisting of Jay, Run, and DMC, was a trio that helped bring hip-hop into the mainstream in the 1980s with hits like \"It\\'s Tricky\" and a remake of Aerosmith\\'s \"Walk This Way.\" The group was the first hip-hop group to reach mainstream popular music success and appeared on \"Saturday Night Live\" and the cover of Rolling Stone magazine.', 1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('4. The claim is \"Their song with Aerosmith was part of the triple-platinum selling album “Raising Hell,” which also included the hits “It’s Tricky,” “My Adidas” and “You Be Illin’.” The album was nominated for a Grammy Award for Best R&B Vocal Performance by a Duo or Group.\"\\n\\nThe evidence provided does not directly address whether the song with Aerosmith was part of the album \"Raising Hell\" or whether the album was nominated for a Grammy Award. However, the evidence does state that Run-DMC had a hit song with Aerosmith called \"Walk This Way\" and that the group was nominated for a Grammy Award in 1985. Therefore, it is likely that the claim is true.', 4)\n",
      "('1. The claim states that Run-DMC released a total of six albums between 1984 and 1993, with “King of Rock” and “Tougher Than Leather” also reaching platinum status. \\n\\nThe evidence provided includes several references to Run-DMC\\'s successful albums, including \"King of Rock\" and \"Tougher Than Leather\" reaching platinum status. However, there is no specific mention of the total number of albums released by the group. \\n\\nTherefore, the evidence is inconclusive, and I cannot rate the veracity of the claim based solely on the evidence provided.', 1)\n",
      "('0. This claim is 0 (True) because the evidence provided by the second bullet point clearly states that Run-DMC was inducted into the Rock and Roll Hall of Fame in 2009.', 0)\n",
      "('2. This claim is 2 (half true) because Run-D.M.C. was one of the first rap groups to be featured prominently on MTV, appear on Saturday Night Live, grace the cover of Rolling Stone, and win a Grammy Lifetime Achievement Award. However, they were not the first in all of these categories. For example, The Sugarhill Gang was the first rap group to appear on Saturday Night Live in 1981, and Kurtis Blow was the first rapper to be featured on the cover of Rolling Stone in 1984.', 2)\n",
      "(\"0. This claim is 0 (True) because 'This story has been updated with additional information' clearly indicates that the story has been updated with additional information.\", 0)\n",
      "('4. This claim is 4 (false) because Charlie Crist has actually been critical of the Medicare Advantage cuts for years. He specifically said he opposed the reductions in 2009 and 2010, and he still opposes them today. Crist doesnt appear to have come up with other ways to save money on health care without reducing payments to Medicare Advantage.', 4)\n"
     ]
    }
   ],
   "source": [
    "def evaluate_claim(claim, evidence):\n",
    "    prompt = f\"\"\"\n",
    "    \n",
    "    Firstly, please consider and understand the scales below:\n",
    "    Here are some examples of some claims and their veracity classifications on a scale of from 0 to 5,\n",
    "    with 0 = completely true, 1 = mostly true, 2 = half true, 3 = barely true, \n",
    "    4 = false and 5 = entirely false or pants on fire.\n",
    "    \n",
    "    Secondly, please consider several examples of justification to understand the standard:\n",
    "    \n",
    "    Claim 1: 'Hillary Clinton in 2005 co-sponsored legislation that would jail flag burners.'\n",
    "    Classification 1: '0. This claim is 0 (True) because Hillary Clinton co-sponsored legislation in 2005 that would jail flag burners.'\n",
    "    \n",
    "    Claim 2: ''On military recruiters at Harvard, Elena Kagan took a position and the Supreme Court ruled unanimously that she was wrong.'\n",
    "    Classification 2: '0. This claim is 0 (True) because the court did  reject the arguments put forward by the law schools, \n",
    "    which included FAIR  and the brief filed by the Harvard professors. All of the justices who voted opposed the law schools arguments'\n",
    "    \n",
    "    Claim 3: 'Have the suburbs been inundated with former residents of Atlanta housing projects? Absolutely not.'\n",
    "    Classification 3: ''0. This claim is 0 (True) because DeKalb Countys population is slightly more than 747,000, the Census Bureau data show.\n",
    "    The data from the AHA and Georgia State both reach similar conclusions that the percentage of tenants moving out of the city has been small.'\n",
    "    \n",
    "    Claim 4: 'Under legislation that has cleared the Georgia House, some children who are legal refugees \n",
    "    could obtain state scholarships to attend private schools.'\n",
    "    Classification 4: '0. This claim is 0(True) because legislation has cleared the Georgia House that would expand the list of students eligible for a \n",
    "    private school scholarship program created in 2007. The scholarships are now offered in varying amounts to students with disabilities.\n",
    "    The bill would open the program to about 700 legal refugees who are not proficient in English.'\n",
    "    \n",
    "    Claim 5: 'Hillary Clinton agrees with John McCain by voting to give George Bush the benefit of the doubt on Iran.'\n",
    "    Classification 5: '1. This claim is 1 (mostly true). Although Clinton may have \"agreed\" with McCain on the issue, they did not technically vote the same way on it. \n",
    "    To say that voting for Kyl-Lieberman is giving George Bush the benefit of the doubt on Iran remains a contentious issue. But Obamas main point is that Clinton and McCain \n",
    "    were on the same side, and that is correct.'\n",
    "    \n",
    "    Claim 6: 'Mark Pryor votes with Obama 93 percent of the time.'\n",
    "    Classification 6: '1. This claim is 1 (mostly true) because Since Obama became president, Pryor has voted in line with the \n",
    "    presidents positions between 90 and 95 percent of the time, with 92. 6 percent -- basically 93 percent -- as the average, \n",
    "    according to the best rating system at our disposal. Pryor doesnt vote with the Democratic Party quite as often, though, \n",
    "    and in 2013 his presidential support votes were lower than every other Senate Democrat. Cottons number is on target based on \n",
    "    the data, but Pryor has also opposed Obama on a few key issues.'\n",
    "    \n",
    "    Claim 7: 'Numerous studies have shown that these so-called right-to-work laws do not generate jobs and economic growth.'\n",
    "    Classification 10: '2. This claim is 2 (half true) because Right-to-work states have seen greater job increases, but one economist pointed out that such a dynamic doesnt prove\n",
    "    right-to-work laws were the cause. Another professor who believes job growth results from right-to-work laws acknowledged that many other factors also could be responsible.'\n",
    "    \n",
    "    Claim 8: 'When did the decline of coal start? It started when natural gas took off that started to begin in President George W. Bushs administration.'\n",
    "    Classification 11: '2. This claim is 2 (half true) because there is no doubt, natural gas has been gaining ground on coal in generating electricity. \n",
    "    The trend started in the 1990s but clearly gained speed during the Bush administration when the production of natural gas -- a competitor of coal -- picked up. \n",
    "    But analysts give little credit or blame to Bush for that trend. They note that other factors, such as technological innovation, entrepreneurship and\n",
    "    policies of previous administrations, had more to do with laying the groundwork for the natural gas boom.'\n",
    "\n",
    "    Claim 9: 'Hillary Clinton said the Veterans Affairs scandal is over-exaggerated. She said she was satisfied with what was going on.'\n",
    "    Classification 7: 'This claim is 3 (barely true) because while Clinton has said problems at the VA have not been as widespread as it has been made out to be,\n",
    "    she has also acknowledged systemic problems within the system and repeatedly urged reform so veterans can get care quickly.'\n",
    "    \n",
    "    Claim 10: 'the paperback edition of Mitt Romneys book deleted line that Massachusetts individual mandate should be the model for the country'\n",
    "    Classification 8: '3. This claim is 3 (barely true) because Perry's right that Romney's comments about health care were edited between editions. Among other things, \n",
    "    a line that advocated the Massachusetts model as a strong option for other states was replaced by a shorter, more generic sentence. But that line was preceded\n",
    "    by an argument for state-level solutions, exactly the argument Romney extends now. That's not how Perry characterized it.'\n",
    "    \n",
    "    Claim 11: 'This year in Congress Connie Mack IV has missed almost half of his votes.'\n",
    "    Classification 9: '3. This claim is 3 (barely true) because Mack has more votes than the average member of\n",
    "    the U. S.  House of Representatives, and hes missed high-profile votes on health care and the budget. He hasnt missed almost half of his votes, though.'\n",
    "    \n",
    "    Claim 12: 'Of Virginias 98,000 teachers who are K-12, over 53,000 of those teachers today are over 50 years old.'\n",
    "    Classification 12: '4. This claim is 4 (false) because Virginia does not keep data that is solely focused on the age of teachers. The best statistics available show that the states \n",
    "    instructional staff -- including teachers, librarians, guidance counselors and technology instructors -- was 98,792 during the 2010-11 school year and, of them, 33,462 were 50 or older.\n",
    "    The teaching corps is not moving toward retirement with anything close to the speed described.'\n",
    "    \n",
    "    Claim 13: 'What the Obama administration is going to come out with in the next several months is youre not even going to be able to burn coal very \n",
    "    limitedly in the existing plants.'\n",
    "    Classification 13: '4. This claim is 4 (false) because The proposal the claim is referring to is an EPA plan to cut carbon emissions in existing power plants. Those rules do not prohibit \n",
    "    current facilities from burning coal, and even Capitos spokeswoman said the rule doesnt mean that every plant has to close. Some facilities will close down within the next decade, \n",
    "    but many of those plants were scheduled to be retired anyway due to age and other factors. States and power companies have options to continue to utilize coal for energy, and experts said \n",
    "    they expect coal to remain part of the national portfolio for years to come.'\n",
    "    \n",
    "    Claim 14: 'Charlie Crist supports cuts to the Medicare Advantage program.'\n",
    "    Classification 14: '4. This claim is 4 (false) because Crist has flip-flopped on a lot of issues, including the federal health care law. He used to oppose the Affordable Care Act, \n",
    "    but now he supports it. The law tries to bring down future health care costs by reducing Medicare Advantage payments. But on the issue of Medicare Advantage, Crist has actually been\n",
    "    consistent: Hes been critical of the Medicare Advantage cuts for years. He specifically said he opposed the reductions in 2009 and 2010, and he \n",
    "    still opposes them today. Crist doesnt appear to have come up with other ways to save money on health care without reducing payments to Medicare Advantage.'\n",
    "    \n",
    "    Claim 15: 'Says Ohio budget item later signed into law by Gov. John Kasich requires women seeking an abortion to undergo a mandatory vaginal probe.'\n",
    "    Classification 15: '5. This claim is 5 (entirely false/fabricated) because there should be no debate about what types of ultrasounds these new regulations require. there is a mandate \n",
    "    -- but for external detection methods.'\n",
    "    \n",
    "    Claim 16: 'Roughly 25% of RI has a criminal record'\n",
    "    Classification 16: '5. This claim is 5 (entirely false/fabricated) because They cant be sure of the exact percentage because of the way the data is collected, but theyre certain its nowhere near 25 percent. \n",
    "    DAREs statement is based on old, flawed statistics that were tweaked and re-tweaked to make a point.'\n",
    "    \n",
    "    Claim 17: 'We spend less on defense today as percentage of GDP than at any time since Pearl Harbor.'\n",
    "    Classification 17: '5. This claim is 5 (entirely false/fabricated) because since the Cold War ended, and in a few other years as well, the percent of GDP used for defense has been consistently lower \n",
    "    than current spending levels.'\n",
    "    \n",
    "    Thirdly, Please rate the veracity of the following claim on the same 0 to 5 scale. Ensure that the first character \n",
    "    in your response is a single integer between 0 and 5, and explain your reasoning like the examples above, citing\n",
    "    any evidence in a chain of thought fashion. {claim} \n",
    "    \n",
    "    Fourthly, please consider the the evidence of this claim: {evidence}\n",
    "    \n",
    "    Finally, if the evidence is inconclusive or does not directly address the claim, please base your rating on your knowledge and indicate the lack of direct evidence in your explanation.\n",
    "    \"\"\"\n",
    "\n",
    "    response = llm.invoke(prompt).content\n",
    "\n",
    "    rating = response[0]\n",
    "\n",
    "    return response, int(rating)\n",
    "\n",
    "for i in range (len(content)):\n",
    "    print(evaluate_claim(content[i], evidence[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d23770d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}