predict_liar_factors.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4c78c872",
   "metadata": {},
   "outputs": [],
   "source": [
    "# For sanity only\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7ce01c84",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "\n",
    "def scrape_site(url):\n",
    "    response = requests.get(url)\n",
    "\n",
    "    if response.status_code == 200:\n",
    "        soup = BeautifulSoup(response.content, 'html.parser')\n",
    "        \n",
    "        # Extract header\n",
    "        header = soup.find(['h1']).get_text().strip()\n",
    "\n",
    "        # Extract content\n",
    "        content_tags = soup.find_all(['p'])\n",
    "        content = [tag.get_text().strip().replace('\\xa0', ' ') for tag in content_tags]\n",
    "\n",
    "        # Find the keyword 'By' to extract the author's name\n",
    "        page_text = soup.get_text()\n",
    "        match = re.search(r'\\bBy\\s+([A-Za-z\\s.,]+)', page_text)\n",
    "        authors = match.group(1).strip().replace('and', ',') if match else 'Author not found'\n",
    "        author_lst = [auth.strip() for auth in authors.split(',')]\n",
    "        return header, content, author_lst\n",
    "    else:\n",
    "        print(f\"Failed to retrieve the webpage. Status code: {response.status_code}\")\n",
    "        return None, None, None\n",
    "\n",
    "url = \"https://www.cnn.com/2024/01/17/politics/biden-ukraine-white-house-meeting/index.html\"\n",
    "header, content, authors = scrape_site(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f893cdd7",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-03-05 17:58:03.345235: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "import context_veracity\n",
    "def context_veracity_score(content):\n",
    "    content_concat = \" \".join(content)\n",
    "    sent_score = context_veracity.sentiment_shift(content_concat)\n",
    "    topic_score = context_veracity.topic_shift(content_concat)\n",
    "    ner_score = context_veracity.ner_shift(content_concat)\n",
    "    context_veracity_score = context_veracity.calculate_contextual_drift(sent_score, topic_score, ner_score)\n",
    "    return context_veracity_score\n",
    "# print(f\"The context veracity score of the article is {context_veracity_score(content)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7ac6796a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/zhj003/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /home/zhj003/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /home/zhj003/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n"
     ]
    }
   ],
   "source": [
    "import political_bias\n",
    "import numpy as np\n",
    "def political_bias_score(content):\n",
    "    pred_labels = []\n",
    "    political_bias.download_pretrained_model()\n",
    "    for paragraph in content:\n",
    "        processed_article = political_bias.preprocess_article(header, paragraph)\n",
    "        label = political_bias.predict_label(processed_article)\n",
    "        pred_labels.append(label[0])\n",
    "    poli_bias_score = np.mean(pred_labels)\n",
    "    return poli_bias_score\n",
    "# print(\"The political bias score of the article is \", political_bias_score(content))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f2861538",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sentiment import SentimentModel\n",
    "# import pickle\n",
    "# import numpy as np\n",
    "# def sentiment_score(content):\n",
    "#     with open('models/sentiM.pkl', 'rb') as f:\n",
    "#         sentM = pickle.load(f)\n",
    "#     pred_labels = []\n",
    "#     for paragraph in content:\n",
    "#         label = sentM.predict_article(header, paragraph)[0]\n",
    "#         pred_labels.append(label)\n",
    "#     sent_score = np.mean(pred_labels)\n",
    "#     return sent_score\n",
    "# # print(\"The sentiment score of the article is \", sentiment_score(content))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c0f36bdc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import pipeline\n",
    "def sentiment_score(content):\n",
    "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "    distilled_student_sentiment_classifier = pipeline(\n",
    "        model=\"lxyuan/distilbert-base-multilingual-cased-sentiments-student\", \n",
    "        return_all_scores=False,\n",
    "        device=device\n",
    "    )\n",
    "    pred_labels = []\n",
    "    for paragraph in content:\n",
    "        result = distilled_student_sentiment_classifier(paragraph)[0]['label']\n",
    "        if result == 'positive':\n",
    "            pred_labels.append(0)\n",
    "        elif result == 'negative':\n",
    "            pred_labels.append(2)\n",
    "        else:\n",
    "            pred_labels.append(1)\n",
    "    return np.mean(pred_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "36a8e8d9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import html\n",
    "import re\n",
    "import credibility\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "def credibility_score(authors):\n",
    "    with open('models/credibility_model.pkl', 'rb') as f:\n",
    "        cred_model = pickle.load(f)\n",
    "\n",
    "    search_results = []\n",
    "    for author in authors:\n",
    "        search_results.append(credibility.search_wikipedia(author, num_results=15))\n",
    "\n",
    "    search_pd = pd.DataFrame(search_results, columns=['text'])\n",
    "    embedded_result = credibility.text_embedding(search_pd['text'])[:, :50]\n",
    "    cred_scores = cred_model.predict(embedded_result)\n",
    "    if len(cred_scores) == 1:\n",
    "        print(f'The score of {authors[0]} is {cred_scores[0]}')\n",
    "        return cred_scores[0]\n",
    "    else:\n",
    "        for i in range(len(authors)):\n",
    "            print(f'The score of {authors[i]} is {cred_scores[i]}')\n",
    "        cred_score = np.mean(cred_scores)\n",
    "        return cred_score\n",
    "# print(f'The credibility score is {credibility_score(authors)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5f78f1ff",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import text_manipulation\n",
    "import numpy as np\n",
    "def style_score(content):\n",
    "    text_manipulation.download_pretrained_model()\n",
    "    pred_labels = []\n",
    "    for paragraph in content:\n",
    "        label = text_manipulation.predict(paragraph)\n",
    "        pred_labels.append(label)\n",
    "    style_score = np.mean(pred_labels)\n",
    "    return style_score\n",
    "# print(\"The text manipulation (style) score of the article is \", style_score(content))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d96e1cbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "from spam import SpamModel\n",
    "import pickle\n",
    "def spam_score(header):\n",
    "    with open('models/spamM.pkl', 'rb') as f:\n",
    "        spamM = pickle.load(f)\n",
    "    pred_label = spamM.predict_article(header)[0]\n",
    "    if pred_label:\n",
    "        spam_score = 1\n",
    "    else:\n",
    "        spam_score = 0\n",
    "    return spam_score\n",
    "# print(\"The spam score of the article headline is \", spam_score(header))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "83f0965d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def source_reliability_score(content):\n",
    "    with open('models/srcM.pkl', 'rb') as f:\n",
    "        srcM = pickle.load(f)\n",
    "    pred_labels = []\n",
    "    for paragraph in content:\n",
    "        label = srcM.predict_text(paragraph)[0]\n",
    "        pred_labels.append(label)\n",
    "    return np.mean(pred_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1398664d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clickbait_score(header):\n",
    "    with open('models/clickM.pkl', 'rb') as f:\n",
    "        clickM = pickle.load(f)\n",
    "    label, proba = clickM.predict_text(header)\n",
    "    return label[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "1def6c59",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv('original_files/politifact_data_combined.csv')\n",
    "\n",
    "# df = df.drop(columns = [0])\n",
    "# df.rename({1: 'id', 2: 'label', 3: 'statement', 4: 'subject', 5: 'speaker', 6: 'job-title',\n",
    "#            7: 'state_info', 8: 'party_affiliation', 9: 'barely_true_counts', 10: 'false_counts',\n",
    "#            11: 'half_true_counts', 12: 'mostly_true_counts', 13: 'pants_on_fire_counts', 14: 'context',\n",
    "#            15: 'justification'\n",
    "#           }, axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3aa9c3a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "label_map = {'pants-fire': 5, 'false': 4, 'half-true': 3,\n",
    "             'barely-true': 2, 'mostly-true': 1, 'true': 0}\n",
    "df['documented_time'] = pd.to_datetime(df['documented_time'])\n",
    "df = df[~df['label'].isin({'full-flop', 'half-flip', 'no-flip'})]\n",
    "df['label'] = df['label'].replace(label_map)\n",
    "df = df[df['documented_time'].dt.year >= 2022]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "6588dbcf",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running at iteration 3050\n",
      "File 'models/poli_bias_bert.pkl' already exists. No download needed.\n",
      "The score of Tom Kertscher is 4.742999999999999\n",
      "File 'models/txt_manipulation_model.pt' already exists. No download needed.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of RobertaModel were not initialized from the model checkpoint at roberta-base and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "from IPython.display import clear_output\n",
    "\n",
    "scores = []\n",
    "for index, row in df.iterrows():\n",
    "    content = [row['content']]\n",
    "    authors = [row['speaker']]\n",
    "    try:\n",
    "        pb_score = political_bias_score(content)\n",
    "    except:\n",
    "        pb_score = None\n",
    "    try:\n",
    "        sent_score = sentiment_score(content)\n",
    "    except:\n",
    "        sent_score = None\n",
    "    try:\n",
    "        cred_score = credibility_score(authors)\n",
    "    except:\n",
    "        cred_score = None\n",
    "    try:\n",
    "        sty_score = style_score(content)\n",
    "    except:\n",
    "        style_score = None\n",
    "    try:\n",
    "        reliability_score = source_reliability_score(content)\n",
    "    except:\n",
    "        reliability_score = None\n",
    "    scores.append([pb_score, sent_score, cred_score, sty_score, reliability_score])\n",
    "    if index % 50 == 0:\n",
    "        clear_output(wait=True)\n",
    "        print(f\"Running at iteration {index}\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e03bf0dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores_df = pd.DataFrame(scores, columns=['Political_Bias', 'Sentiment', 'Credibility', 'Style', 'Reliability'])\n",
    "df = pd.concat([df, scores_df], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6e149a43",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('original_files/politifact_data_2022_score.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "5285f458",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min_val, max_val = 4, 6\n",
    "\n",
    "def normalization(score):\n",
    "    if max_val - min_val == 0:  # Check for zero division\n",
    "        return 0\n",
    "    elif score > max_val:\n",
    "        return 1\n",
    "    else:\n",
    "        return (score - min_val) / (max_val - min_val)\n",
    "normalization(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34b494ac",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}