data_collection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "accessing appropriate pages on MediaBiasFactCheck to collect sites to visit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "science_url = 'https://mediabiasfactcheck.com/pro-science/'\n",
    "conspiracy_url = 'https://mediabiasfactcheck.com/conspiracy/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "\n",
    "science_page = requests.get(science_url, timeout=5)\n",
    "conspiracy_page = requests.get(conspiracy_url, timeout=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "parsing through both pages and finding all sites that can be visited"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "\n",
    "science_soup = BeautifulSoup(science_page.content, 'lxml')\n",
    "science_sites = science_soup.find_all('span', {'style': 'font-size: 12pt;'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "conspiracy_soup = BeautifulSoup(conspiracy_page.content, 'lxml')\n",
    "conspiracy_sites = conspiracy_soup.find_all('span', {'style': 'font-size: 12pt;'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "extracting all science and conspiracy pages that can be visited"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_db(sites):\n",
    "    # will ensure request can deal with all links\n",
    "    def format_links(link):\n",
    "        if not link.startswith(('https://www', \"http://www.\", \"http://\", \"https://\")):\n",
    "            if not link.startswith(('www.')):\n",
    "                link = 'www.' + link\n",
    "            if not link.startswith(('http://', 'https://')):\n",
    "                link = 'https://' + link\n",
    "        return link\n",
    "\n",
    "    db = []\n",
    "    for site in sites:\n",
    "        link = format_links(site.text[site.text.rfind('(')+1:-1])\n",
    "        source = site.text[:site.text.rfind('(')-1]\n",
    "\n",
    "        if link.count(' ') == 0 and len(source) >= 2:\n",
    "            db.append({\"name\": source, \"url\": link})    \n",
    "    return db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "science_db = create_db(science_sites)\n",
    "conspiracy_db = create_db(conspiracy_sites)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'name': 'Acoustics Journal',\n",
       "  'url': 'https://www.mdpi.com/journal/acoustics'},\n",
       " {'name': 'Acta Neuropathologica Communications',\n",
       "  'url': 'https://www.actaneurocomms.biomedcentral.com'},\n",
       " {'name': 'Actuators Journal',\n",
       "  'url': 'https://www.mdpi.com/journal/actuators'},\n",
       " {'name': 'Administrative Sciences Journal',\n",
       "  'url': 'https://www.mdpi.com/journal/admsci'},\n",
       " {'name': 'Adolescents Journal',\n",
       "  'url': 'https://www.mdpi.com/journal/adolescents'},\n",
       " {'name': 'Advances in Respiratory Medicine',\n",
       "  'url': 'https://www.mdpi.com/journal/arm'},\n",
       " {'name': 'Aerospace Journal',\n",
       "  'url': 'https://www.mdpi.com/journal/aerospace'},\n",
       " {'name': 'Agriculture Journal',\n",
       "  'url': 'https://www.mdpi.com/journal/agriculture'},\n",
       " {'name': 'AgriEngineering Journal',\n",
       "  'url': 'https://www.mdpi.com/journal/agriengineering'},\n",
       " {'name': 'Agrochemicals Journal',\n",
       "  'url': 'https://www.mdpi.com/journal/agrochemicals'}]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "science_db[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'name': '2020ElectionCenter.c', 'url': 'https://www.2020ElectionCenter.co'},\n",
       " {'name': '21st Century Wire', 'url': 'https://www.21stcenturywire.com'},\n",
       " {'name': '79Days.Ne', 'url': 'https://www.79Days.New'},\n",
       " {'name': '369 News', 'url': 'https://www.369news.net'},\n",
       " {'name': '911Truth.org', 'url': 'https://www.911truth.org'},\n",
       " {'name': 'Above Top Secret', 'url': 'https://www.abovetopsecret.com'},\n",
       " {'name': 'A Call for an Uprising',\n",
       "  'url': 'https://www.acallforanuprising.com'},\n",
       " {'name': 'ACNLatitudes', 'url': 'https://www.latitudes.org'},\n",
       " {'name': 'Activist Post', 'url': 'https://www.activistpost.com'},\n",
       " {'name': 'Actualized.o', 'url': 'https://www.Actualized.or'}]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conspiracy_db[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "validation to ensure all links have valid headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib.parse import urlparse\n",
    "\n",
    "def is_valid_url(url):\n",
    "    try:\n",
    "        result = urlparse(url)\n",
    "        return all([result.scheme in ['http', 'https'], result.netloc])\n",
    "    except ValueError:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert [is_valid_url(page['url']) for page in science_db].count(False) == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert [is_valid_url(page['url']) for page in conspiracy_db].count(False) == 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "saving both lists to ensure we don't have to scrape MediaBiasFactCheck every single time we want to find new articles from these websites for the database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "science_json = json.dumps(science_db, indent=4)\n",
    "conspiracy_json = json.dumps(conspiracy_db, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/website_metadata/science.json', 'w') as file:\n",
    "    file.write(science_json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/website_metadata/conspiracy.json', 'w') as file:\n",
    "    file.write(conspiracy_json)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "collecting articles related to COVID-19 using the ArticleFinder class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/isobarbaric/Desktop/code/projects/COVID19-Classifier/src/model/article_finder.py:56: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.\n",
      "  soup = BeautifulSoup(page['html'], 'lxml')\n"
     ]
    }
   ],
   "source": [
    "from src.article_finder import ArticleFinder\n",
    "\n",
    "# finding the scientific articles\n",
    "science_articles = ArticleFinder.find_articles(article_type=\"science\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'title': 'Can Artificial Intelligence Aid Diagnosis by Teleguided Point-of-Care Ultrasound? A Pilot Study for Evaluating a Novel Computer Algorithm for COVID-19 Diagnosis Using Lung Ultrasound',\n",
       "  'link': 'https://www.mdpi.com/journal/ai/2673-2688/4/4/44'},\n",
       " {'title': 'Changes in Air Quality, Meteorology and Energy Consumption during the COVID-19 Lockdown and Unlock Periods in India',\n",
       "  'link': 'https://www.mdpi.com/journal/air/2813-4168/1/2/10'},\n",
       " {'title': 'Using Low-Cost Sensing Technology to Assess Ambient and Indoor Fine Particulate Matter Concentrations in New York during the COVID-19 Lockdown',\n",
       "  'link': 'https://www.mdpi.com/journal/air/2813-4168/1/3/15'},\n",
       " {'title': 'Children and COVID-19 Vaccination Trends',\n",
       "  'link': 'https://www.aap.org/en/pages/2019-novel-coronavirus-covid-19-infections/children-and-covid-19-vaccination-trends/'},\n",
       " {'title': 'COVID-19 State-Level Data Reports',\n",
       "  'link': 'https://www.aap.org/en/pages/2019-novel-coronavirus-covid-19-infections/children-and-covid-19-state-level-data-report/'},\n",
       " {'title': '\\\\n\\\\t\\\\t\\\\n\\\\t\\\\t\\\\t\\\\t\\\\n\\\\t\\\\t\\\\t\\\\t\\\\n \\\\n\\\\n \\\\n \\\\n \\\\n \\\\n \\\\n \\\\n \\\\n \\\\n \\\\n \\\\n\\\\n\\\\n \\\\n\\\\n\\\\n\\\\t\\\\t\\\\t\\\\t \\\\n\\\\t\\\\t\\\\t\\\\n\\\\t\\\\t\\\\t\\\\n\\\\t\\\\t\\\\t\\\\t\\\\n\\\\t\\\\t\\\\t\\\\t\\\\n\\\\t\\\\t\\\\t\\\\t\\\\tHPV Prevention\\\\n\\\\t\\\\t\\\\t\\\\t\\\\t\\\\n\\\\t\\\\t\\\\t\\\\t\\\\n\\\\t\\\\t\\\\t\\\\t\\\\n\\\\t\\\\t\\\\t\\\\t\\\\t\\\\n\\\\t\\\\t\\\\t\\\\t\\\\t\\\\tHuman Papillomavirus can cause 6 types of cancer. But preventing it \\\\xe2\\\\x80\\\\x94 and the cancers it causes \\\\xe2\\\\x80\\\\x94 is simple.\\\\r\\\\n\\\\n\\\\t\\\\t\\\\t\\\\t\\\\t\\\\t\\\\n\\\\t\\\\t\\\\t\\\\t\\\\t\\\\n\\\\t\\\\t\\\\t\\\\t\\\\n\\\\t\\\\t\\\\t\\\\n\\\\t\\\\t\\\\n\\\\t',\n",
       "  'link': 'https://www.cancer.org/cancer/risk-prevention/hpv/hpv-vaccine.html'},\n",
       " {'title': 'COVID-19',\n",
       "  'link': 'https://www.lung.org/lung-health-diseases/lung-disease-lookup/covid-19'},\n",
       " {'title': \"\\\\n \\\\n End Youth Vaping\\\\n Let\\\\'s join together to end the youth vaping epidemic by supporting parents, schools and students.\\\\n \\\\n\",\n",
       "  'link': 'https://www.lung.org/quit-smoking/end-youth-vaping'},\n",
       " {'title': 'What doctors wish patients knew about COVID-19...',\n",
       "  'link': 'https://www.ama-assn.org/delivering-care/public-health/what-doctors-wish-patients-knew-about-covid-19-oral-antivirals'},\n",
       " {'title': 'Overdose Epidemic',\n",
       "  'link': 'https://www.ama-assn.org/delivering-care/overdose-epidemic'}]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "science_articles[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "169"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(science_articles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# finding the conspiracy articles\n",
    "conspiracy_articles = ArticleFinder.find_articles(article_type=\"conspiracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'title': 'Italian Health Minister Under Investigation for Murder for Concealing COVID-19 Vaccine Deaths',\n",
       "  'link': 'https://www.abovetopsecret.com/forum/thread1342167/pg1'},\n",
       " {'title': 'COVID vaccination rates \\\\xe2\\\\x80\\\\x98alarmingly\\\\xe2\\\\x80\\\\x99 low among nursing home staff',\n",
       "  'link': 'https://www.abovetopsecret.com/forum/thread1341843/pg1'},\n",
       " {'title': '25% of COVID Vaxxed Now Have VAIDS, Cambridge Scientists Warn',\n",
       "  'link': 'https://www.abovetopsecret.com/forum/thread1341936/pg1'},\n",
       " {'title': 'Part 5. COVID Vaccines: The Chicken or the Egg? (From \\\\xe2\\\\x80\\\\x9cThe WHO\\\\xe2\\\\x80\\\\x99s Proposed Treaty Will Increase Man-Made Pandemics\\\\xe2\\\\x80\\\\x9d)',\n",
       "  'link': 'https://ahrp.org/part-5-covid-vaccines-the-chicken-or-the-egg/'},\n",
       " {'title': 'Part 5. COVID Vaccines: The Chicken or the Egg? (From “The WHO\\\\xe2\\\\x80\\\\x99s Proposed Treaty Will Increase Man-Made Pandemics”)',\n",
       "  'link': 'https://ahrp.org/part-5-covid-vaccines-the-chicken-or-the-egg/'},\n",
       " {'title': 'Part 8. The Genomic Sequencing Conundrum (From \\\\xe2\\\\x80\\\\x9cThe WHO\\\\xe2\\\\x80\\\\x99s Proposed Treaty Will Increase Man-Made Pandemics\\\\xe2\\\\x80\\\\x9d)',\n",
       "  'link': 'https://ahrp.org/part-8-the-genomic-sequencing-conundrum/'},\n",
       " {'title': 'Part 3. How Would You Create a Biological Warfare Agent? (From \\\\xe2\\\\x80\\\\x9cThe WHO\\\\xe2\\\\x80\\\\x99s Proposed Treaty Will Increase Man-Made Pandemics\\\\xe2\\\\x80\\\\x9d)',\n",
       "  'link': 'https://ahrp.org/part-3-how-would-you-create-a-biological-warfare-agent/'},\n",
       " {'title': 'Part 4. Maiming with Myocarditis (From \\\\xe2\\\\x80\\\\x9cThe WHO\\\\xe2\\\\x80\\\\x99s Proposed Treaty Will Increase Man-Made Pandemics\\\\xe2\\\\x80\\\\x9d)',\n",
       "  'link': 'https://ahrp.org/part-4-maiming-with-myocarditis/'},\n",
       " {'title': 'Part 3. How Would You Create a Biological Warfare Agent? (From “The WHO\\\\xe2\\\\x80\\\\x99s Proposed Treaty Will Increase Man-Made Pandemics”)',\n",
       "  'link': 'https://ahrp.org/part-3-how-would-you-create-a-biological-warfare-agent/'},\n",
       " {'title': 'Part 7. The WHO\\\\xe2\\\\x80\\\\x99s Draft Pandemic Treaty Incentivizes Gain-of-Function Research (From \\\\xe2\\\\x80\\\\x9cThe WHO\\\\xe2\\\\x80\\\\x99s Proposed Treaty Will Increase Man-Made Pandemics\\\\xe2\\\\x80\\\\x9d)',\n",
       "  'link': 'https://ahrp.org/part-7-the-whos-draft-pandemic-treaty-incentivizes-gain-of-function-research/'}]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conspiracy_articles[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "355"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(conspiracy_articles)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "adding new articles found to the existing database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# adds new articles discovered to existing database (extending the database)\n",
    "def consolidate_data(existing_data, new_data):\n",
    "    def remove_dup(curr):\n",
    "        titles = set()\n",
    "        rev = []\n",
    "        for item in curr:\n",
    "            if item['title'] in titles:\n",
    "                continue\n",
    "            rev.append(item)\n",
    "            titles.add(item['title'])\n",
    "        return rev\n",
    "\n",
    "    print(f\"Before combining: {len(existing_data)}\")\n",
    "    existing_data += new_data\n",
    "    existing_data = remove_dup(existing_data)\n",
    "    print(f\"After combining and removing duplicates: {len(existing_data)}\")\n",
    "    \n",
    "    return remove_dup(existing_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "saving JSON results obtained from scraping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f\"data/article_metadata/science.json\") as file:\n",
    "    existing_science_articles = json.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f\"data/article_metadata/conspiracy.json\") as file:\n",
    "    existing_conspiracy_articles = json.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before combining: 705\n",
      "After combining and removing duplicates: 728\n"
     ]
    }
   ],
   "source": [
    "updated_science_articles = consolidate_data(existing_data=existing_science_articles, \n",
    "                                            new_data=science_articles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before combining: 1143\n",
      "After combining and removing duplicates: 1187\n"
     ]
    }
   ],
   "source": [
    "updated_conspiracy_articles = consolidate_data(existing_data=existing_conspiracy_articles,\n",
    "                                               new_data=conspiracy_articles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f\"data/article_metadata/science.json\", 'w') as file:\n",
    "    json.dump(updated_science_articles, file, indent = 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f\"data/article_metadata/conspiracy.json\", 'w') as file:\n",
    "    json.dump(updated_conspiracy_articles, file, indent = 4)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "covid19-classifier-EHzMhhWe-py3.9",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}