fda-drugs/get_fda_approved_drugs_by_year.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_links_from_fda_drugname(table_provided):\n",
    "    \"\"\"\n",
    "    Extracts hyperlinks and corresponding drug names from an HTML table.\n",
    "\n",
    "    Parameters:\n",
    "    - table_provided (BeautifulSoup): HTML table containing drug information.\n",
    "\n",
    "    Returns:\n",
    "    - links (list): List of hyperlinks.\n",
    "    - names (list): List of drug names.\n",
    "    \"\"\"\n",
    "\n",
    "    # Initialize lists to store links and names\n",
    "    links, names = [], []\n",
    "\n",
    "    # Iterate through each row in the provided table, excluding the header (first row)\n",
    "    for tr in table_provided.select(\"tr\")[1:]:\n",
    "        try: \n",
    "            # Try to find the first hyperlink in the row\n",
    "            trs = tr.find(\"a\")\n",
    "            \n",
    "            # Check if trs is not None before trying to access attributes\n",
    "            if trs is not None:\n",
    "                actual_link, name = trs.get('href', ''), trs.get_text()\n",
    "            else:\n",
    "                actual_link, name = '', ''\n",
    "            \n",
    "        except (AttributeError, IndexError): \n",
    "            # Handle cases where there's an attribute error or indexing error\n",
    "            actual_link, name = '', ''\n",
    "\n",
    "        # Append the extracted link and name to the respective lists\n",
    "        links.append(actual_link)\n",
    "        names.append(name)\n",
    "        \n",
    "    return links, names\n",
    "\n",
    "def scrape_fda_drug_approvals(start_year, end_year):\n",
    "    \"\"\"\n",
    "    Scrapes FDA drug approvals data from specified years.\n",
    "\n",
    "    Parameters:\n",
    "    - start_year (int): The starting year for scraping.\n",
    "    - end_year (int): The ending year for scraping.\n",
    "\n",
    "    Returns:\n",
    "    - df_final (DataFrame): Pandas DataFrame containing drug approval information.\n",
    "    \"\"\"\n",
    "\n",
    "    # Initialize an empty list to store DataFrames\n",
    "    tables = []\n",
    "\n",
    "    # Iterate through each year in the specified range\n",
    "    for year in range(start_year, end_year + 1):\n",
    "        print(f\"Scraping data for year {year}\")\n",
    "\n",
    "        # Construct the URL for the FDA drug approvals page for the current year\n",
    "        url = f'https://www.fda.gov/drugs/new-drugs-fda-cders-new-molecular-entities-and-new-therapeutic-biological-products/novel-drug-approvals-{year}'\n",
    "\n",
    "        # Make a request to the URL and get the HTML content\n",
    "        response = requests.get(url)\n",
    "\n",
    "        # Check if the request was successful\n",
    "        if response.status_code != 200:\n",
    "            print(f\"Failed to retrieve content for year {year}. Status code: {response.status_code}\")\n",
    "            continue  # Skip to the next iteration\n",
    "\n",
    "        # Extract the table from the HTML content\n",
    "        df_list = pd.read_html(response.content)\n",
    "\n",
    "        # Check if any tables were found\n",
    "        if not df_list:\n",
    "            print(f\"No tables found for year {year}.\")\n",
    "            continue  # Skip to the next iteration\n",
    "\n",
    "        # Use the first table found\n",
    "        df = df_list[0]\n",
    "\n",
    "        # Rename columns for consistency\n",
    "        df.rename(columns={'Date': 'Approval Date', 'Drug  Name': 'Drug Name'}, inplace=True)\n",
    "\n",
    "        # Extract links and names from the drug names in the table\n",
    "        soup = BeautifulSoup(response.text, 'html.parser')\n",
    "        table = soup.find('table')\n",
    "\n",
    "        # Check if the table is found\n",
    "        if table is None:\n",
    "            print(f\"No table found for year {year}.\")\n",
    "            continue  # Skip to the next iteration\n",
    "\n",
    "        links, names = extract_links_from_fda_drugname(table)\n",
    "\n",
    "        # Add links and names as new columns in the DataFrame\n",
    "        df['links'], df['check_names'] = links, names\n",
    "\n",
    "        # Append the DataFrame to the list of tables\n",
    "        tables.append(df)\n",
    "        \n",
    "    df_final = pd.concat(tables, ignore_index=True)\n",
    "    return df_final\n",
    "\n",
    "# Specify the range of years for scraping\n",
    "start_year = 2015\n",
    "end_year = 2023\n",
    "\n",
    "# Call the function to scrape FDA drug approvals data\n",
    "df_result = scrape_fda_drug_approvals(start_year, end_year)\n",
    "df_result.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "all_main_label_pdf_links = []\n",
    "\n",
    "for counter, each_url in enumerate(df_result['links']):\n",
    "    # Check if the URL is correctly formatted\n",
    "    if each_url.startswith(('http://', 'https://')):\n",
    "        try:\n",
    "            html = requests.get(each_url).content\n",
    "            soup = BeautifulSoup(html, 'html5lib')\n",
    "\n",
    "            possible_label_pdf_links = []\n",
    "            if soup:\n",
    "                for link in soup.findAll('a'):\n",
    "                    current_link = link.get('href')\n",
    "                    if current_link is not None:\n",
    "                        label_pdf_pattern = ['https://www.accessdata.fda.gov/drugsatfda_docs/label/', '.pdf']\n",
    "                        if all(x in current_link for x in label_pdf_pattern):\n",
    "                            if '#' in current_link:\n",
    "                                hashsymbol_stripped = current_link[:current_link.find('#')]\n",
    "                            else:\n",
    "                                hashsymbol_stripped = current_link\n",
    "                            possible_label_pdf_links.append(hashsymbol_stripped)\n",
    "\n",
    "            possible_label_pdf_links = list(set(possible_label_pdf_links))\n",
    "\n",
    "            try:\n",
    "                all_main_label_pdf_links.append(possible_label_pdf_links[0]) if possible_label_pdf_links else all_main_label_pdf_links.append('')\n",
    "            except IndexError:\n",
    "                all_main_label_pdf_links.append('')\n",
    "\n",
    "        except requests.exceptions.RequestException as e:\n",
    "            print(f\"Error fetching content for {each_url}: {e}\")\n",
    "            all_main_label_pdf_links.append('')\n",
    "    else:\n",
    "        # Skip invalid URLs\n",
    "        all_main_label_pdf_links.append('')\n",
    "\n",
    "# Check if the final lists have the same number of items as the number of rows in the DataFrame\n",
    "if len(all_main_label_pdf_links) != len(df_result):\n",
    "    print(\"The lengths of the lists do not match the number of rows in the DataFrame.\")\n",
    "    \n",
    "df_result['main_label_pdf'] = all_main_label_pdf_links\n",
    "df_result.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result.loc[(df_result['Drug Name'] != df_result['check_names'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result = df_result.drop(columns=['No.'])\n",
    "df_result.to_csv(f'fda_approved_drugs_{start_year}_{end_year}.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}