Scrape KYAG Site.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "ad5db199-25e5-4a35-99b6-38db243dfde7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fetching  1993\n",
      "fetching  1994\n",
      "fetching  1995\n",
      "fetching  1996\n",
      "fetching  1997\n",
      "fetching  1998\n",
      "fetching  1999\n",
      "fetching  2000\n",
      "fetching  2001\n",
      "fetching  2002\n",
      "fetching  2003\n",
      "fetching  2004\n",
      "fetching  2005\n",
      "fetching  2006\n",
      "fetching  2007\n",
      "fetching  2008\n",
      "fetching  2009\n",
      "fetching  2010\n",
      "fetching  2011\n",
      "fetching  2012\n",
      "fetching  2013\n",
      "fetching  2014\n",
      "fetching  2015\n",
      "fetching  2016\n",
      "fetching  2017\n",
      "fetching  2018\n",
      "fetching  2019\n",
      "fetching  2020\n",
      "fetching  2021\n",
      "fetching  2022\n",
      "fetching  2023\n",
      "fetching  2024\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Title</th>\n",
       "      <th>Date</th>\n",
       "      <th>Parties</th>\n",
       "      <th>Number</th>\n",
       "      <th>Download</th>\n",
       "      <th>Year New</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>7512</td>\n",
       "      <td>7480</td>\n",
       "      <td>7480</td>\n",
       "      <td>7480</td>\n",
       "      <td>5789</td>\n",
       "      <td>7480</td>\n",
       "      <td>5789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>32</td>\n",
       "      <td>7447</td>\n",
       "      <td>3777</td>\n",
       "      <td>6181</td>\n",
       "      <td>282</td>\n",
       "      <td>7480</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>2023</td>\n",
       "      <td></td>\n",
       "      <td>5/15/2003</td>\n",
       "      <td>Uriah Pasha/Kentucky State Reformatory</td>\n",
       "      <td></td>\n",
       "      <td>https://www.ag.ky.gov/Resources/orom/1993/93ORD002.htm</td>\n",
       "      <td>2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>373</td>\n",
       "      <td>3</td>\n",
       "      <td>22</td>\n",
       "      <td>34</td>\n",
       "      <td>200</td>\n",
       "      <td>1</td>\n",
       "      <td>280</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Year Title       Date                                 Parties Number  \\\n",
       "count   7512  7480       7480                                    7480   5789   \n",
       "unique    32  7447       3777                                    6181    282   \n",
       "top     2023        5/15/2003  Uriah Pasha/Kentucky State Reformatory          \n",
       "freq     373     3         22                                      34    200   \n",
       "\n",
       "                                                      Download Year New  \n",
       "count                                                     7480     5789  \n",
       "unique                                                    7480       25  \n",
       "top     https://www.ag.ky.gov/Resources/orom/1993/93ORD002.htm     2016  \n",
       "freq                                                         1      280  "
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import datetime\n",
    "import time\n",
    "import os\n",
    "\n",
    "base_url_pre_2023 = 'https://www.ag.ky.gov/Resources/orom/Pages/{}-OROM.aspx'\n",
    "base_url_post_2023 = 'https://www.ag.ky.gov/Resources/orom/{}-OROM/Pages/default.aspx'\n",
    "# Generate a list of years from 1993 to 2024\n",
    "start_year = 1993\n",
    "end_year = datetime.date.today().year\n",
    "\n",
    "# Create a list of years\n",
    "years = [year for year in range(start_year, end_year + 1)]\n",
    "\n",
    "# Initialize an empty DataFrame to store all data\n",
    "master_df = pd.DataFrame()\n",
    "\n",
    "# Headers to mimic a request from my web browser\n",
    "html_headers = {\n",
    "    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',\n",
    "    'Referer': 'https://www.ag.ky.gov/Resources/orom/Pages/default.aspx',\n",
    "    'Sec-Ch-Ua': '\"Google Chrome\";v=\"123\", \"Not:A-Brand\";v=\"8\", \"Chromium\";v=\"123\"'\n",
    "}\n",
    "\n",
    "# Strip html tags from a string.\n",
    "def strip_html_bs(html_str):\n",
    "    soup = BeautifulSoup(html_str, features=\"html.parser\")\n",
    "    return soup.get_text().strip()\n",
    "\n",
    "# Get the href attribute form the first <a> tag found in a string.\n",
    "def get_href(html_str):\n",
    "    # Assuming the value is a string containing HTML\n",
    "    soup = BeautifulSoup(html_str, 'html.parser')\n",
    "    a_tag = soup.find('a')\n",
    "    if a_tag:\n",
    "        return a_tag.get('href')\n",
    "    else:\n",
    "        return None\n",
    "\n",
    "# Take raw html from each cell and get the preprocessed values we want.\n",
    "def process_row_data(row_dict):\n",
    "    new_object = {}\n",
    "    for key, value in row_dict.items():\n",
    "        if key == \"Download\":\n",
    "            new_object[key] = get_href(value)\n",
    "        else:\n",
    "            new_object[key] = strip_html_bs(value)\n",
    "    return new_object\n",
    "\n",
    "for year in years:\n",
    "    # Fetch the page content with headers\n",
    "    print('fetching ', year)\n",
    "    url = base_url_pre_2023.format(year) if year < 2023 else base_url_post_2023.format(year)\n",
    "    time.sleep(2)\n",
    "    response = requests.get(url, headers=html_headers)\n",
    "    \n",
    "    # Parse the HTML content\n",
    "    soup = BeautifulSoup(response.text, 'html.parser')\n",
    "    \n",
    "    # Find the table (assuming it's the first table found)\n",
    "    table = soup.find('table')\n",
    "    \n",
    "    # Extract table headers\n",
    "    headers = [header.text for header in table.find_all('th')]\n",
    "    \n",
    "    # Extract table rows\n",
    "    rows = table.find_all('tr')\n",
    "    \n",
    "    # Extract data from each row\n",
    "    for row in rows:\n",
    "        data = [cell.prettify() for cell in row.find_all('td')]\n",
    "        # Create a dictionary with headers as keys and data as values\n",
    "        row_dict = dict(zip(headers, data))\n",
    "        row = process_row_data(row_dict)\n",
    "        row[\"Year\"] = str(year)\n",
    "        # Convert the dictionary to a DataFrame and append it to the master DataFrame\n",
    "        row_df = pd.DataFrame([row])\n",
    "        master_df = pd.concat([master_df, row_df])\n",
    "\n",
    "# Now master_df contains all the data from the tables across the pages\n",
    "master_df.to_csv(os.path.join('opinions', 'kyag_scraped.csv'), index=False)\n",
    "\n",
    "master_df.describe(include=\"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "1e97fcee-209b-44bd-80d8-655e43f49df4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Title</th>\n",
       "      <th>Date</th>\n",
       "      <th>Parties</th>\n",
       "      <th>Number</th>\n",
       "      <th>Download</th>\n",
       "      <th>Year New</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1993</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1993</td>\n",
       "      <td>93-ORD-002</td>\n",
       "      <td>1/19/1993</td>\n",
       "      <td>Richard W. Richards/City of West Buechel</td>\n",
       "      <td>2</td>\n",
       "      <td>https://www.ag.ky.gov/Resources/orom/1993/93ORD002.htm</td>\n",
       "      <td>1993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1993</td>\n",
       "      <td>93-ORD-004</td>\n",
       "      <td>1/19/1993</td>\n",
       "      <td>Richard W. Richards/City of West Buechel</td>\n",
       "      <td>4</td>\n",
       "      <td>https://www.ag.ky.gov/Resources/orom/1993/93ORD004.htm</td>\n",
       "      <td>1993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1993</td>\n",
       "      <td>93-ORD-005</td>\n",
       "      <td>1/25/1993</td>\n",
       "      <td>Nick Comer/City of Maysville</td>\n",
       "      <td>5</td>\n",
       "      <td>https://www.ag.ky.gov/Resources/orom/1993/93ORD005.htm</td>\n",
       "      <td>1993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1993</td>\n",
       "      <td>93-ORD-006</td>\n",
       "      <td>1/25/1993</td>\n",
       "      <td>Susan Gayle Reed/Eastern Kentucky University</td>\n",
       "      <td>6</td>\n",
       "      <td>https://www.ag.ky.gov/Resources/orom/1993/93ORD006.htm</td>\n",
       "      <td>1993</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Year       Title       Date                                       Parties  \\\n",
       "0  1993         NaN        NaN                                           NaN   \n",
       "0  1993  93-ORD-002  1/19/1993      Richard W. Richards/City of West Buechel   \n",
       "0  1993  93-ORD-004  1/19/1993      Richard W. Richards/City of West Buechel   \n",
       "0  1993  93-ORD-005  1/25/1993                  Nick Comer/City of Maysville   \n",
       "0  1993  93-ORD-006  1/25/1993  Susan Gayle Reed/Eastern Kentucky University   \n",
       "\n",
       "  Number                                                Download Year New  \n",
       "0    NaN                                                     NaN      NaN  \n",
       "0      2  https://www.ag.ky.gov/Resources/orom/1993/93ORD002.htm     1993  \n",
       "0      4  https://www.ag.ky.gov/Resources/orom/1993/93ORD004.htm     1993  \n",
       "0      5  https://www.ag.ky.gov/Resources/orom/1993/93ORD005.htm     1993  \n",
       "0      6  https://www.ag.ky.gov/Resources/orom/1993/93ORD006.htm     1993  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('max_colwidth', None) # Show full width of columns\n",
    "master_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "96f58fd8-a17f-4fc7-a0c8-dd110e688ce9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Title</th>\n",
       "      <th>Date</th>\n",
       "      <th>Parties</th>\n",
       "      <th>Number</th>\n",
       "      <th>Download</th>\n",
       "      <th>Year New</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>7512.000000</td>\n",
       "      <td>7480</td>\n",
       "      <td>7480</td>\n",
       "      <td>7480</td>\n",
       "      <td>5789</td>\n",
       "      <td>7480</td>\n",
       "      <td>5789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>NaN</td>\n",
       "      <td>7447</td>\n",
       "      <td>3777</td>\n",
       "      <td>6181</td>\n",
       "      <td>282</td>\n",
       "      <td>7480</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>NaN</td>\n",
       "      <td></td>\n",
       "      <td>5/15/2003</td>\n",
       "      <td>Uriah Pasha/Kentucky State Reformatory</td>\n",
       "      <td></td>\n",
       "      <td>https://www.ag.ky.gov/Resources/orom/1993/93ORD002.htm</td>\n",
       "      <td>2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>22</td>\n",
       "      <td>34</td>\n",
       "      <td>200</td>\n",
       "      <td>1</td>\n",
       "      <td>280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2009.178381</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>8.834029</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1993.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2002.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2009.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2017.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2024.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Year Title       Date                                 Parties  \\\n",
       "count   7512.000000  7480       7480                                    7480   \n",
       "unique          NaN  7447       3777                                    6181   \n",
       "top             NaN        5/15/2003  Uriah Pasha/Kentucky State Reformatory   \n",
       "freq            NaN     3         22                                      34   \n",
       "mean    2009.178381   NaN        NaN                                     NaN   \n",
       "std        8.834029   NaN        NaN                                     NaN   \n",
       "min     1993.000000   NaN        NaN                                     NaN   \n",
       "25%     2002.000000   NaN        NaN                                     NaN   \n",
       "50%     2009.000000   NaN        NaN                                     NaN   \n",
       "75%     2017.000000   NaN        NaN                                     NaN   \n",
       "max     2024.000000   NaN        NaN                                     NaN   \n",
       "\n",
       "       Number                                                Download Year New  \n",
       "count    5789                                                    7480     5789  \n",
       "unique    282                                                    7480       25  \n",
       "top            https://www.ag.ky.gov/Resources/orom/1993/93ORD002.htm     2016  \n",
       "freq      200                                                       1      280  \n",
       "mean      NaN                                                     NaN      NaN  \n",
       "std       NaN                                                     NaN      NaN  \n",
       "min       NaN                                                     NaN      NaN  \n",
       "25%       NaN                                                     NaN      NaN  \n",
       "50%       NaN                                                     NaN      NaN  \n",
       "75%       NaN                                                     NaN      NaN  \n",
       "max       NaN                                                     NaN      NaN  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_df.describe(include=\"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "de45de9b-0e7b-437e-b022-62244adb38a8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    Year       Title        Date  \\\n",
      "0   1993  93-ORD-105   9/14/1993   \n",
      "0   1994  94-ORD-003   1/11/1994   \n",
      "0   1994  94-ORD-010   1/28/1994   \n",
      "0   1994  94-ORD-075    6/7/1994   \n",
      "0   1994  94-ORD-120  10/25/1994   \n",
      "..   ...         ...         ...   \n",
      "0   2024  24-ORD-075   3/20/2024   \n",
      "0   2024  24-ORD-076   3/20/2024   \n",
      "0   2024  24-ORD-077   3/20/2024   \n",
      "0   2024  24-ORD-078   3/20/2024   \n",
      "0   2024  24-ORD-079   3/20/2024   \n",
      "\n",
      "                                                                                        Parties  \\\n",
      "0                                         Carol White/Office of the Lyon County Judge/Executive   \n",
      "0                          Bill Estep/Office of Whitley County Judge/Executive, Jerry F. Taylor   \n",
      "0                   Virginia Luttrell/Office of Pulaski County Judge/Executive Darrell Beshears   \n",
      "0                                  Larry Dale Thacker/Office of the Pike County Judge/Executive   \n",
      "0   Courier-Journal and Louisville Times Company/Office of the Jefferson County Judge/Executive   \n",
      "..                                                                                          ...   \n",
      "0                                                     Randy Skaggs/Scott County Judge/Executive   \n",
      "0                                                   Randy Skaggs/Spencer County Judge/Executive   \n",
      "0                                                     Randy Skaggs/Trigg County Judge/Executive   \n",
      "0                                                   Randy Skaggs/Trimble County Judge/Executive   \n",
      "0                                                     Randy Skaggs/Wolfe County Judge/Executive   \n",
      "\n",
      "   Number                                                            Download  \\\n",
      "0     105              https://www.ag.ky.gov/Resources/orom/1993/93ORD105.htm   \n",
      "0       3              https://www.ag.ky.gov/Resources/orom/1994/94ORD003.htm   \n",
      "0      10              https://www.ag.ky.gov/Resources/orom/1994/94ORD010.htm   \n",
      "0      75              https://www.ag.ky.gov/Resources/orom/1994/94ORD075.htm   \n",
      "0     120              https://www.ag.ky.gov/Resources/orom/1994/94ORD120.htm   \n",
      "..    ...                                                                 ...   \n",
      "0     NaN  https://www.ag.ky.gov/Resources/orom/2024-OROM/2024/24-ORD-075.pdf   \n",
      "0     NaN  https://www.ag.ky.gov/Resources/orom/2024-OROM/2024/24-ORD-076.pdf   \n",
      "0     NaN  https://www.ag.ky.gov/Resources/orom/2024-OROM/2024/24-ORD-077.pdf   \n",
      "0     NaN  https://www.ag.ky.gov/Resources/orom/2024-OROM/2024/24-ORD-078.pdf   \n",
      "0     NaN  https://www.ag.ky.gov/Resources/orom/2024-OROM/2024/24-ORD-079.pdf   \n",
      "\n",
      "   Year New  \n",
      "0      1993  \n",
      "0      1994  \n",
      "0      1994  \n",
      "0      1994  \n",
      "0      1994  \n",
      "..      ...  \n",
      "0       NaN  \n",
      "0       NaN  \n",
      "0       NaN  \n",
      "0       NaN  \n",
      "0       NaN  \n",
      "\n",
      "[137 rows x 7 columns]\n",
      "Number of rows with two or more '/' characters in the 'Parties' column: 137\n"
     ]
    }
   ],
   "source": [
    "# Assuming master_df is your DataFrame and 'Parties' is the column of interest\n",
    "rows_with_two_slashes = master_df[master_df['Parties'].str.count('/') >= 2]\n",
    "\n",
    "# To get the count of such rows\n",
    "count_of_rows_with_two_slashes = rows_with_two_slashes.shape[0]\n",
    "\n",
    "print(rows_with_two_slashes)\n",
    "print(f\"Number of rows with two or more '/' characters in the 'Parties' column: {count_of_rows_with_two_slashes}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "8ba0132b-1d11-46cb-85a1-c6c30fc90f62",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Carol White/Office of the Lyon County Judge/Executive\n",
      "Bill Estep/Office of Whitley County Judge/Executive, Jerry F. Taylor\n",
      "Virginia Luttrell/Office of Pulaski County Judge/Executive Darrell Beshears\n",
      "Larry Dale Thacker/Office of the Pike County Judge/Executive\n",
      "Courier-Journal and Louisville Times Company/Office of the Jefferson County Judge/Executive\n",
      "Dave Baker/Franklin County Judge/Executive\n",
      "R. Keith Cullinan/Louisville/Jefferson County Revenue Commission\n",
      "Cheryl A. Martin/Office of Butler County Judge/Executive\n",
      "Donald J. Ruberg/Kenton/Boone Counties Cable Television Board\n",
      "Kimberly A. Hobbs/McCracken County Judge/Executive\n",
      "Randy Skaggs/Office of the Leslie County Judge/Executive\n",
      "Randy Skaggs/Office of the Marshall County Judge/Executive\n",
      "Randy Skaggs/Office of the Bath County Judge/Executive\n",
      "Randy Skaggs/Office of the Floyd County Judge/Executive\n",
      "Randy Skaggs/Office of the Jackson County Judge/Executive\n",
      "Randy Skaggs/Office of the Knott County Judge/Executive\n",
      "Randy Skaggs/Office of the Knox County Judge/Executive\n",
      "Randy Skaggs/Office of the Ohio County Judge/Executive\n",
      "Randy Skaggs/Office of the Owsley County Judge/Executive\n",
      "Randy Skaggs/Office of the Pendleton County Judge/Executive\n",
      "Randy Skaggs/Office of the Perry County Judge/Executive\n",
      "Randy Skaggs/Office of the Whitley County Judge/Executive\n",
      "Randy Skaggs/Office of the Webster County Judge/Executive\n",
      "Randy Skaggs/Office of the Gallatin County Judge/Executive\n",
      "Randy Skaggs/Office of the Allen County Judge/Executive\n",
      "Randy Skaggs/Office of the Fulton County Judge/Executive\n",
      "Randy Skaggs/Office of the Trigg County Judge/Executive\n",
      "Randy Skaggs/Office of the Todd County Judge/Executive\n",
      "David M. Cross/Office of the Clinton County Judge/Executive\n",
      "Don Mahan/Office of the Whitley County Judge/Executive\n",
      "Paul B. Osborne/Office of the Russell County Judge/Executive\n",
      "Elizabeth Ullmer Mendel/Kentucky Association of Counties/Workers \tCompensation Fund\n",
      "The Owensboro Messenger-Inquirer/Daviess County Judge/Executive’s  Planning and Zoning Citizens Task Force\n",
      "Glen D. Mills/ Office of the Knox County Judge/Executive and East Knox Water District\n",
      "Pamela Rogers/Hardin County Judge/Executive, Hardin County Fiscal  Court, and Hardin County Dog Warden\n",
      "Nancy Perry/Office of the Jefferson County Judge/Executive\n",
      "Jo Coffman/Shepherdsville/Bullitt County Tourist & Convention Commission\n",
      "Christina Heavrin/Office of Jefferson County Commissioner/Jefferson County Fiscal Court\n",
      "Willis Barmore/Office of the Henry County Judge/Executive\n",
      "William F. Simpson/Senior Executive Coach, Inc./DBA Metro Taxicab\n",
      "Mayor Gene Flaugher/Pendleton County Judge/Executive\n",
      "Joseph L. Silverburg/Eastern Kentucky Correctional Complex/Department of Corrections\n",
      "Elbert A. Powell/Hopkins County Judge/Executive  and Assistance Center Fund, Inc.\n",
      "Kathy Fekete/Offices of Bullitt County Fiscal Court Clerk; County Judge/Executive; and Animal Control Officer\n",
      "The Courier-Journal/Bullitt County Judge/Executive\n",
      "Stephen Slaughter/Christian County Circuit/District Court Clerk\n",
      "Frank A. Brancato / Crittenden/Livingston County Water District\n",
      "Gerald T. Kemper / Office of the Owen County Judge/Executive\n",
      "Vicki Henderson/Ohio County Judge/Executive and Ohio County Water  \tDistrict\n",
      "Mike Nance/City of Beaver Dam/Beaver Dam Volunteer Fire Department\n",
      "W. W. Chilton/Louisville/Jefferson County Metro Department of Public \tWorks\n",
      "The Courier-Journal/Office of the Bullitt County Judge/Executive\n",
      "Winston Briscoe/Lexington/Fayette Urban County Government-Division of Community Corrections\n",
      "Ronald W. Jones/Oldham County Judge/Executive\n",
      "Mike Nance/City of McHenry/McHenry Volunteer Fire Department\n",
      "Mike Swartz/Office of Bath County Judge/Executive\n",
      "David Wilson/Garrard County Judge/Executive\n",
      "Samuel Edwards/University Medical Center, Inc., \td/b/a University of Louisville Hospital\n",
      "Amy Payne Korb/Bluegrass Regional Mental Health/Mental Retardation Board, Inc.\n",
      "Appalachian News-Express/Pike County Judge/Executive\n",
      "Clarence Hurst/Office of the Warren County Judge/Executive\n",
      "Dewey R. Wotring/Oldham County Judge/Executive\n",
      "Curtis and Nina Cornett/Letcher County Judge/Executive\n",
      "Jennifer A. Peterson/Murray-Calloway County Public Hospital Corporation d/b/a Murray-Calloway County Hospital\n",
      "Richard L. Shivel/Russell County Judge/Executive\n",
      "Mike Stephens/Office of the McCreary County Judge/Executive and McCreary County 911 Emergency Dispatch Center\n",
      "Mikel F. Fitch/Knott County Judge/Executive\n",
      "James F. Francis/Floyd County Judge/Executive\n",
      "Kevin Brumley/Nelson County Judge/Executive\n",
      "Shane Johnson/Laurel County Judge/Executive\n",
      "Corbin News Journal/Whitley County Police Department/911 Communications\n",
      "Samuel M. Gaddie/Jefferson County Attorney/Child Support Division and Cabinet for Health and Family Services Department for Income Support/Child Support Enforcement\n",
      "Ronald L. Cook/Bullitt County Judge/Executive\n",
      "Kevin Brumley/Office of the Nelson County Judge/Executive\n",
      "Samuel M. Gaddie/Cabinet for Health and Family Services – Department for Income Support/Child Support Enforcement\n",
      "D. Duane Cook/Leslie County Judge/Executive, et al.\n",
      "The Courier-Journal/University of Louisville Medical School Practice Association, Inc., d/b/a/University Physicians Associates\n",
      "The LaRue County Herald News/LaRue County Judge/Executive and LaRue County Jailer\n",
      "Lewis County Judge/Executive Thomas Massie/Buffalo Trace Area Development District\n",
      "Kevin Brumley/City of Bardstown Safety Committee/Selection Committee\n",
      "Jureen Hendrickson/Bell County Judge/Executive\n",
      "Carroll Paul Jackson/Letcher District/Circuit Court Clerk\n",
      "Terrance Donoghue/Boone County Judge/Executive\n",
      "Tom Stone/Woodside Preservation, LP (Middletown Apartments/ Woodside Village)\n",
      "Lawrence Trageser/Spencer County 911 Commission, Spencer County Judge/Executive, and Mayor of the City of Taylorsville\n",
      "Doug Goodman/Hardin County Judge/Executive\n",
      "Dennis Gibson/Knox County Judge/Executive\n",
      "McCreary County Record/McCreary County Judge/Executive\n",
      "David Cross/Bullitt County Judge/Executive\n",
      "David Cross/Bullitt County Judge/Executive\n",
      "Bryan Reinhardt/Todd County Standard, Inc., D/B/A Quality Web Printing\n",
      "William Epling/Saint Joseph Health System, Inc. d/b/a Saint Joseph Berea\n",
      "Darrell W. Peters/Hyden-Leslie County Water District and Office of the Leslie County Judge/Executive\n",
      "Scotty Hedgespeth/Taylor County Judge/Executive\n",
      "Joshua A. Puckett/Magoffin County Judge/Executive\n",
      "Danny McIntosh/Wolfe County Sheriff’s Department and County \t Judge/Executive\n",
      "Oscar McGuffey/Office of the LaRue County Judge/Executive\n",
      "Brennan Crain/Metcalfe County Judge/Executive\n",
      "Glenn S. Hayden/Office of the Graves County Judge/Executive\n",
      "Glenn Hayden/Graves County Judge/Executive\n",
      "Redacted/Nelson County Judge/Executive\n",
      "Andrew Mize/Carter County School District; and Andrew Mize/Grayson Police Department\n",
      "WPSD Local 6 News/Office of Marshall County Judge/Executive\n",
      "Robbie Smither/Office of the Bullitt County Judge/Executive\n",
      "WSPD Local 6 News/Marshall County Judge/Executive\n",
      "Charity Bird/Office of the Meade County Judge/Executive\n",
      "Mary Jasper/Office of the Pulaski County Judge/Executive\n",
      "Lawrence Trageser / Spencer County Judge/ Executive\n",
      "Jonathan Raikes/Nelson County Judge/Executive\n",
      "Eddie McGuire/Marshall County Judge/Executive\n",
      "In re: Troy Seelye/Office of the Meade County Judge/Executive\n",
      "Kurt Wallace/Jefferson County Judge/Executive\n",
      "The Courier-Journal/Louisville/Jefferson County Metro Government\n",
      "Kurt Wallace/Oldham County Judge/Executive\n",
      "Lawrence Trageser/Spencer County Judge/Executive’s Office\n",
      "Lynn Hesselbrock/Spencer County Judge/Executive’s Office\n",
      "J. Brooken Smith/Louisville/Jefferson County Metro Government & Louisville/Jefferson County Metro Government Ethics Commission\n",
      "J. Brooken Smith/Louisville/Jefferson County Metro Government & Louisville/Jefferson County Metro Government Ethics Commission\n",
      "Anita Ellison/Nelson County Judge/Executive’s Office\n",
      "Telemachus Harrison/Estill County Judge/Executive’s Office\n",
      "Elizabeth Harrison/Estill County Judge/Executive’s Office\n",
      "J. Brooken Smith/Louisville/Jefferson County Metro Government\n",
      "Randy Skaggs/Anderson County Judge/Executive\n",
      "Randy Skaggs/Breathitt County Judge/Executive\n",
      "Randy Skaggs/Breckinridge County Judge/Executive\n",
      "Randy Skaggs/Christian County Judge/Executive\n",
      "Randy Skaggs/Franklin County Judge/Executive\n",
      "Randy Skaggs/Gallatin County Judge/Executive\n",
      "Randy Skaggs/Hopkins County Judge/Executive\n",
      "Randy Skaggs/Lyon County Judge/Executive\n",
      "Randy Skaggs/Knott County Judge/Executive\n",
      "Randy Skaggs/Nicholas County Judge/Executive\n",
      "Randy Skaggs/Scott County Judge/Executive\n",
      "Randy Skaggs/Spencer County Judge/Executive\n",
      "Randy Skaggs/Trigg County Judge/Executive\n",
      "Randy Skaggs/Trimble County Judge/Executive\n",
      "Randy Skaggs/Wolfe County Judge/Executive\n"
     ]
    }
   ],
   "source": [
    "for row in rows_with_two_slashes['Parties'].to_list():\n",
    "    print(row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "ce18ab1e-f283-46fb-97a3-3a83c0685816",
   "metadata": {},
   "outputs": [],
   "source": [
    "# It looks like when there are multiple, we can safely assume the appellant is before the first / and the agency is after the first /\n",
    "master_df = master_df.dropna(subset=['Parties']) # Drops rows with NaN in 'Parties'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "2f0bd67a-e9c9-40b9-a7b8-aa66a6e4bed8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the 'Parties' column at the first occurrence of '/'\n",
    "master_df[['Appellant', 'Agency']] = master_df['Parties'].str.split('/', n=1, expand=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "aa0fd327-e2cf-44e2-9308-5be45505802e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Title</th>\n",
       "      <th>Date</th>\n",
       "      <th>Parties</th>\n",
       "      <th>Number</th>\n",
       "      <th>Download</th>\n",
       "      <th>Year New</th>\n",
       "      <th>Appellant</th>\n",
       "      <th>Agency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>7480.000000</td>\n",
       "      <td>7480</td>\n",
       "      <td>7480</td>\n",
       "      <td>7480</td>\n",
       "      <td>5789</td>\n",
       "      <td>7480</td>\n",
       "      <td>5789</td>\n",
       "      <td>7480</td>\n",
       "      <td>7470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>NaN</td>\n",
       "      <td>7447</td>\n",
       "      <td>3777</td>\n",
       "      <td>6181</td>\n",
       "      <td>282</td>\n",
       "      <td>7480</td>\n",
       "      <td>25</td>\n",
       "      <td>4133</td>\n",
       "      <td>2808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>NaN</td>\n",
       "      <td></td>\n",
       "      <td>5/15/2003</td>\n",
       "      <td>Uriah Pasha/Kentucky State Reformatory</td>\n",
       "      <td></td>\n",
       "      <td>https://www.ag.ky.gov/Resources/orom/1993/93ORD002.htm</td>\n",
       "      <td>2016</td>\n",
       "      <td>Randy Skaggs</td>\n",
       "      <td>Kentucky State Police</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>22</td>\n",
       "      <td>34</td>\n",
       "      <td>200</td>\n",
       "      <td>1</td>\n",
       "      <td>280</td>\n",
       "      <td>132</td>\n",
       "      <td>320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2009.181283</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>8.832171</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1993.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2002.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2009.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2017.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2024.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Year Title       Date                                 Parties  \\\n",
       "count   7480.000000  7480       7480                                    7480   \n",
       "unique          NaN  7447       3777                                    6181   \n",
       "top             NaN        5/15/2003  Uriah Pasha/Kentucky State Reformatory   \n",
       "freq            NaN     3         22                                      34   \n",
       "mean    2009.181283   NaN        NaN                                     NaN   \n",
       "std        8.832171   NaN        NaN                                     NaN   \n",
       "min     1993.000000   NaN        NaN                                     NaN   \n",
       "25%     2002.000000   NaN        NaN                                     NaN   \n",
       "50%     2009.000000   NaN        NaN                                     NaN   \n",
       "75%     2017.000000   NaN        NaN                                     NaN   \n",
       "max     2024.000000   NaN        NaN                                     NaN   \n",
       "\n",
       "       Number                                                Download  \\\n",
       "count    5789                                                    7480   \n",
       "unique    282                                                    7480   \n",
       "top            https://www.ag.ky.gov/Resources/orom/1993/93ORD002.htm   \n",
       "freq      200                                                       1   \n",
       "mean      NaN                                                     NaN   \n",
       "std       NaN                                                     NaN   \n",
       "min       NaN                                                     NaN   \n",
       "25%       NaN                                                     NaN   \n",
       "50%       NaN                                                     NaN   \n",
       "75%       NaN                                                     NaN   \n",
       "max       NaN                                                     NaN   \n",
       "\n",
       "       Year New     Appellant                 Agency  \n",
       "count      5789          7480                   7470  \n",
       "unique       25          4133                   2808  \n",
       "top        2016  Randy Skaggs  Kentucky State Police  \n",
       "freq        280           132                    320  \n",
       "mean        NaN           NaN                    NaN  \n",
       "std         NaN           NaN                    NaN  \n",
       "min         NaN           NaN                    NaN  \n",
       "25%         NaN           NaN                    NaN  \n",
       "50%         NaN           NaN                    NaN  \n",
       "75%         NaN           NaN                    NaN  \n",
       "max         NaN           NaN                    NaN  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_df.describe(include=\"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "96f0329e-7e7f-4997-8f37-e05ec8362bdd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Title</th>\n",
       "      <th>Date</th>\n",
       "      <th>Parties</th>\n",
       "      <th>Number</th>\n",
       "      <th>Download</th>\n",
       "      <th>Year New</th>\n",
       "      <th>Appellant</th>\n",
       "      <th>Agency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>7480</td>\n",
       "      <td>7480</td>\n",
       "      <td>7480</td>\n",
       "      <td>7480</td>\n",
       "      <td>5789</td>\n",
       "      <td>7480</td>\n",
       "      <td>5789</td>\n",
       "      <td>7480</td>\n",
       "      <td>7470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>32</td>\n",
       "      <td>7447</td>\n",
       "      <td>3777</td>\n",
       "      <td>6181</td>\n",
       "      <td>282</td>\n",
       "      <td>7480</td>\n",
       "      <td>25</td>\n",
       "      <td>4052</td>\n",
       "      <td>2653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>2023</td>\n",
       "      <td></td>\n",
       "      <td>5/15/2003</td>\n",
       "      <td>Uriah Pasha/Kentucky State Reformatory</td>\n",
       "      <td></td>\n",
       "      <td>https://www.ag.ky.gov/Resources/orom/1993/93ORD002.htm</td>\n",
       "      <td>2016</td>\n",
       "      <td>Randy Skaggs</td>\n",
       "      <td>Kentucky State Police</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>372</td>\n",
       "      <td>3</td>\n",
       "      <td>22</td>\n",
       "      <td>34</td>\n",
       "      <td>200</td>\n",
       "      <td>1</td>\n",
       "      <td>280</td>\n",
       "      <td>156</td>\n",
       "      <td>331</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Year Title       Date                                 Parties Number  \\\n",
       "count   7480  7480       7480                                    7480   5789   \n",
       "unique    32  7447       3777                                    6181    282   \n",
       "top     2023        5/15/2003  Uriah Pasha/Kentucky State Reformatory          \n",
       "freq     372     3         22                                      34    200   \n",
       "\n",
       "                                                      Download Year New  \\\n",
       "count                                                     7480     5789   \n",
       "unique                                                    7480       25   \n",
       "top     https://www.ag.ky.gov/Resources/orom/1993/93ORD002.htm     2016   \n",
       "freq                                                         1      280   \n",
       "\n",
       "           Appellant                 Agency  \n",
       "count           7480                   7470  \n",
       "unique          4052                   2653  \n",
       "top     Randy Skaggs  Kentucky State Police  \n",
       "freq             156                    331  "
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master_df['Appellant'] = master_df['Appellant'].str.strip()\n",
    "master_df['Agency'] = master_df['Agency'].str.strip()\n",
    "master_df.describe(include=\"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "d913fe91-ea63-43b0-a450-ab8f27bdd39a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Title</th>\n",
       "      <th>Date</th>\n",
       "      <th>Parties</th>\n",
       "      <th>Number</th>\n",
       "      <th>Download</th>\n",
       "      <th>Year New</th>\n",
       "      <th>Appellant</th>\n",
       "      <th>Agency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>7480</td>\n",
       "      <td>7480</td>\n",
       "      <td>7480</td>\n",
       "      <td>7480</td>\n",
       "      <td>5789</td>\n",
       "      <td>7480</td>\n",
       "      <td>5789</td>\n",
       "      <td>7480</td>\n",
       "      <td>7470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>32</td>\n",
       "      <td>7447</td>\n",
       "      <td>3777</td>\n",
       "      <td>6181</td>\n",
       "      <td>282</td>\n",
       "      <td>7480</td>\n",
       "      <td>25</td>\n",
       "      <td>4047</td>\n",
       "      <td>2653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>2023</td>\n",
       "      <td></td>\n",
       "      <td>5/15/2003</td>\n",
       "      <td>Uriah Pasha/Kentucky State Reformatory</td>\n",
       "      <td></td>\n",
       "      <td>https://www.ag.ky.gov/Resources/orom/1993/93ORD002.htm</td>\n",
       "      <td>2016</td>\n",
       "      <td>Randy Skaggs</td>\n",
       "      <td>Kentucky State Police</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>372</td>\n",
       "      <td>3</td>\n",
       "      <td>22</td>\n",
       "      <td>34</td>\n",
       "      <td>200</td>\n",
       "      <td>1</td>\n",
       "      <td>280</td>\n",
       "      <td>156</td>\n",
       "      <td>331</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Year Title       Date                                 Parties Number  \\\n",
       "count   7480  7480       7480                                    7480   5789   \n",
       "unique    32  7447       3777                                    6181    282   \n",
       "top     2023        5/15/2003  Uriah Pasha/Kentucky State Reformatory          \n",
       "freq     372     3         22                                      34    200   \n",
       "\n",
       "                                                      Download Year New  \\\n",
       "count                                                     7480     5789   \n",
       "unique                                                    7480       25   \n",
       "top     https://www.ag.ky.gov/Resources/orom/1993/93ORD002.htm     2016   \n",
       "freq                                                         1      280   \n",
       "\n",
       "           Appellant                 Agency  \n",
       "count           7480                   7470  \n",
       "unique          4047                   2653  \n",
       "top     Randy Skaggs  Kentucky State Police  \n",
       "freq             156                    331  "
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove 'In re: ' from the beginning of the 'Appellant' column\n",
    "master_df['Appellant'] = master_df['Appellant'].str.replace('^In re: ', '', regex=True)\n",
    "master_df.describe(include=\"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "e7cb3de8-59c4-47f2-a24a-89af6b5db2a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "master_df.to_csv(os.path.join('opinions', 'kyag_scraped.csv'), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "7db0fda3-1f52-4c58-b749-baa9c08f866a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most frequent appellant: Randy Skaggs\n",
      "Top 10 top appellants: ['Randy Skaggs', 'Uriah Pasha', 'Lawrence Trageser', 'The Courier-Journal', 'Chris Hawkins', 'Uriah M. Pasha', 'James Harrison', 'Lexington Herald-Leader', 'Leonel Martinez', 'The State Journal']\n"
     ]
    }
   ],
   "source": [
    "most_frequent_values = master_df['Appellant'].value_counts()\n",
    "\n",
    "# To get the most frequent value(s)\n",
    "most_frequent_value = most_frequent_values.idxmax()\n",
    "\n",
    "# If you want to get the top N most frequent values, you can do:\n",
    "N = 10 # Change this to the number of top values you want\n",
    "top_N_values = most_frequent_values[:N].index.tolist()\n",
    "\n",
    "print(\"Most frequent appellant:\", most_frequent_value)\n",
    "print(\"Top\", N, \"top appellants:\", top_N_values)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "44b2fad7-9768-443e-8591-a6ab806d64e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "fetching page_id: 0 \n",
      "current total: 0\n",
      "\n",
      "fetching page_id: 623 \n",
      "current total: 663\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Url</th>\n",
       "      <th>FileSize</th>\n",
       "      <th>Title</th>\n",
       "      <th>Date</th>\n",
       "      <th>Year</th>\n",
       "      <th>Subject</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>975.000000</td>\n",
       "      <td>975</td>\n",
       "      <td>975</td>\n",
       "      <td>975</td>\n",
       "      <td>975</td>\n",
       "      <td>975.000000</td>\n",
       "      <td>975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>NaN</td>\n",
       "      <td>663</td>\n",
       "      <td>452</td>\n",
       "      <td>663</td>\n",
       "      <td>507</td>\n",
       "      <td>NaN</td>\n",
       "      <td>661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.ag.ky.gov/Resources/Opinions/Opinions/Opinion%20of%20the%20Attorney%20General%20%2024-1.pdf</td>\n",
       "      <td>35.00 KB</td>\n",
       "      <td>OAG 24-1</td>\n",
       "      <td>2/25/1992</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Relates to the accuracy of computations to be used in adjusting salaries of constitutional officers in relation to changes in the Consumer Price Index.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>690.742564</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2002.726154</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>352.215223</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11.343339</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1992.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>408.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1993.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>761.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1996.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1006.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1169.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2024.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Id  \\\n",
       "count    975.000000   \n",
       "unique          NaN   \n",
       "top             NaN   \n",
       "freq            NaN   \n",
       "mean     690.742564   \n",
       "std      352.215223   \n",
       "min        1.000000   \n",
       "25%      408.000000   \n",
       "50%      761.000000   \n",
       "75%     1006.000000   \n",
       "max     1169.000000   \n",
       "\n",
       "                                                                                                            Url  \\\n",
       "count                                                                                                       975   \n",
       "unique                                                                                                      663   \n",
       "top     https://www.ag.ky.gov/Resources/Opinions/Opinions/Opinion%20of%20the%20Attorney%20General%20%2024-1.pdf   \n",
       "freq                                                                                                          2   \n",
       "mean                                                                                                        NaN   \n",
       "std                                                                                                         NaN   \n",
       "min                                                                                                         NaN   \n",
       "25%                                                                                                         NaN   \n",
       "50%                                                                                                         NaN   \n",
       "75%                                                                                                         NaN   \n",
       "max                                                                                                         NaN   \n",
       "\n",
       "        FileSize     Title       Date         Year  \\\n",
       "count        975       975        975   975.000000   \n",
       "unique       452       663        507          NaN   \n",
       "top     35.00 KB  OAG 24-1  2/25/1992          NaN   \n",
       "freq          14         2         10          NaN   \n",
       "mean         NaN       NaN        NaN  2002.726154   \n",
       "std          NaN       NaN        NaN    11.343339   \n",
       "min          NaN       NaN        NaN  1992.000000   \n",
       "25%          NaN       NaN        NaN  1993.000000   \n",
       "50%          NaN       NaN        NaN  1996.000000   \n",
       "75%          NaN       NaN        NaN  2016.000000   \n",
       "max          NaN       NaN        NaN  2024.000000   \n",
       "\n",
       "                                                                                                                                                        Subject  \n",
       "count                                                                                                                                                       975  \n",
       "unique                                                                                                                                                      661  \n",
       "top     Relates to the accuracy of computations to be used in adjusting salaries of constitutional officers in relation to changes in the Consumer Price Index.  \n",
       "freq                                                                                                                                                          4  \n",
       "mean                                                                                                                                                        NaN  \n",
       "std                                                                                                                                                         NaN  \n",
       "min                                                                                                                                                         NaN  \n",
       "25%                                                                                                                                                         NaN  \n",
       "50%                                                                                                                                                         NaN  \n",
       "75%                                                                                                                                                         NaN  \n",
       "max                                                                                                                                                         NaN  "
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# AG site stores the OAGs separately with a different API and result structure\n",
    "oag_base_search_url = \"https://www.ag.ky.gov/_layouts/15/Fwk.Webparts.Agency.Ui/Search/Search.ashx?SiteUrl=https%3A%2F%2Fwww.ag.ky.gov%2FResources%2FOpinions&ListsToCount=%7C9328041e-55d9-4eee-a2f2-818633a986a6&ListId=9328041e-55d9-4eee-a2f2-818633a986a6&Lookup=431acfbd-683c-4cfb-8eed-ec00b33dbdf7%3A36%2C35%2C31%2C30%2C29%2C28%2C27%2C25%2C1%2C2%2C3%2C26%2C4%2C5%2C6%2C7%2C8%2C9%2C10%2C11%2C12%2C13%2C14%2C15%2C16%2C17%2C18%2C19%2C20%2C21%2C22%2C33%2C34%2C%7C&SearchText=&PageId={}&PageSize={}&ShowNoResultMessage=True&OperatorType=AND&SortField=&SortDirection=Ascending&SortValue=&Sort=Year+New%3ADescending%7CTitle%3ADescending%7C&ItemTemplateFields=URL%3BDate%3BTitle%3BSubject%3B&_=1711898189865\"\n",
    "next_page_id = 0\n",
    "page_size = 1000\n",
    "keep_going = True\n",
    "\n",
    "html_headers = {\n",
    "    'User-Agent': 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',\n",
    "    'Referer': 'https://www.ag.ky.gov/Resources/Opinions/Pages/Opinions.aspx',\n",
    "}\n",
    "\n",
    "oag_df = pd.DataFrame(columns=['Id', 'Url', 'FileSize', 'Title', 'Date', 'Year', 'Subject'])\n",
    "\n",
    "# Define the dtypes dictionary\n",
    "dtype_dict = {\n",
    "    'Id': 'int', \n",
    "    'Url': 'object', \n",
    "    'FileSize': 'object',\n",
    "    'Title': 'object',\n",
    "    'Date': 'object',\n",
    "    'Year': 'int',\n",
    "    'Subject': 'object', \n",
    "}\n",
    "\n",
    "# Convert each column to the specified dtype\n",
    "for column, dtype in dtype_dict.items():\n",
    "    oag_df[column] = oag_df[column].astype(dtype)\n",
    "\n",
    "\n",
    "while keep_going:\n",
    "    # Fetch the page content with headers\n",
    "    print('\\nfetching page_id:', next_page_id, '\\ncurrent total:', oag_df.shape[0])\n",
    "    url = oag_base_search_url.format(next_page_id, page_size)\n",
    "    time.sleep(2)\n",
    "    response = requests.get(url, headers=html_headers)\n",
    "    \n",
    "    if data['HasNext']:\n",
    "        next_page_id = data['PageNextId']\n",
    "    else:\n",
    "        keep_going = False\n",
    "    \n",
    "    data = response.json()\n",
    "    \n",
    "    # Extract data from each row\n",
    "    for item in data['Items']:\n",
    "        \"\"\"\n",
    "        Ex: item\n",
    "        {\n",
    "            \"Id\":721,\n",
    "            \"Url\":\"https://www.ag.ky.gov/Resources/Opinions/Opinions/OAG94-064.htm\",\n",
    "            \"Fields\":[\n",
    "                {\"Name\":\"Url\",\"Value\":\"https://www.ag.ky.gov/Resources/Opinions/Opinions/OAG94-064.htm\"},\n",
    "                {\"Name\":\"FileSize\",\"Value\":\"4.88 KB\"},{\"Name\":\"Title\",\"Value\":\"OAG 94-064\"},\n",
    "                {\"Name\":\"Title\",\"Value\":\"OAG 94-064\"},\n",
    "                {\"Name\":\"Date\",\"Value\":\"11/1/1994\"},\n",
    "                {\"Name\":\"Subject\",\"Value\":\"Whether a county attorney may supplement a sum set for his or her salary by the fiscal court, with monies obtained in connection with collection of delinquent taxes or related actions\"}\n",
    "            ]\n",
    "        }\n",
    "        \"\"\"\n",
    "        # Initialize an empty dictionary for the flattened object\n",
    "        flattened_object = {\"Id\": item[\"Id\"], \"Url\": item[\"Url\"]}\n",
    "\n",
    "        # Iterate over the \"Fields\" list\n",
    "        for field in item[\"Fields\"]:\n",
    "            # Add each \"Name\" and \"Value\" to the flattened object\n",
    "            flattened_object[field[\"Name\"]] = field[\"Value\"]\n",
    "            \n",
    "        # Get year off of the date\n",
    "        flattened_object['Year'] = int(flattened_object['Date'].split('/')[-1])\n",
    "        row_df = pd.DataFrame([flattened_object])\n",
    "        oag_df = pd.concat([oag_df, row_df])\n",
    "\n",
    "# Now master_df contains all the data from the tables across the pages\n",
    "oag_df.to_csv(os.path.join('opinions', 'kyag_oag_scraped.csv'), index=False)\n",
    "\n",
    "oag_df.describe(include=\"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "3d3c2441-9d66-4997-87ba-99103dc01af3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2024 2023 2022 2021 2020 2019 2018 2017 2016 2015 2014 2013 2012 2011\n",
      " 2010 2009 2008 2007 2006 2005 2004 2003 2002 2001 2000 1999 1998 1997\n",
      " 1996 1995 1994 1993 1992]\n"
     ]
    }
   ],
   "source": [
    "unique_years = oag_df['Year'].unique()\n",
    "print(unique_years)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "8b7b4e32-7e18-46c5-a988-1fcc225e7611",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Opinion%20of%20the%20Attorney%20General%20%2024-1.pdf downloaded successfully.\n",
      "Opinion%20of%20the%20Attorney%20General%2023-10.pdf downloaded successfully.\n",
      "Opinion%20of%20the%20Attorney%20General%2023-11.pdf downloaded successfully.\n",
      "Opinion%20of%20the%20Attorney%20General%2023-09.pdf downloaded successfully.\n",
      "Opinion%20of%20the%20Attorney%20General%2023-08.pdf downloaded successfully.\n",
      "Opinion%20of%20the%20Attorney%20General%2023-07.pdf downloaded successfully.\n",
      "Opinion%20of%20the%20Attorney%20General%2023-06.pdf downloaded successfully.\n",
      "Opinion%20of%20the%20Attorney%20General%2023-05.pdf downloaded successfully.\n",
      "Opinion%20of%20the%20Attorney%20General%2023-04.pdf downloaded successfully.\n",
      "Opinion%20of%20the%20Attorney%20General%2023-03.pdf downloaded successfully.\n",
      "Opinion%20of%20the%20Attorney%20General%2023-02.pdf downloaded successfully.\n",
      "Opinion%20of%20the%20Attorney%20General%2023-01.pdf downloaded successfully.\n",
      "OAG%2022-06.pdf downloaded successfully.\n",
      "OAG%2022-05.pdf downloaded successfully.\n",
      "OAG%2022-04.pdf downloaded successfully.\n",
      "OAG%2022-03.pdf downloaded successfully.\n",
      "OAG%2022-02.pdf downloaded successfully.\n",
      "OAG%2022-01.pdf downloaded successfully.\n",
      "OAG%2021-14.pdf downloaded successfully.\n",
      "OAG%2021-13.pdf downloaded successfully.\n",
      "OAG%2021-12.pdf downloaded successfully.\n",
      "Opinion%20of%20the%20Attorney%20General%2021-10.pdf downloaded successfully.\n",
      "Opinion%20of%20the%20Attorney%20General%2021-11.pdf downloaded successfully.\n",
      "OAG%2021-09.pdf downloaded successfully.\n",
      "OAG%2021-08.pdf downloaded successfully.\n",
      "OAG%2021-07.pdf downloaded successfully.\n",
      "OAG%2021-06.pdf downloaded successfully.\n",
      "OAG%2021-05.pdf downloaded successfully.\n",
      "OAG%2021-04.pdf downloaded successfully.\n",
      "OAG%2021-03.pdf downloaded successfully.\n",
      "OAG%2021-02.pdf downloaded successfully.\n",
      "OAG%2021-01.pdf downloaded successfully.\n",
      "OAG%2020-20.pdf downloaded successfully.\n",
      "OAG%2020-19.pdf downloaded successfully.\n",
      "OAG%2020-18.pdf downloaded successfully.\n",
      "OAG%2020-17.pdf downloaded successfully.\n",
      "OAG%2020-16.pdf downloaded successfully.\n",
      "OAG%2020-15.pdf downloaded successfully.\n",
      "OAG%2020-14.pdf downloaded successfully.\n",
      "OAG%2020-13.pdf downloaded successfully.\n",
      "OAG%2020-12.pdf downloaded successfully.\n",
      "OAG%2020-11.pdf downloaded successfully.\n",
      "OAG%2020-10.pdf downloaded successfully.\n",
      "OAG%2020-09.pdf downloaded successfully.\n",
      "OAG%2020-08.pdf downloaded successfully.\n",
      "OAG%2020-07.pdf downloaded successfully.\n",
      "OAG%2020-06.pdf downloaded successfully.\n",
      "OAG%2020-05.pdf downloaded successfully.\n",
      "OAG%2020-04.pdf downloaded successfully.\n",
      "OAG%2020-03.pdf downloaded successfully.\n",
      "OAG%2020-02.pdf downloaded successfully.\n",
      "OAG%2020-01.pdf downloaded successfully.\n",
      "OAG19-023.pdf downloaded successfully.\n",
      "OAG%2019-021.pdf downloaded successfully.\n",
      "OAG%2019-022.pdf downloaded successfully.\n",
      "OAG%2019-020.pdf downloaded successfully.\n",
      "OAG19-019.pdf downloaded successfully.\n",
      "OAG19-018.pdf downloaded successfully.\n",
      "OAG19-015.pdf downloaded successfully.\n",
      "OAG19-017.pdf downloaded successfully.\n",
      "OAG19-016.pdf downloaded successfully.\n",
      "OAG19-014.pdf downloaded successfully.\n",
      "OAG19-013.pdf downloaded successfully.\n",
      "OAG19-012.pdf downloaded successfully.\n",
      "OAG19-011.pdf downloaded successfully.\n",
      "OAG19-007.pdf downloaded successfully.\n",
      "OAG19-008.pdf downloaded successfully.\n",
      "OAG19-009.pdf downloaded successfully.\n",
      "OAG19-010.pdf downloaded successfully.\n",
      "OAG19-006.pdf downloaded successfully.\n",
      "OAG19-005.pdf downloaded successfully.\n",
      "OAG19-004.pdf downloaded successfully.\n",
      "OAG19-003.pdf downloaded successfully.\n",
      "OAG19-002.pdf downloaded successfully.\n",
      "OAG19-001.pdf downloaded successfully.\n",
      "OAG18-020.pdf downloaded successfully.\n",
      "OAG18-021.pdf downloaded successfully.\n",
      "OAG18-019.pdf downloaded successfully.\n",
      "OAG18-018.pdf downloaded successfully.\n",
      "OAG18-017.pdf downloaded successfully.\n",
      "OAG18-016.pdf downloaded successfully.\n",
      "OAG18-015.pdf downloaded successfully.\n",
      "OAG18-014.pdf downloaded successfully.\n",
      "OAG18-013.pdf downloaded successfully.\n",
      "OAG18-012.pdf downloaded successfully.\n",
      "OAG18-011.pdf downloaded successfully.\n",
      "OAG18-010.pdf downloaded successfully.\n",
      "OAG18-009.pdf downloaded successfully.\n",
      "OAG18-008.pdf downloaded successfully.\n",
      "OAG18-007.pdf downloaded successfully.\n",
      "OAG18-006.pdf downloaded successfully.\n",
      "OAG17-020.pdf downloaded successfully.\n",
      "OAG18-004.pdf downloaded successfully.\n",
      "OAG18-005.pdf downloaded successfully.\n",
      "OAG18-003.pdf downloaded successfully.\n",
      "OAG18-002.pdf downloaded successfully.\n",
      "OAG18-001.pdf downloaded successfully.\n",
      "OAG17-031.pdf downloaded successfully.\n",
      "OAG17-030.pdf downloaded successfully.\n",
      "OAG17-029.pdf downloaded successfully.\n",
      "OAG17-028.pdf downloaded successfully.\n",
      "OAG17-027.pdf downloaded successfully.\n",
      "OAG17-026.pdf downloaded successfully.\n",
      "OAG17-025.pdf downloaded successfully.\n",
      "OAG17-024.pdf downloaded successfully.\n",
      "OAG17-023.pdf downloaded successfully.\n",
      "OAG17-022.pdf downloaded successfully.\n",
      "OAG17-021.pdf downloaded successfully.\n",
      "OAG17-018.pdf downloaded successfully.\n",
      "OAG17-019.pdf downloaded successfully.\n",
      "OAG17-017.pdf downloaded successfully.\n",
      "OAG17-016.pdf downloaded successfully.\n",
      "OAG17-015.pdf downloaded successfully.\n",
      "OAG17-014.pdf downloaded successfully.\n",
      "OAG17-012.pdf downloaded successfully.\n",
      "OAG17-013.pdf downloaded successfully.\n",
      "OAG17-011.pdf downloaded successfully.\n",
      "OAG17-010.pdf downloaded successfully.\n",
      "OAG17-009.pdf downloaded successfully.\n",
      "OAG17-007.pdf downloaded successfully.\n",
      "OAG17-008.pdf downloaded successfully.\n",
      "OAG17-006.pdf downloaded successfully.\n",
      "OAG17-005.pdf downloaded successfully.\n",
      "OAG17-003.pdf downloaded successfully.\n",
      "OAG17-004.pdf downloaded successfully.\n",
      "OAG17-002.pdf downloaded successfully.\n",
      "OAG17-001.pdf downloaded successfully.\n",
      "OAG16-013.pdf downloaded successfully.\n",
      "OAG16-012.pdf downloaded successfully.\n",
      "OAG16-011.pdf downloaded successfully.\n",
      "OAG16-009.pdf downloaded successfully.\n",
      "OAG16-010.pdf downloaded successfully.\n",
      "OAG16-008.pdf downloaded successfully.\n",
      "OAG16-007.pdf downloaded successfully.\n",
      "OAG16-006.pdf downloaded successfully.\n",
      "OAG16-005.pdf downloaded successfully.\n",
      "OAG16-004.pdf downloaded successfully.\n",
      "OAG16-003.pdf downloaded successfully.\n",
      "OAG16-002.pdf downloaded successfully.\n",
      "OAG16-001.pdf downloaded successfully.\n",
      "OAG15-020.doc downloaded successfully.\n",
      "OAG15-022.doc downloaded successfully.\n",
      "OAG15-021.doc downloaded successfully.\n",
      "OAG15-019.doc downloaded successfully.\n",
      "OAG15-018.doc downloaded successfully.\n",
      "OAG15-017.doc downloaded successfully.\n",
      "OAG15-016.doc downloaded successfully.\n",
      "OAG15-015.doc downloaded successfully.\n",
      "OAG15-014.doc downloaded successfully.\n",
      "OAG15-013.doc downloaded successfully.\n",
      "OAG15-012.doc downloaded successfully.\n",
      "OAG15-010.doc downloaded successfully.\n",
      "OAG15-011.doc downloaded successfully.\n",
      "OAG15-009.doc downloaded successfully.\n",
      "OAG15-008.doc downloaded successfully.\n",
      "OAG15-007.doc downloaded successfully.\n",
      "OAG15-006.doc downloaded successfully.\n",
      "OAG15-004.doc downloaded successfully.\n",
      "OAG15-005.doc downloaded successfully.\n",
      "OAG15-003.doc downloaded successfully.\n",
      "OAG15-002.doc downloaded successfully.\n",
      "OAG15-001.doc downloaded successfully.\n",
      "OAG14-007.doc downloaded successfully.\n",
      "OAG14-006.doc downloaded successfully.\n",
      "OAG14-004.doc downloaded successfully.\n",
      "OAG14-005.doc downloaded successfully.\n",
      "OAG14-003.doc downloaded successfully.\n",
      "OAG14-002.doc downloaded successfully.\n",
      "OAG14-001.doc downloaded successfully.\n",
      "OAG13-013.doc downloaded successfully.\n",
      "OAG13-012.doc downloaded successfully.\n",
      "OAG13-011.doc downloaded successfully.\n",
      "OAG13-010.doc downloaded successfully.\n",
      "OAG13-009.doc downloaded successfully.\n",
      "OAG13-008.doc downloaded successfully.\n",
      "OAG13-007.doc downloaded successfully.\n",
      "OAG13-006.doc downloaded successfully.\n",
      "OAG13-005.doc downloaded successfully.\n",
      "OAG13-003.doc downloaded successfully.\n",
      "OAG13-004.doc downloaded successfully.\n",
      "OAG13-002.doc downloaded successfully.\n",
      "OAG13-001.doc downloaded successfully.\n",
      "OAG12-014.doc downloaded successfully.\n",
      "OAG12-015.doc downloaded successfully.\n",
      "OAG12-013.doc downloaded successfully.\n",
      "OAG12-012.doc downloaded successfully.\n",
      "OAG12-011.doc downloaded successfully.\n",
      "OAG12-010.doc downloaded successfully.\n",
      "OAG12-009.doc downloaded successfully.\n",
      "OAG12-008.doc downloaded successfully.\n",
      "OAG12-007.doc downloaded successfully.\n",
      "OAG12-006.doc downloaded successfully.\n",
      "OAG12-005.doc downloaded successfully.\n",
      "OAG12-004.doc downloaded successfully.\n",
      "OAG12-003.doc downloaded successfully.\n",
      "OAG12-002.doc downloaded successfully.\n",
      "OAG12-001.doc downloaded successfully.\n",
      "OAG11-005.doc downloaded successfully.\n",
      "OAG11-004.doc downloaded successfully.\n",
      "OAG11-003.doc downloaded successfully.\n",
      "OAG11-002.doc downloaded successfully.\n",
      "OAG11-001.doc downloaded successfully.\n",
      "OAG10-010.doc downloaded successfully.\n",
      "OAG10-009.doc downloaded successfully.\n",
      "OAG10-008.doc downloaded successfully.\n",
      "OAG10-007.doc downloaded successfully.\n",
      "OAG10-006.doc downloaded successfully.\n",
      "OAG10-005.doc downloaded successfully.\n",
      "OAG10-004.doc downloaded successfully.\n",
      "OAG10-003.doc downloaded successfully.\n",
      "OAG10-002.doc downloaded successfully.\n",
      "OAG10-001.doc downloaded successfully.\n",
      "OAG09-011.doc downloaded successfully.\n",
      "OAG09-010.doc downloaded successfully.\n",
      "OAG09-009.doc downloaded successfully.\n",
      "OAG09-008.doc downloaded successfully.\n",
      "OAG09-006.doc downloaded successfully.\n",
      "OAG09-005.doc downloaded successfully.\n",
      "OAG09-004.doc downloaded successfully.\n",
      "OAG09-003.doc downloaded successfully.\n",
      "OAG09-002.doc downloaded successfully.\n",
      "OAG09-001.doc downloaded successfully.\n",
      "OAG08-008.doc downloaded successfully.\n",
      "OAG08-007.doc downloaded successfully.\n",
      "OAG08-006.doc downloaded successfully.\n",
      "OAG08-004.doc downloaded successfully.\n",
      "OAG08-005.doc downloaded successfully.\n",
      "OAG08-003.doc downloaded successfully.\n",
      "OAG08-002.doc downloaded successfully.\n",
      "OAG08-001.doc downloaded successfully.\n",
      "OAG07-007.doc downloaded successfully.\n",
      "OAG07-006.doc downloaded successfully.\n",
      "OAG07-005.doc downloaded successfully.\n",
      "OAG07-004.doc downloaded successfully.\n",
      "OAG07-003.doc downloaded successfully.\n",
      "OAG07-002.doc downloaded successfully.\n",
      "OAG07-001.doc downloaded successfully.\n",
      "OAG06-003.doc downloaded successfully.\n",
      "OAG06-004.doc downloaded successfully.\n",
      "OAG06-002.doc downloaded successfully.\n",
      "OAG06-001.doc downloaded successfully.\n",
      "OAG05-010.doc downloaded successfully.\n",
      "OAG05-009.doc downloaded successfully.\n",
      "OAG05-008.doc downloaded successfully.\n",
      "OAG05-007.doc downloaded successfully.\n",
      "OAG05-005.doc downloaded successfully.\n",
      "OAG05-006.doc downloaded successfully.\n",
      "OAG05-004.doc downloaded successfully.\n",
      "OAG05-003.doc downloaded successfully.\n",
      "OAG05-002.doc downloaded successfully.\n",
      "OAG05-001.doc downloaded successfully.\n",
      "OAG04-014.doc downloaded successfully.\n",
      "OAG04-013.doc downloaded successfully.\n",
      "OAG04-011.doc downloaded successfully.\n",
      "OAG04-012.doc downloaded successfully.\n",
      "OAG04-010.doc downloaded successfully.\n",
      "OAG04-009.doc downloaded successfully.\n",
      "OAG04-008.doc downloaded successfully.\n",
      "OAG04-007.doc downloaded successfully.\n",
      "OAG04-005.doc downloaded successfully.\n",
      "OAG04-006.doc downloaded successfully.\n",
      "OAG04-004.doc downloaded successfully.\n",
      "OAG04-003.doc downloaded successfully.\n",
      "OAG04-002.doc downloaded successfully.\n",
      "OAG04-001.doc downloaded successfully.\n",
      "OAG03-004.doc downloaded successfully.\n",
      "OAG03-003.doc downloaded successfully.\n",
      "OAG03-002.doc downloaded successfully.\n",
      "OAG03-001.doc downloaded successfully.\n",
      "OAG02-006.doc downloaded successfully.\n",
      "OAG02-005.doc downloaded successfully.\n",
      "OAG02-004.doc downloaded successfully.\n",
      "OAG02-003.doc downloaded successfully.\n",
      "OAG02-002.doc downloaded successfully.\n",
      "OAG02-001.doc downloaded successfully.\n",
      "OAG01-010.doc downloaded successfully.\n",
      "OAG01-009.doc downloaded successfully.\n",
      "OAG01-008.doc downloaded successfully.\n",
      "OAG01-007.doc downloaded successfully.\n",
      "OAG01-006.doc downloaded successfully.\n",
      "OAG01-005.doc downloaded successfully.\n",
      "OAG01-004.doc downloaded successfully.\n",
      "OAG01-003.doc downloaded successfully.\n",
      "OAG01-002.doc downloaded successfully.\n",
      "OAG01-001.doc downloaded successfully.\n",
      "OAG00-008.doc downloaded successfully.\n",
      "OAG00-007.doc downloaded successfully.\n",
      "OAG00-006.doc downloaded successfully.\n",
      "OAG00-005.doc downloaded successfully.\n",
      "OAG00-004.doc downloaded successfully.\n",
      "OAG00-003.doc downloaded successfully.\n",
      "OAG00-002.doc downloaded successfully.\n",
      "OAG00-001.doc downloaded successfully.\n",
      "OAG99-010.doc downloaded successfully.\n",
      "OAG99-009.doc downloaded successfully.\n",
      "OAG99-008.doc downloaded successfully.\n",
      "OAG99-006.doc downloaded successfully.\n",
      "OAG99-007.doc downloaded successfully.\n",
      "OAG99-005.doc downloaded successfully.\n",
      "OAG99-004.doc downloaded successfully.\n",
      "OAG99-003.doc downloaded successfully.\n",
      "OAG99-002.doc downloaded successfully.\n",
      "OAG99-001.doc downloaded successfully.\n",
      "OAG98-013.doc downloaded successfully.\n",
      "OAG98-010.doc downloaded successfully.\n",
      "OAG98-011.doc downloaded successfully.\n",
      "OAG98-008.doc downloaded successfully.\n",
      "OAG98-007.doc downloaded successfully.\n",
      "OAG98-006.doc downloaded successfully.\n",
      "OAG98-005.doc downloaded successfully.\n",
      "OAG98-004.doc downloaded successfully.\n",
      "OAG98-003.doc downloaded successfully.\n",
      "OAG98-002.doc downloaded successfully.\n",
      "OAG98-001.doc downloaded successfully.\n",
      "OAG97-037.doc downloaded successfully.\n",
      "OAG97-036.doc downloaded successfully.\n",
      "OAG97-035.doc downloaded successfully.\n",
      "OAG97-034.doc downloaded successfully.\n",
      "OAG97-032.doc downloaded successfully.\n",
      "OAG97-033.doc downloaded successfully.\n",
      "OAG97-031.doc downloaded successfully.\n",
      "OAG97-029.doc downloaded successfully.\n",
      "OAG97-025.doc downloaded successfully.\n",
      "OAG97-027.doc downloaded successfully.\n",
      "OAG97-028.doc downloaded successfully.\n",
      "OAG97-030.doc downloaded successfully.\n",
      "OAG97-026.doc downloaded successfully.\n",
      "OAG97-024.doc downloaded successfully.\n",
      "OAG97-023.doc downloaded successfully.\n",
      "OAG97-022.doc downloaded successfully.\n",
      "OAG97-021.doc downloaded successfully.\n",
      "OAG97-020.doc downloaded successfully.\n",
      "OAG97-019.doc downloaded successfully.\n",
      "OAG97-018.htm downloaded successfully.\n",
      "OAG97-017.htm downloaded successfully.\n",
      "OAG97-016.htm downloaded successfully.\n",
      "OAG97-015.htm downloaded successfully.\n",
      "OAG97-013.htm downloaded successfully.\n",
      "OAG97-014.htm downloaded successfully.\n",
      "OAG97-012.htm downloaded successfully.\n",
      "OAG97-011.htm downloaded successfully.\n",
      "OAG97-010.htm downloaded successfully.\n",
      "OAG97-009.htm downloaded successfully.\n",
      "OAG97-008.htm downloaded successfully.\n",
      "OAG97-007.htm downloaded successfully.\n",
      "OAG97-006.htm downloaded successfully.\n",
      "OAG97-004.htm downloaded successfully.\n",
      "OAG97-005.htm downloaded successfully.\n",
      "OAG97-003.htm downloaded successfully.\n",
      "OAG97-002.htm downloaded successfully.\n",
      "OAG97-001.htm downloaded successfully.\n",
      "OAG96-046.htm downloaded successfully.\n",
      "OAG96-045.htm downloaded successfully.\n",
      "OAG96-044.htm downloaded successfully.\n",
      "OAG95-036.htm downloaded successfully.\n",
      "OAG96-043.htm downloaded successfully.\n",
      "OAG96-042.htm downloaded successfully.\n",
      "OAG96-041.htm downloaded successfully.\n",
      "OAG96-038.htm downloaded successfully.\n",
      "OAG96-040.doc downloaded successfully.\n",
      "OAG96-039.htm downloaded successfully.\n",
      "OAG96-037.htm downloaded successfully.\n",
      "OAG96-036.doc downloaded successfully.\n",
      "OAG96-035.htm downloaded successfully.\n",
      "OAG96-034.htm downloaded successfully.\n",
      "OAG96-033.htm downloaded successfully.\n",
      "OAG96-032.htm downloaded successfully.\n",
      "OAG96-031.htm downloaded successfully.\n",
      "OAG96-030.htm downloaded successfully.\n",
      "OAG96-029.htm downloaded successfully.\n",
      "OAG96-027.htm downloaded successfully.\n",
      "OAG96-028.htm downloaded successfully.\n",
      "OAG96-026.htm downloaded successfully.\n",
      "OAG96-025.htm downloaded successfully.\n",
      "OAG96-024.htm downloaded successfully.\n",
      "OAG96-023.htm downloaded successfully.\n",
      "OAG96-022.htm downloaded successfully.\n",
      "OAG96-021.htm downloaded successfully.\n",
      "OAG96-020.htm downloaded successfully.\n",
      "OAG96-019.htm downloaded successfully.\n",
      "OAG96-018.htm downloaded successfully.\n",
      "OAG96-017.htm downloaded successfully.\n",
      "OAG96-016.htm downloaded successfully.\n",
      "OAG96-015.htm downloaded successfully.\n",
      "OAG96-014.htm downloaded successfully.\n",
      "OAG96-013.htm downloaded successfully.\n",
      "OAG96-012.htm downloaded successfully.\n",
      "OAG96-009.htm downloaded successfully.\n",
      "OAG96-010.htm downloaded successfully.\n",
      "OAG96-011.htm downloaded successfully.\n",
      "OAG96-008.htm downloaded successfully.\n",
      "OAG96-006.htm downloaded successfully.\n",
      "OAG96-005.htm downloaded successfully.\n",
      "OAG96-007.htm downloaded successfully.\n",
      "OAG96-004.htm downloaded successfully.\n",
      "OAG96-003.htm downloaded successfully.\n",
      "OAG96-002.htm downloaded successfully.\n",
      "OAG96-001.htm downloaded successfully.\n",
      "OAG95-038.htm downloaded successfully.\n",
      "OAG95-037.htm downloaded successfully.\n",
      "OAG95-035.htm downloaded successfully.\n",
      "OAG95-034.htm downloaded successfully.\n",
      "OAG95-033.htm downloaded successfully.\n",
      "OAG95-032.htm downloaded successfully.\n",
      "OAG95-031.htm downloaded successfully.\n",
      "OAG95-030.htm downloaded successfully.\n",
      "OAG95-029.htm downloaded successfully.\n",
      "OAG95-027.htm downloaded successfully.\n",
      "OAG95-028.htm downloaded successfully.\n",
      "OAG95-025.htm downloaded successfully.\n",
      "OAG95-026.htm downloaded successfully.\n",
      "OAG95-024.htm downloaded successfully.\n",
      "OAG95-023.htm downloaded successfully.\n",
      "OAG95-021.htm downloaded successfully.\n",
      "OAG95-022.htm downloaded successfully.\n",
      "OAG95-020.htm downloaded successfully.\n",
      "OAG95-019.htm downloaded successfully.\n",
      "OAG95-018.htm downloaded successfully.\n",
      "OAG95-017.htm downloaded successfully.\n",
      "OAG95-014.htm downloaded successfully.\n",
      "OAG95-016.htm downloaded successfully.\n",
      "OAG95-015.htm downloaded successfully.\n",
      "OAG95-013.htm downloaded successfully.\n",
      "OAG95-012.htm downloaded successfully.\n",
      "OAG95-010.htm downloaded successfully.\n",
      "OAG95-011.htm downloaded successfully.\n",
      "OAG95-009.htm downloaded successfully.\n",
      "OAG95-008.htm downloaded successfully.\n",
      "OAG95-007.htm downloaded successfully.\n",
      "OAG95-006.htm downloaded successfully.\n",
      "OAG95-004.htm downloaded successfully.\n",
      "OAG95-005.htm downloaded successfully.\n",
      "OAG95-002.htm downloaded successfully.\n",
      "OAG95-003.htm downloaded successfully.\n",
      "OAG95-001.htm downloaded successfully.\n",
      "OAG94-071.htm downloaded successfully.\n",
      "OAG94-070.htm downloaded successfully.\n",
      "OAG94-069.htm downloaded successfully.\n",
      "OAG94-068.htm downloaded successfully.\n",
      "OAG94-066.htm downloaded successfully.\n",
      "OAG94-067.htm downloaded successfully.\n",
      "OAG94-064.htm downloaded successfully.\n",
      "OAG94-063.htm downloaded successfully.\n",
      "OAG94-065.htm downloaded successfully.\n",
      "OAG94-062.htm downloaded successfully.\n",
      "OAG94-061.htm downloaded successfully.\n",
      "OAG94-060.htm downloaded successfully.\n",
      "OAG94-059.htm downloaded successfully.\n",
      "OAG94-058.htm downloaded successfully.\n",
      "OAG94-057.htm downloaded successfully.\n",
      "OAG94-056.htm downloaded successfully.\n",
      "OAG94-055.htm downloaded successfully.\n",
      "OAG94-054.htm downloaded successfully.\n",
      "OAG94-053.htm downloaded successfully.\n",
      "OAG94-051.htm downloaded successfully.\n",
      "OAG94-052.htm downloaded successfully.\n",
      "OAG94-050.htm downloaded successfully.\n",
      "OAG94-049.htm downloaded successfully.\n",
      "OAG94-048.htm downloaded successfully.\n",
      "OAG94-047.htm downloaded successfully.\n",
      "OAG94-046.htm downloaded successfully.\n",
      "OAG94-044.htm downloaded successfully.\n",
      "OAG94-045.htm downloaded successfully.\n",
      "OAG94-043.htm downloaded successfully.\n",
      "OAG94-042.htm downloaded successfully.\n",
      "OAG94-041.htm downloaded successfully.\n",
      "OAG94-040.htm downloaded successfully.\n",
      "OAG94-039.htm downloaded successfully.\n",
      "OAG94-038.htm downloaded successfully.\n",
      "OAG94-037.htm downloaded successfully.\n",
      "OAG94-036.htm downloaded successfully.\n",
      "OAG94-035.htm downloaded successfully.\n",
      "OAG94-032.htm downloaded successfully.\n",
      "OAG94-034.htm downloaded successfully.\n",
      "OAG94-033.htm downloaded successfully.\n",
      "OAG94-031.htm downloaded successfully.\n",
      "OAG94-030.htm downloaded successfully.\n",
      "OAG94-028.htm downloaded successfully.\n",
      "OAG94-029.htm downloaded successfully.\n",
      "OAG94-027.htm downloaded successfully.\n",
      "OAG94-026.htm downloaded successfully.\n",
      "OAG94-025.htm downloaded successfully.\n",
      "OAG94-022.htm downloaded successfully.\n",
      "OAG94-024.htm downloaded successfully.\n",
      "OAG94-023.htm downloaded successfully.\n",
      "OAG94-021.htm downloaded successfully.\n",
      "OAG94-020.htm downloaded successfully.\n",
      "OAG94-019.htm downloaded successfully.\n",
      "OAG94-018.htm downloaded successfully.\n",
      "OAG94-017.htm downloaded successfully.\n",
      "OAG94-016.htm downloaded successfully.\n",
      "OAG94-015.htm downloaded successfully.\n",
      "OAG94-014.htm downloaded successfully.\n",
      "OAG94-013.htm downloaded successfully.\n",
      "OAG94-010.htm downloaded successfully.\n",
      "OAG94-012.htm downloaded successfully.\n",
      "OAG94-011.htm downloaded successfully.\n",
      "OAG94-009.htm downloaded successfully.\n",
      "OAG94-008.htm downloaded successfully.\n",
      "OAG94-007.htm downloaded successfully.\n",
      "OAG94-006.htm downloaded successfully.\n",
      "OAG94-005.htm downloaded successfully.\n",
      "OAG94-004.htm downloaded successfully.\n",
      "OAG94-003.htm downloaded successfully.\n",
      "OAG94-002.htm downloaded successfully.\n",
      "OAG94-001.htm downloaded successfully.\n",
      "OAG93-084.htm downloaded successfully.\n",
      "OAG93-083.htm downloaded successfully.\n",
      "OAG93-082.htm downloaded successfully.\n",
      "OAG93-080.htm downloaded successfully.\n",
      "OAG93-081.htm downloaded successfully.\n",
      "OAG93-079.htm downloaded successfully.\n",
      "OAG93-078.htm downloaded successfully.\n",
      "OAG93-077.htm downloaded successfully.\n",
      "OAG93-076.htm downloaded successfully.\n",
      "OAG93-075.htm downloaded successfully.\n",
      "OAG93-074.htm downloaded successfully.\n",
      "OAG93-073.htm downloaded successfully.\n",
      "OAG93-072.htm downloaded successfully.\n",
      "OAG93-071.htm downloaded successfully.\n",
      "OAG93-070.htm downloaded successfully.\n",
      "OAG93-068.htm downloaded successfully.\n",
      "OAG93-069.htm downloaded successfully.\n",
      "OAG93-067.htm downloaded successfully.\n",
      "OAG93-066.htm downloaded successfully.\n",
      "OAG93-065.htm downloaded successfully.\n",
      "OAG93-064.htm downloaded successfully.\n",
      "OAG93-063.htm downloaded successfully.\n",
      "OAG93-062.htm downloaded successfully.\n",
      "OAG93-061.htm downloaded successfully.\n",
      "OAG93-060.htm downloaded successfully.\n",
      "OAG93-059.htm downloaded successfully.\n",
      "OAG93-058.htm downloaded successfully.\n",
      "OAG93-057.htm downloaded successfully.\n",
      "OAG93-056.htm downloaded successfully.\n",
      "OAG93-055.htm downloaded successfully.\n",
      "OAG93-054.htm downloaded successfully.\n",
      "OAG93-053.htm downloaded successfully.\n",
      "OAG93-052.htm downloaded successfully.\n",
      "OAG93-050.htm downloaded successfully.\n",
      "OAG93-049.htm downloaded successfully.\n",
      "OAG93-051.htm downloaded successfully.\n",
      "OAG93-047.htm downloaded successfully.\n",
      "OAG93-048.htm downloaded successfully.\n",
      "OAG93-046.htm downloaded successfully.\n",
      "OAG93-045.htm downloaded successfully.\n",
      "OAG93-044.htm downloaded successfully.\n",
      "OAG93-043.htm downloaded successfully.\n",
      "OAG93-042.htm downloaded successfully.\n",
      "OAG93-041.htm downloaded successfully.\n",
      "OAG93-040.htm downloaded successfully.\n",
      "OAG93-039.htm downloaded successfully.\n",
      "OAG93-037.htm downloaded successfully.\n",
      "OAG93-015.htm downloaded successfully.\n",
      "OAG93-038.htm downloaded successfully.\n",
      "OAG93-036.htm downloaded successfully.\n",
      "OAG93-035.htm downloaded successfully.\n",
      "OAG93-034.htm downloaded successfully.\n",
      "OAG93-033.htm downloaded successfully.\n",
      "OAG93-031.htm downloaded successfully.\n",
      "OAG93-032.htm downloaded successfully.\n",
      "OAG93-030.htm downloaded successfully.\n",
      "OAG93-029.htm downloaded successfully.\n",
      "OAG93-028.htm downloaded successfully.\n",
      "OAG93-026.htm downloaded successfully.\n",
      "OAG93-027.htm downloaded successfully.\n",
      "OAG93-025.htm downloaded successfully.\n",
      "OAG93-022.htm downloaded successfully.\n",
      "OAG93-024.htm downloaded successfully.\n",
      "OAG93-023.htm downloaded successfully.\n",
      "OAG93-020.htm downloaded successfully.\n",
      "OAG93-021.htm downloaded successfully.\n",
      "OAG93-019.htm downloaded successfully.\n",
      "OAG93-017.htm downloaded successfully.\n",
      "OAG93-018.htm downloaded successfully.\n",
      "OAG93-014.htm downloaded successfully.\n",
      "OAG93-013.doc downloaded successfully.\n",
      "OAG93-016.htm downloaded successfully.\n",
      "OAG93-012.htm downloaded successfully.\n",
      "OAG93-011.htm downloaded successfully.\n",
      "OAG93-009.htm downloaded successfully.\n",
      "OAG93-010.htm downloaded successfully.\n",
      "OAG93-008.htm downloaded successfully.\n",
      "OAG93-007.htm downloaded successfully.\n",
      "OAG93-006.htm downloaded successfully.\n",
      "OAG93-005.htm downloaded successfully.\n",
      "OAG93-004.htm downloaded successfully.\n",
      "OAG93-003.htm downloaded successfully.\n",
      "OAG93-002.htm downloaded successfully.\n",
      "OAG93-001.htm downloaded successfully.\n",
      "OAG92-163.htm downloaded successfully.\n",
      "OAG92-147.htm downloaded successfully.\n",
      "OAG92-137.htm downloaded successfully.\n",
      "OAG92-134.htm downloaded successfully.\n",
      "OAG92-132.htm downloaded successfully.\n",
      "OAG92-124.htm downloaded successfully.\n",
      "OAG92-114.htm downloaded successfully.\n",
      "OAG92-113.htm downloaded successfully.\n",
      "OAG92-108.htm downloaded successfully.\n",
      "OAG92-097.htm downloaded successfully.\n",
      "OAG92-096.htm downloaded successfully.\n",
      "OAG92-094.htm downloaded successfully.\n",
      "OAG92-095.htm downloaded successfully.\n",
      "OAG92-093.htm downloaded successfully.\n",
      "OAG92-092.htm downloaded successfully.\n",
      "OAG92-087.htm downloaded successfully.\n",
      "OAG92-086.htm downloaded successfully.\n",
      "OAG92-073.htm downloaded successfully.\n",
      "OAG92-070.htm downloaded successfully.\n",
      "OAG92-069.htm downloaded successfully.\n",
      "OAG92-072.htm downloaded successfully.\n",
      "OAG92-071.htm downloaded successfully.\n",
      "OAG92-058.htm downloaded successfully.\n",
      "OAG92-057.doc downloaded successfully.\n",
      "OAG92-055.htm downloaded successfully.\n",
      "OAG92-052.doc downloaded successfully.\n",
      "OAG92-048.doc downloaded successfully.\n",
      "OAG92-047.doc downloaded successfully.\n",
      "OAG92-045.doc downloaded successfully.\n",
      "OAG92-043.doc downloaded successfully.\n",
      "OAG92-042.doc downloaded successfully.\n",
      "OAG92-041.doc downloaded successfully.\n",
      "OAG92-040.htm downloaded successfully.\n",
      "OAG92-039.doc downloaded successfully.\n",
      "OAG92-038.htm downloaded successfully.\n",
      "OAG92-037.doc downloaded successfully.\n",
      "OAG92-036.doc downloaded successfully.\n",
      "OAG92-035.doc downloaded successfully.\n",
      "OAG92-034.doc downloaded successfully.\n",
      "OAG92-033.doc downloaded successfully.\n",
      "OAG92-029.doc downloaded successfully.\n",
      "OAG92-030.doc downloaded successfully.\n",
      "OAG92-031.doc downloaded successfully.\n",
      "OAG92-032.doc downloaded successfully.\n",
      "OAG92-028.doc downloaded successfully.\n",
      "OAG92-024.doc downloaded successfully.\n",
      "OAG92-027.doc downloaded successfully.\n",
      "OAG92-026.doc downloaded successfully.\n",
      "OAG92-025.doc downloaded successfully.\n",
      "OAG92-022.htm downloaded successfully.\n",
      "OAG92-023.htm downloaded successfully.\n",
      "OAG92-021.htm downloaded successfully.\n",
      "OAG92-018.doc downloaded successfully.\n",
      "OAG92-019.htm downloaded successfully.\n",
      "OAG92-020.doc downloaded successfully.\n",
      "OAG92-016.doc downloaded successfully.\n",
      "OAG92-015.htm downloaded successfully.\n",
      "OAG92-014.doc downloaded successfully.\n",
      "OAG92-013.doc downloaded successfully.\n",
      "OAG92-010.doc downloaded successfully.\n",
      "OAG92-011.doc downloaded successfully.\n",
      "OAG92-012.htm downloaded successfully.\n",
      "OAG92-007.doc downloaded successfully.\n",
      "OAG92-006.doc downloaded successfully.\n",
      "OAG92-008.doc downloaded successfully.\n",
      "OAG92-009.doc downloaded successfully.\n",
      "OAG92-004.doc downloaded successfully.\n",
      "OAG92-005.doc downloaded successfully.\n",
      "OAG92-002.doc downloaded successfully.\n",
      "OAG92-003.doc downloaded successfully.\n",
      "OAG92-001.doc downloaded successfully.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import requests\n",
    "\n",
    "# Sort the DataFrame by \"Date\" in descending order\n",
    "oag_df['Date'] = oag_df['Date'].apply(pd.to_datetime)\n",
    "oag_df = oag_df.sort_values(by=\"Date\", ascending=False)\n",
    "\n",
    "# Define the directory where you want to save the files\n",
    "directory = \"opinions\"\n",
    "\n",
    "# Iterate through each row of the DataFrame\n",
    "for index, row in oag_df.iterrows():\n",
    "    # Construct the target directory path\n",
    "    target_dir = os.path.join(directory, str(row[\"Year\"]), row[\"Title\"])\n",
    "    \n",
    "    # Create the target directory if it doesn't exist\n",
    "    os.makedirs(target_dir, exist_ok=True)\n",
    "    \n",
    "    # Construct the target file path\n",
    "    # Assuming the file name is the last part of the URL in the \"Download\" column\n",
    "    file_name = row[\"Url\"].split('/')[-1]\n",
    "    target_file = os.path.join(target_dir, file_name)\n",
    "    \n",
    "    # Check if the file already exists\n",
    "    if not os.path.exists(target_file):\n",
    "        # Download the file\n",
    "        response = requests.get(row[\"Url\"], headers=html_headers)\n",
    "        \n",
    "        if response.status_code == 200:\n",
    "            # Save the file to the target directory\n",
    "            with open(target_file, 'wb') as f:\n",
    "                f.write(response.content)\n",
    "            print(f\"{file_name} downloaded successfully.\")\n",
    "        else:\n",
    "            print(f\"Failed to download {row['Url']}. HTTP response status code: {response.status_code}\")\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "2637662a-6a46-4c88-a230-8a2d3a745ea7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "def print_statusline(msg: str):\n",
    "    last_msg_length = len(getattr(print_statusline, 'last_msg', ''))\n",
    "    print(' ' * last_msg_length, end='\\r')\n",
    "    print(msg, end='\\r')\n",
    "    sys.stdout.flush() # Ensures the output is immediately visible\n",
    "    setattr(print_statusline, 'last_msg', msg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "f22f5418-cb72-4bc6-b88a-c2bd75485bc2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "09ORD156.doc downloaded successfully.  \r"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import requests\n",
    "import json\n",
    "\n",
    "# Sort the DataFrame by \"Date\" in descending order\n",
    "master_df['Date'] = master_df['Date'].apply(pd.to_datetime)\n",
    "master_df = master_df.sort_values(by=\"Date\", ascending=False)\n",
    "\n",
    "# Define the directory where you want to save the files\n",
    "directory = \"opinions\"\n",
    "\n",
    "# Iterate through each row of the DataFrame\n",
    "for index, row in master_df.iterrows():\n",
    "    if isinstance(row[\"Download\"], str) == False:\n",
    "        continue;\n",
    "    \n",
    "    # Construct the target directory path\n",
    "    target_dir = os.path.join(directory, str(row[\"Year\"]), row[\"Title\"])\n",
    "    \n",
    "    # Create the target directory if it doesn't exist\n",
    "    os.makedirs(target_dir, exist_ok=True)\n",
    "    \n",
    "    # Construct the target file path\n",
    "    # Assuming the file name is the last part of the URL in the \"Download\" column\n",
    "    file_name = row[\"Download\"].split('/')[-1]\n",
    "    target_file = os.path.join(target_dir, file_name)\n",
    "    \n",
    "    # Check if the file already exists\n",
    "    if not os.path.exists(target_file):\n",
    "        # Download the file\n",
    "        response = requests.get(row[\"Download\"], headers=html_headers)\n",
    "        \n",
    "        if response.status_code == 200:\n",
    "            # Save the file to the target directory\n",
    "            with open(target_file, 'wb') as f:\n",
    "                f.write(response.content)\n",
    "            print_statusline(f\"{file_name} downloaded successfully.\")\n",
    "        else:\n",
    "            print(f\"Failed to download {row['Download']}. HTTP response status code: {response.status_code}\")\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f1b472a-1d19-4aad-b45e-b2aa657c505a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "my-python3-kernel",
   "language": "python",
   "name": "my-python3-kernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}