provisonal.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Identifying Healthcare Professionals (HCP) and their specialization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Code Fighters \n",
    "- 063_ARKAJYOTI KARMAKAR (Leader)\n",
    "- Amisha Kumari\n",
    "- Gagan Agarwal\n",
    "- Parag Ranjan"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importing necessary libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importing csv to pandas dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.read_csv('Doceree-HCP_Test.csv', encoding='latin1')\n",
    "train_df = pd.read_csv('Doceree-HCP_Train.csv', encoding='latin1')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analysing the Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>DEVICETYPE</th>\n",
       "      <th>PLATFORM_ID</th>\n",
       "      <th>BIDREQUESTIP</th>\n",
       "      <th>USERPLATFORMUID</th>\n",
       "      <th>USERCITY</th>\n",
       "      <th>USERZIPCODE</th>\n",
       "      <th>USERAGENT</th>\n",
       "      <th>PLATFORMTYPE</th>\n",
       "      <th>CHANNELTYPE</th>\n",
       "      <th>URL</th>\n",
       "      <th>KEYWORDS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>115501</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>2</td>\n",
       "      <td>75.189.231.103</td>\n",
       "      <td>0d5041ff-f0b6-4d1a-9ad7-0a29f7d485b4</td>\n",
       "      <td>Fayetteville</td>\n",
       "      <td>28305.0</td>\n",
       "      <td>Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.clinicaladvisor.com/home/features/...</td>\n",
       "      <td>Family Practice|Drainage|Clinical|Dermatology|...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>115502</td>\n",
       "      <td>Mobile</td>\n",
       "      <td>2</td>\n",
       "      <td>24.101.33.158</td>\n",
       "      <td>c8396dd0-969f-4d99-a40b-b7bb1f516154</td>\n",
       "      <td>Conneaut Lake</td>\n",
       "      <td>16316.0</td>\n",
       "      <td>Mozilla/5.0 (iPhone; CPU iPhone OS 15_6_1 like...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.ophthalmologyadvisor.com/topics/ca...</td>\n",
       "      <td>General|Clinical|Operative|Medicine|Cardiology...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>115503</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>2</td>\n",
       "      <td>172.118.216.142</td>\n",
       "      <td>3c97a081-6518-43f8-9f26-369759cfb471</td>\n",
       "      <td>Covina</td>\n",
       "      <td>91724.0</td>\n",
       "      <td>Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.psychiatryadvisor.com/author/tori-...</td>\n",
       "      <td>Abortion|Anxiety Disorders|Apnea|False|Trauma|...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>115504</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>7</td>\n",
       "      <td>71.105.120.171</td>\n",
       "      <td>3e2578c8-f794-41af-a38c-c5cfb3c0f014</td>\n",
       "      <td>Brooklyn</td>\n",
       "      <td>11226.0</td>\n",
       "      <td>Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.cureus.com/articles/105482-diverti...</td>\n",
       "      <td>Health|Male|Neurological Surgery|Otolaryngolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>115505</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>2</td>\n",
       "      <td>73.82.211.73</td>\n",
       "      <td>ec2ae7ce-6a8c-4156-98a7-07203e60f483</td>\n",
       "      <td>Marietta</td>\n",
       "      <td>30062.0</td>\n",
       "      <td>Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.renalandurologynews.com/home/confe...</td>\n",
       "      <td>chronic kidney disease|pain|nephrology|disease...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>115506</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>2</td>\n",
       "      <td>169.147.254.129</td>\n",
       "      <td>0313febc-12e6-4289-8d74-5f0c1aecc385</td>\n",
       "      <td>Kansas City</td>\n",
       "      <td>64113.0</td>\n",
       "      <td>Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.renalandurologynews.com/home/decis...</td>\n",
       "      <td>Abscess|Liver Transplantation|Transplantation|...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>115507</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>9</td>\n",
       "      <td>66.249.66.2</td>\n",
       "      <td>ce1f2f66-056e-4684-8c24-6d70e632a269</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Bu...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.2minutemedicine.com/studygraphics-...</td>\n",
       "      <td>the|medicine|oncology|</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>115508</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>2</td>\n",
       "      <td>50.236.245.242</td>\n",
       "      <td>8fc0a64c-0c71-4888-aff4-833823802bba</td>\n",
       "      <td>Seattle</td>\n",
       "      <td>98188.0</td>\n",
       "      <td>Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.cancertherapyadvisor.com/home/canc...</td>\n",
       "      <td>General|Endocrine|False|Medicine|Surgery|Urolo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>115509</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>2</td>\n",
       "      <td>204.99.118.10</td>\n",
       "      <td>12ec92c0-a55b-45a2-8c4e-4237dcfc8eef</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>85260.0</td>\n",
       "      <td>Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.empr.com/drug/evrysdi/</td>\n",
       "      <td>Medicine|Metabolic|Dermatology|Child|False|Mal...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>115510</td>\n",
       "      <td>Mobile</td>\n",
       "      <td>2</td>\n",
       "      <td>71.228.67.64</td>\n",
       "      <td>d0a2d8ab-6ea1-440e-a2f1-03acd6a1db99</td>\n",
       "      <td>Sarasota</td>\n",
       "      <td>34232.0</td>\n",
       "      <td>Mozilla/5.0 (iPhone; CPU iPhone OS 16_3_1 like...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.clinicaladvisor.com/home/dermatolo...</td>\n",
       "      <td>Hemoptysis|Oral|Rheumatology|Cardiovascular|Sm...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       ID DEVICETYPE  PLATFORM_ID     BIDREQUESTIP  \\\n",
       "0  115501    Desktop            2   75.189.231.103   \n",
       "1  115502     Mobile            2    24.101.33.158   \n",
       "2  115503    Desktop            2  172.118.216.142   \n",
       "3  115504    Desktop            7   71.105.120.171   \n",
       "4  115505    Desktop            2     73.82.211.73   \n",
       "5  115506    Desktop            2  169.147.254.129   \n",
       "6  115507    Desktop            9      66.249.66.2   \n",
       "7  115508    Desktop            2   50.236.245.242   \n",
       "8  115509    Desktop            2    204.99.118.10   \n",
       "9  115510     Mobile            2     71.228.67.64   \n",
       "\n",
       "                        USERPLATFORMUID       USERCITY  USERZIPCODE  \\\n",
       "0  0d5041ff-f0b6-4d1a-9ad7-0a29f7d485b4   Fayetteville      28305.0   \n",
       "1  c8396dd0-969f-4d99-a40b-b7bb1f516154  Conneaut Lake      16316.0   \n",
       "2  3c97a081-6518-43f8-9f26-369759cfb471         Covina      91724.0   \n",
       "3  3e2578c8-f794-41af-a38c-c5cfb3c0f014       Brooklyn      11226.0   \n",
       "4  ec2ae7ce-6a8c-4156-98a7-07203e60f483       Marietta      30062.0   \n",
       "5  0313febc-12e6-4289-8d74-5f0c1aecc385    Kansas City      64113.0   \n",
       "6  ce1f2f66-056e-4684-8c24-6d70e632a269            NaN          NaN   \n",
       "7  8fc0a64c-0c71-4888-aff4-833823802bba        Seattle      98188.0   \n",
       "8  12ec92c0-a55b-45a2-8c4e-4237dcfc8eef     Scottsdale      85260.0   \n",
       "9  d0a2d8ab-6ea1-440e-a2f1-03acd6a1db99       Sarasota      34232.0   \n",
       "\n",
       "                                           USERAGENT            PLATFORMTYPE  \\\n",
       "0  Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6...  Online Medical Journal   \n",
       "1  Mozilla/5.0 (iPhone; CPU iPhone OS 15_6_1 like...  Online Medical Journal   \n",
       "2  Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...  Online Medical Journal   \n",
       "3  Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...  Online Medical Journal   \n",
       "4  Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...  Online Medical Journal   \n",
       "5  Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...  Online Medical Journal   \n",
       "6  Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Bu...  Online Medical Journal   \n",
       "7  Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...  Online Medical Journal   \n",
       "8  Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...  Online Medical Journal   \n",
       "9  Mozilla/5.0 (iPhone; CPU iPhone OS 16_3_1 like...  Online Medical Journal   \n",
       "\n",
       "  CHANNELTYPE                                                URL  \\\n",
       "0     Website  https://www.clinicaladvisor.com/home/features/...   \n",
       "1     Website  https://www.ophthalmologyadvisor.com/topics/ca...   \n",
       "2     Website  https://www.psychiatryadvisor.com/author/tori-...   \n",
       "3     Website  https://www.cureus.com/articles/105482-diverti...   \n",
       "4     Website  https://www.renalandurologynews.com/home/confe...   \n",
       "5     Website  https://www.renalandurologynews.com/home/decis...   \n",
       "6     Website  https://www.2minutemedicine.com/studygraphics-...   \n",
       "7     Website  https://www.cancertherapyadvisor.com/home/canc...   \n",
       "8     Website                 https://www.empr.com/drug/evrysdi/   \n",
       "9     Website  https://www.clinicaladvisor.com/home/dermatolo...   \n",
       "\n",
       "                                            KEYWORDS  \n",
       "0  Family Practice|Drainage|Clinical|Dermatology|...  \n",
       "1  General|Clinical|Operative|Medicine|Cardiology...  \n",
       "2  Abortion|Anxiety Disorders|Apnea|False|Trauma|...  \n",
       "3  Health|Male|Neurological Surgery|Otolaryngolog...  \n",
       "4  chronic kidney disease|pain|nephrology|disease...  \n",
       "5  Abscess|Liver Transplantation|Transplantation|...  \n",
       "6                             the|medicine|oncology|  \n",
       "7  General|Endocrine|False|Medicine|Surgery|Urolo...  \n",
       "8  Medicine|Metabolic|Dermatology|Child|False|Mal...  \n",
       "9  Hemoptysis|Oral|Rheumatology|Cardiovascular|Sm...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(113937, 14)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ID                   int64\n",
       "DEVICETYPE          object\n",
       "PLATFORM_ID          int64\n",
       "BIDREQUESTIP        object\n",
       "USERPLATFORMUID     object\n",
       "USERCITY            object\n",
       "USERZIPCODE        float64\n",
       "USERAGENT           object\n",
       "PLATFORMTYPE        object\n",
       "CHANNELTYPE         object\n",
       "URL                 object\n",
       "KEYWORDS            object\n",
       "TAXONOMY            object\n",
       "IS_HCP             float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 113937 entries, 0 to 113936\n",
      "Data columns (total 14 columns):\n",
      " #   Column           Non-Null Count   Dtype  \n",
      "---  ------           --------------   -----  \n",
      " 0   ID               113937 non-null  int64  \n",
      " 1   DEVICETYPE       113937 non-null  object \n",
      " 2   PLATFORM_ID      113937 non-null  int64  \n",
      " 3   BIDREQUESTIP     113937 non-null  object \n",
      " 4   USERPLATFORMUID  113933 non-null  object \n",
      " 5   USERCITY         107578 non-null  object \n",
      " 6   USERZIPCODE      109345 non-null  float64\n",
      " 7   USERAGENT        113935 non-null  object \n",
      " 8   PLATFORMTYPE     113937 non-null  object \n",
      " 9   CHANNELTYPE      113937 non-null  object \n",
      " 10  URL              113937 non-null  object \n",
      " 11  KEYWORDS         113937 non-null  object \n",
      " 12  TAXONOMY         32313 non-null   object \n",
      " 13  IS_HCP           113936 non-null  float64\n",
      "dtypes: float64(2), int64(2), object(10)\n",
      "memory usage: 12.2+ MB\n"
     ]
    }
   ],
   "source": [
    "train_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Online Medical Journal      110120\n",
      "Online Learning Portal        2978\n",
      "Medical News Platform          798\n",
      "Electronic Health Record        40\n",
      "Tele-medicine Platform           1\n",
      "Name: PLATFORMTYPE, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "value_counts = train_df['PLATFORMTYPE'].value_counts()\n",
    "print(value_counts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imputation, Feature Engineering and Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_to_drop = [\"ID\", \"DEVICETYPE\", \"PLATFORM_ID\", \"CHANNELTYPE\", \"USERAGENT\"]\n",
    "train_df.drop(columns=columns_to_drop, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking distribution of values in features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "++- BIDREQUESTIP -++\n",
      "71.105.120.171     1718\n",
      "142.154.205.202    1143\n",
      "68.132.98.211       866\n",
      "108.175.114.100     727\n",
      "108.175.114.101     707\n",
      "                   ... \n",
      "66.66.9.45            1\n",
      "97.80.251.36          1\n",
      "136.32.32.117         1\n",
      "64.127.133.6          1\n",
      "108.41.233.175        1\n",
      "Name: BIDREQUESTIP, Length: 33664, dtype: int64\n",
      "=======================\n",
      "++- USERPLATFORMUID -++\n",
      "3e2578c8-f794-41af-a38c-c5cfb3c0f014    1447\n",
      "d76f7c0b-1a64-4d4b-b36d-86c832be8837    1406\n",
      "d00f28d6-3a50-46cd-92f3-8601bd57ad0e     675\n",
      "fcd3d327-0ad5-425e-8ea8-33ffe6a31543     629\n",
      "fe53f32c-4646-4abe-afea-90b21d6b8531     617\n",
      "                                        ... \n",
      "6759cbf8-cf4f-4d0e-9eb8-00808e69e841       1\n",
      "bd2007ca-d0b5-4620-8078-b7c6db62037c       1\n",
      "0534212a-e0e9-49e7-ace9-608e0d0eae8a       1\n",
      "26a06716-2716-4ac2-9131-fa0f01f84526       1\n",
      "d26887c5-15f9-4b72-80dc-728039529e60       1\n",
      "Name: USERPLATFORMUID, Length: 53780, dtype: int64\n",
      "=======================\n",
      "++- USERCITY -++\n",
      "New York          4943\n",
      "Brooklyn          3237\n",
      "St Louis          2545\n",
      "Los Angeles       2154\n",
      "Houston           1747\n",
      "                  ... \n",
      "Foothill Ranch       1\n",
      "Henrietta            1\n",
      "Wartburg             1\n",
      "Winfield             1\n",
      "Gambier              1\n",
      "Name: USERCITY, Length: 4420, dtype: int64\n",
      "=======================\n",
      "++- USERZIPCODE -++\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "63169.0        2116\n",
      "11226.0        1752\n",
      "22202.0        1259\n",
      "10001.0        1190\n",
      "90060.0        1058\n",
      "               ... \n",
      "56748.0           1\n",
      "77021.0           1\n",
      "74880.0           1\n",
      "93630.0           1\n",
      "760865488.0       1\n",
      "Name: USERZIPCODE, Length: 11278, dtype: int64\n",
      "=======================\n",
      "++- PLATFORMTYPE -++\n",
      "Online Medical Journal      110120\n",
      "Online Learning Portal        2978\n",
      "Medical News Platform          798\n",
      "Electronic Health Record        40\n",
      "Tele-medicine Platform           1\n",
      "Name: PLATFORMTYPE, dtype: int64\n",
      "=======================\n",
      "++- URL -++\n",
      "https://www.clinicaladvisor.com/home/features/clinical-challenge/pain-tingling-and-limited-range-of-motion-after-a-cat-bite-to-the-finger/                                                                                                                                                                                                                                                                                                                                                                                   5310\n",
      "https://www.gastroenterologyadvisor.com/                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     4299\n",
      "https://www.cureus.com/articles/105482-diverticulitis-with-microperforation                                                                                                                                                                                                                                                                                                                                                                                                                                                  2838\n",
      "https://www.clinicaladvisor.com/home/dermatologic-look-alikes/red-rashes-on-the-lower-legs/2/                                                                                                                                                                                                                                                                                                                                                                                                                                2708\n",
      "https://www.rheumatologyadvisor.com/home/general-rheumatology/deep-dive-overview-of-liver-manifestations-in-rheumatic-diseases/                                                                                                                                                                                                                                                                                                                                                                                              2587\n",
      "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             ... \n",
      "https://www.infectiousdiseaseadvisor.com/home/topics/covid19/mixed-results-for-paxlovid-in-standard-risk-patients-with-covid-19/?utm_source=newsletter&utm_medium=email&utm_campaign=NWSLT_IDA_UPDT_062222_JM&hmEmail=kTgLcihhhDEW1GwCOOdG13UHIqcIj%2F6O&hmSubId=&NID=1316018294&elqTrackId=e3e7472aa24848f5b776f0dfa4cae5e5&elq=e6d2fec6b6d24ee89c79463b98493739&elqaid=353&elqat=1&elqCampaignId=194                                                                                                                          1\n",
      "https://www.clinicaladvisor.com/home/meeting-coverage/acaai-2022/early-atopic-dermatitis-modifies-the-risk-for-allergic-respiratory-disease/?utm_source=newsletter&utm_medium=email&utm_campaign=NWLTR_CAD_CONF_ACAAI_ASTEPRO_112222_RM&hmEmail=Ysc25hIpkOHdmgLIaUrt9Bhfr1jEQITVgqbP1HaB5lo=&sha256email=c372454efec279dee597fd62192657913f5a5108d9d7a648a2d518ed2f4abcad&hmSubId=&NID=1003876251&elqTrackId=031b0c57d3bc455eb8736ad8693830db&elq=db6bd4eba8354842aff4c502062d50df&elqaid=7476&elqat=1&elqCampaignId=6232       1\n",
      "https://www.rheumatologyadvisor.com/home/news/early-screening-and-management-of-cardiovascular-risk-factors-important-in-psoriatic-arthritis-and-ankylosing-spondylitis/?utm_source=google&utm_medium=ppc&utm_campaign=traffic-drivers-rheum&gclid=EAIaIQobChMIpf6cxoWQ_AIVAqvICh1JwA4XEAEYASAAEgKTN_D_BwE                                                                                                                                                                                                                      1\n",
      "https://allmedx.com/q/Flu vaccine lowers risk of stroke?utm_source=Email8271&utm_medium=email&utm_campaign=Email8271&em_tag=94264\u000b\u000b\u000b                                                                                                                                                                                                                                                                                                                                                                                            1\n",
      "https://radrounds.com/radiology-case-images-teaching-file/bilothorax/?fbclid=IwAR3Yw9WM9wiMWr1a1QPTfo4ktG3z5v7TJqCASTbHHi4jNub55m23gxSvsFg                                                                                                                                                                                                                                                                                                                                                                                      1\n",
      "Name: URL, Length: 5231, dtype: int64\n",
      "=======================\n",
      "++- KEYWORDS -++\n",
      "Family Practice|Drainage|Clinical|Dermatology|Ophthalmology|Medicine|Allergic|General|Diagnosis|Neurology|Hepatology|Health|Cardiovascular|Surgical|Psychiatry|Oral|Urology|Cardiology|Gastroenterology|Orthopedics|Erythema|Arthritis|Asthma|Cellulitis|Pediatrics|Surgery|False|Oncology|Rheumatology                                                                                                                                                                                                                                                                                                                                                                                                                                          5317\n",
      "Transplantation|Technology|Chronic|Neoplasms|Urology|Pancreas|Mental Health|Arthritis|Refractory|Myocardial Infarction|Aged|Cardiology|Oncology|Surgical|Balloon|Anesthesia|Synthetic|Hepatic Transplantation|Diagnosis|Neurology|Ophthalmology|Surgery|Total|Atrial Fibrillation|Overweight|Autoimmune|Atherectomy|Heart Defects|Neonatal|Psychological|Stomach|Metabolic|Hospitals|Graves|Esophagus|Liver|Oral|Rheumatology|Insomnia|Acquired|General|Induced|Gastroenterology|Psychiatry|Angioplasty|Cardiovascular|Child|Arrhythmia|Dermatology|Liver Transplantation|Anxiety Disorders|Delivery|Flu|Small|Resuscitation|Asthma|Pathology|Medicine|Health|Clinical|False|Biliary Tract|Lung|Sleep Disorders|Arrhythmias|Bariatric Surgery    4530\n",
      "Health|Male|Neurological Surgery|Otolaryngology|Preventive Medicine|Surgery|Analgesia|Neutrophils|Resuscitation|Dermatology|Neurology|General|Intravenous|Oral|Surgical|Trauma|Orthopedics|Medicine|Monocytes|Specialties|Preventive|Anastomosis|Physical Medicine & Rehabilitation|Rheumatology|Urology|Critical Care|General Practice|Abscess|Dentistry|Small|Gastroenterology|Cardiac|Technology|Operative|Large|Emergency Medicine|Rehabilitation|Drainage|False|Anesthesiology|Oncology|Ophthalmology|Pediatrics|Transplantation|Clinical|Diagnosis|Metabolic|Pathology|Total|Cardiology|Nuclear Medicine|Psychiatry                                                                                                                        2838\n",
      "Myopathies|Small|Rheumatoid|Psychiatry|Urology|General|False|Chronic|Induced|Total|Neurology|Arthralgia|Joints|Medicine|Health|Connective Tissue Diseases|Intrahepatic|Polyarthritis|Gastroenterology|Autoimmune Diseases|Liver Dysfunction|Liver Diseases|Arthritis|Controlled|Synthetic|Clinical|Metabolic|Cardiology|Rheumatology|Liver|Rheumatic Diseases|Genetic|Autoimmune|Dermatology|Oncology|Ophthalmology                                                                                                                                                                                                                                                                                                                              2741\n",
      "Hemoptysis|Oral|Rheumatology|Cardiovascular|Small|Rheumatoid|Oncology|Ophthalmology|Clinical|Cardiology|Hemorrhage|Health|Biopsy|Hepatology|Hypersensitivity|Refractory|False|Chronic|Psychiatry|Urology|Medicine|Neutrophils|Erythema|Asthma|Diagnosis|Neurology|Orthopedics|Pediatrics|General|Dermatology|Gastroenterology|Surgery|Hematuria|Disseminated Intravascular Coagulation|Arthritis                                                                                                                                                                                                                                                                                                                                                 2708\n",
      "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 ... \n",
      "small|hospitals|biliary tract|thrombocytopenia|advanced|rheumatology|hepatology|oncology|angioedema|thyroid carcinoma|physicians|cystic fibrosis|pulmonary fibrosis|induced|alagille syndrome|pulmonary arterial hypertension|neurology|new|pain|myelofibrosis|disease|large|liver|vasculitis|biomarkers|multiple sclerosis|neuromyelitis optica|diagnosis|health|gastroenterology|idiopathic pulmonary fibrosis|cardiology|intrahepatic|the|hematology|clinical|carcinoma|urology|hereditary|hypertension|endocrinology|                                                                                                                                                                                                                           1\n",
      "Medicine|Gastroenterology|Health|Total|Neurology|General|Dermatology|Oral|Psychiatry|Graves|Molecular|Oncology|Graves Disease|Clinical|Ophthalmology|Rheumatology|Urology|False|Cardiology                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          1\n",
      "biliary tract|rheumatology|stomach|psychiatry|oncology|general|total|pancreas|patient|ophthalmology|neurology|pain|disease|biologics|liver|medicine|gastroenterology|cardiology|the|controlled|hematology|clinical|dermatology|urology|esophagus|endocrinology|                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     1\n",
      "chronic kidney disease|nephrology|rheumatology|psychiatry|lupus|oncology|erectile dysfunction|nephrology|nephritis|general|ophthalmology|chronic|neurology|cardiovascular disease|pain|sleep disorders|disease|large|health|medicine|gastroenterology|cardiovascular|anemia|reproductive|the|clinical|hematology|premature|dermatology|arthritis|carcinoma|transplantation|urology|urinary tract infections|diabetes|hypertension|endocrinology|                                                                                                                                                                                                                                                                                                    1\n",
      "the|lung|educational|pneumonia|radiology|pulmonary fibrosis|dyspnea|technology|pleural effusion|                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    1\n",
      "Name: KEYWORDS, Length: 2460, dtype: int64\n",
      "=======================\n",
      "++- TAXONOMY -++\n",
      "2084P0800X    7930\n",
      "2084N0400X    6621\n",
      "207Q00000X    3997\n",
      "207R00000X    3190\n",
      "208000000X     837\n",
      "              ... \n",
      "163WC0400X       1\n",
      "246QM0706X       1\n",
      "1835N1003X       1\n",
      "207RI0008X       1\n",
      "207XX0801X       1\n",
      "Name: TAXONOMY, Length: 207, dtype: int64\n",
      "=======================\n",
      "++- IS_HCP -++\n",
      "0.0    79756\n",
      "1.0    34180\n",
      "Name: IS_HCP, dtype: int64\n",
      "=======================\n"
     ]
    }
   ],
   "source": [
    "for column in train_df.columns:\n",
    "    print(\"++- \"+column+\" -++\")\n",
    "    print(train_df[column].value_counts())\n",
    "    print('=======================')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking for null values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sum of Null Values:\n",
      "BIDREQUESTIP           0\n",
      "USERPLATFORMUID        4\n",
      "USERCITY            6359\n",
      "USERZIPCODE         4592\n",
      "PLATFORMTYPE           0\n",
      "URL                    0\n",
      "KEYWORDS               0\n",
      "TAXONOMY           81624\n",
      "IS_HCP                 1\n",
      "dtype: int64\n",
      "\n"
     ]
    }
   ],
   "source": [
    "null_sum = train_df.isnull().sum()\n",
    "\n",
    "print(\"Sum of Null Values:\")\n",
    "print(null_sum)\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking the relation between USERPLATFORMUID and BIDREQUESTIP for possible imputation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "59158     152.11.230.114\n",
      "80923      73.59.230.146\n",
      "99416     152.11.230.114\n",
      "102512     73.59.230.146\n",
      "Name: BIDREQUESTIP, dtype: object\n"
     ]
    }
   ],
   "source": [
    "missing_uid_rows = train_df[train_df['USERPLATFORMUID'].isnull()]\n",
    "bidrequestip_values = missing_uid_rows['BIDREQUESTIP']\n",
    "print(bidrequestip_values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dropping the rows with NA values in USERPLATFORMUID feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.dropna(subset=['USERPLATFORMUID'], inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imputing missing value in IS_HCP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "73790    47.147.27.131\n",
      "Name: BIDREQUESTIP, dtype: object\n"
     ]
    }
   ],
   "source": [
    "missing_uid_rows = train_df[train_df['IS_HCP'].isnull()]\n",
    "bidrequestip_values = missing_uid_rows['BIDREQUESTIP']\n",
    "print(bidrequestip_values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The missing value in IS_HCP has this BIDREQUESTIP, so decided to follow the other values at this BIDREQUESTIP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df['IS_HCP'] = train_df.groupby('USERPLATFORMUID')['IS_HCP'].ffill()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       IS_HCP   BIDREQUESTIP\n",
      "412       1.0  47.147.27.131\n",
      "4376      1.0  47.147.27.131\n",
      "17491     1.0  47.147.27.131\n",
      "73790     1.0  47.147.27.131\n"
     ]
    }
   ],
   "source": [
    "filtered_df = train_df[train_df['BIDREQUESTIP'] == '47.147.27.131']\n",
    "print(filtered_df[['IS_HCP', 'BIDREQUESTIP']])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Since IS_HCP is binary -->!float -->>>int"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "train_df['IS_HCP'] = train_df['IS_HCP'].apply(np.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sum of Null Values:\n",
      "BIDREQUESTIP           0\n",
      "USERPLATFORMUID        0\n",
      "USERCITY            6359\n",
      "USERZIPCODE         4592\n",
      "PLATFORMTYPE           0\n",
      "URL                    0\n",
      "KEYWORDS               0\n",
      "TAXONOMY           81624\n",
      "IS_HCP                 0\n",
      "dtype: int64\n",
      "\n"
     ]
    }
   ],
   "source": [
    "null_sum = train_df.isnull().sum()\n",
    "\n",
    "print(\"Sum of Null Values:\")\n",
    "print(null_sum)\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking for the possible realtion between USERZIPCODE and USERCITY for Imputation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5009      Plymouth\n",
      "12542      Detroit\n",
      "19351     Plymouth\n",
      "21258      Detroit\n",
      "29326      Detroit\n",
      "42271     Plymouth\n",
      "64092     Plymouth\n",
      "65229     Plymouth\n",
      "66293      Detroit\n",
      "70688      Detroit\n",
      "87318     Plymouth\n",
      "94143      Detroit\n",
      "94208     Plymouth\n",
      "112735         NaN\n",
      "Name: USERCITY, dtype: object\n"
     ]
    }
   ],
   "source": [
    "zipcode = 482012153\n",
    "usercity_values = train_df[train_df['USERZIPCODE'] == zipcode]['USERCITY']\n",
    "print(usercity_values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Performimg join operation to fill the NA values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df['USERCITY'] = train_df['USERCITY'].fillna(train_df.groupby('USERZIPCODE')['USERCITY'].transform('first'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Finding the Total unique no. of keywords in the kEYWORDS FEATURE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Separating the delimiter and storing it in word list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of word_list: 1613\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['Endotracheal Intubation',\n",
       " 'congenital heart defect',\n",
       " 'coronavirus infections',\n",
       " 'digestive system neoplasms',\n",
       " 'Age Groups',\n",
       " 'Lung Abscess',\n",
       " 'brentuximab vedotin',\n",
       " 'Thoracotomy',\n",
       " 'cardiac output',\n",
       " 'retinal artery occlusion',\n",
       " 'Neuropharmacology',\n",
       " 'olfactory mucosa',\n",
       " 'Transplantation',\n",
       " 'Leukopenia',\n",
       " 'abortion',\n",
       " 'body fluids',\n",
       " 'macrolides',\n",
       " 'the',\n",
       " 'experimental',\n",
       " 'difficulty concentrating',\n",
       " 'Technology',\n",
       " 'ventricular dysfunction',\n",
       " 'Nerve Transfer',\n",
       " 'amyotrophic lateral sclerosis',\n",
       " 'bevacizumab',\n",
       " 'macrophages',\n",
       " 'Arthralgia',\n",
       " 'intubation',\n",
       " 'nonvalvular atrial fibrillation',\n",
       " 'gonorrhea',\n",
       " 'burkitt lymphoma',\n",
       " 'infliximab',\n",
       " 'Necrolytic Migratory Erythema',\n",
       " 'polyneuropathy',\n",
       " 'eosinophilic',\n",
       " 'advanced',\n",
       " 'Status Asthmaticus',\n",
       " 'hand',\n",
       " 'Orthopedics',\n",
       " 'paronychia',\n",
       " 'family practice',\n",
       " 'kidney diseases',\n",
       " 'panitumumab',\n",
       " 'nausea',\n",
       " 'catheterization',\n",
       " 'post-traumatic',\n",
       " 'Dermatology',\n",
       " 'Cannulation',\n",
       " 'encephalomyelitis',\n",
       " 'Circadian Rhythm',\n",
       " 'Vascular Malformations',\n",
       " 'Health Education',\n",
       " 'tetralogy of fallot',\n",
       " 'Skeleton',\n",
       " 'Neonates',\n",
       " 'Multiple Sclerosis',\n",
       " 'sglt2',\n",
       " 'phlebitis',\n",
       " 'coinfection',\n",
       " 'noac',\n",
       " 'thoracic outlet syndrome',\n",
       " 'Allergic',\n",
       " 'Heart Transplantation',\n",
       " 'carney complex',\n",
       " 'Clinical Pathology',\n",
       " 'molecular pathology',\n",
       " 'acanthamoeba keratitis',\n",
       " 'middle ear',\n",
       " 'Neonatal',\n",
       " 'Medical Specialties',\n",
       " 'curettage',\n",
       " 'Pelvic Inflammatory Disease',\n",
       " 'Polyneuropathy',\n",
       " 'blastomycosis',\n",
       " 'muscle weakness',\n",
       " 'pregnancy',\n",
       " 'elective surgical procedures',\n",
       " 'arrhythmogenic right ventricular dysplasia',\n",
       " 'molecular',\n",
       " 'thyroiditis',\n",
       " 'Thalassemia',\n",
       " 'Nephritis',\n",
       " 'gastrointestinal tract',\n",
       " 'mild to moderate',\n",
       " 'genital herpes',\n",
       " 'Drug Therapy',\n",
       " 'hepatology',\n",
       " 'Neurodegenerative Diseases',\n",
       " 'Preventive Medicine',\n",
       " 'Catheterization',\n",
       " 'Bernard-Soulier Syndrome',\n",
       " 'tonsillitis',\n",
       " 'respiratory insufficiency',\n",
       " 'Malabsorption Syndromes',\n",
       " 'Neutrophils',\n",
       " 'ibuprofen',\n",
       " 'updated',\n",
       " 'bronchopulmonary sequestration',\n",
       " 'Diabetes Insipidus',\n",
       " 'hemopneumothorax',\n",
       " 'physical medicine & rehabilitation',\n",
       " 'skin disease',\n",
       " 'empyema',\n",
       " 'valacyclovir',\n",
       " 'Cysts',\n",
       " 'clavulanic acid',\n",
       " 'Intensive Care',\n",
       " 'internal medicine',\n",
       " 'Anemia',\n",
       " 'Early Intervention',\n",
       " 'Heart Defects',\n",
       " 'cardiac failure',\n",
       " 'Anticoagulant Drug',\n",
       " 'Periodontics',\n",
       " 'neurological surgery',\n",
       " 'bone marrow diseases',\n",
       " 'peritonsillar abscess',\n",
       " 'forensic medicine',\n",
       " 'pneumoconiosis',\n",
       " 'tracheomalacia',\n",
       " 'Anesthesiology',\n",
       " 'Peripheral Nervous System Diseases',\n",
       " 'Denervation',\n",
       " 'apnea',\n",
       " 'Blood Loss',\n",
       " 'embolism',\n",
       " 'early intervention',\n",
       " 'primary prevention',\n",
       " 'health personnel',\n",
       " 'hemophilia b',\n",
       " 'Surgical Intensive',\n",
       " 'large',\n",
       " 'renal impairment',\n",
       " 'Pharmacogenetics',\n",
       " 'restless legs syndrome',\n",
       " 'Atherectomy',\n",
       " 'myocarditis',\n",
       " 'cardiac tamponade',\n",
       " 'Autoimmune',\n",
       " 'chromosomal translocations',\n",
       " 'elderly',\n",
       " 'hemorrhagic fever',\n",
       " 'tramadol',\n",
       " 'child',\n",
       " 'Veterinary Medicine',\n",
       " 'behavioral sciences',\n",
       " 'severe eosinophilic asthma',\n",
       " 'Intrahepatic',\n",
       " 'Urology',\n",
       " 'nsaid',\n",
       " 'Myeloproliferative neoplasms',\n",
       " 'resistance training',\n",
       " 'Allergy Medications',\n",
       " 'hemoptysis',\n",
       " 'bacterial infections',\n",
       " 'Small Intestines',\n",
       " 'bronchiolitis obliterans',\n",
       " 'Clinical Pharmacology',\n",
       " 'vascular diseases',\n",
       " 'Flu',\n",
       " 'peritonitis',\n",
       " 'Sterilization',\n",
       " 'hepatopulmonary syndrome',\n",
       " 'pseudoxanthoma elasticum',\n",
       " 'mevalonate kinase deficiency',\n",
       " 'vibrio infections',\n",
       " 'Cerebrovascular Accidents',\n",
       " 'Rheumatology',\n",
       " 'nasal polyps',\n",
       " 'preventive medicine',\n",
       " 'Aplastic',\n",
       " 'laryngeal mask airway',\n",
       " 'denervation',\n",
       " 'Strokes',\n",
       " 'earache',\n",
       " 'tetanus',\n",
       " 'prostaglandins',\n",
       " 'Impulse Control',\n",
       " 'insomnia',\n",
       " 'viremia',\n",
       " 'aphonia',\n",
       " 'beta cells',\n",
       " 'Allergic Bronchopulmonary',\n",
       " 'dyspnea',\n",
       " 'Biopsy',\n",
       " 'lymphatic vessels',\n",
       " 'Metabolic',\n",
       " 'nervous system',\n",
       " 'metabolic diseases',\n",
       " 'acquired',\n",
       " 'Debridement',\n",
       " \"parkinson's disease\",\n",
       " 'cardiac',\n",
       " 'cardiomegaly',\n",
       " 'muscle pain',\n",
       " 'dapagliflozin',\n",
       " 'adaptation',\n",
       " 'superior vena cava syndrome',\n",
       " 'q fever',\n",
       " 'Renal Transplantation',\n",
       " 'Female',\n",
       " 'ophthalmology',\n",
       " 'Clinical',\n",
       " 'Dentistry',\n",
       " 'digestive tract',\n",
       " 'End-Of-Life Care',\n",
       " 'Physical Medicine & Rehabilitation',\n",
       " 'psittacosis',\n",
       " 'dna sequencing',\n",
       " 'Imperforate',\n",
       " 'Arthritis',\n",
       " 'airway obstruction',\n",
       " 'lymphogranuloma venereum',\n",
       " 'Hair Diseases',\n",
       " 'Lymphedema',\n",
       " 'leptospirosis',\n",
       " 'arrhythmia',\n",
       " 'Hemorrhage',\n",
       " 'metabolic',\n",
       " 'tinea',\n",
       " 'general hospitals',\n",
       " 'pulmonologists',\n",
       " 'granulomatous disease',\n",
       " 'dietetics',\n",
       " 'Connective Tissue Diseases',\n",
       " 'Thoracic Surgery',\n",
       " 'mycoplasma infections',\n",
       " 'Laser',\n",
       " 'Aneurysm',\n",
       " 'lymphatic diseases',\n",
       " 'Aspergillosis',\n",
       " 'orbital cellulitis',\n",
       " 'orthopedic',\n",
       " 'Resistance Training',\n",
       " 'hereditary',\n",
       " 'encephalopathy',\n",
       " 'cns',\n",
       " 'radiation pneumonitis',\n",
       " 'Decompression',\n",
       " 'mental competency',\n",
       " 'mohs surgery',\n",
       " 'sporotrichosis',\n",
       " 'arthralgia',\n",
       " 'whooping cough',\n",
       " 'Aquatic Therapy',\n",
       " 'Pre-Excitation Syndromes',\n",
       " 'iga vasculitis',\n",
       " 'Sinus',\n",
       " 'acute lung injury',\n",
       " 'Cryptorchidism',\n",
       " 'acid-base imbalance',\n",
       " 'hiv infections',\n",
       " 'chalazion',\n",
       " 'stroke',\n",
       " 'Primary Prevention',\n",
       " 'obstetrics & gynecology',\n",
       " 'hypospadias',\n",
       " 'Delivery',\n",
       " 'neonatology',\n",
       " 'thrombocytosis',\n",
       " 'hypersensitivity',\n",
       " 'spondylitis',\n",
       " 'Duodenum',\n",
       " 'conscious sedation',\n",
       " 'strongyloidiasis',\n",
       " 'Neoplasms by Site',\n",
       " 'surgical oncology',\n",
       " 'lung diseases',\n",
       " 'erythema',\n",
       " 'dengue',\n",
       " 'hypopharynx',\n",
       " 'Large',\n",
       " 'antibiotic prophylaxis',\n",
       " 'porphyrias',\n",
       " 'influenza',\n",
       " 'Gastrointestinal Diseases',\n",
       " 'Neurology',\n",
       " 'baloxavir marboxil',\n",
       " 'joint pain',\n",
       " 'pulmonary embolism',\n",
       " 'nasal mucosa',\n",
       " 'Liver',\n",
       " 'surgery',\n",
       " 'anthropometry',\n",
       " 'Nuclear Medicine',\n",
       " 'Small',\n",
       " 'intestines',\n",
       " 'malaria',\n",
       " 'Thyroid Carcinoma',\n",
       " 'sleep medicine',\n",
       " 'Jejunum',\n",
       " 'Bone Diseases',\n",
       " 'endocrine',\n",
       " 'Prostaglandins',\n",
       " 'reperfusion',\n",
       " 'Malnutrition',\n",
       " 'pneumonia',\n",
       " 'doctor',\n",
       " 'veterinary medicine',\n",
       " 'prednisone',\n",
       " 'Deep Sedation',\n",
       " 'invasive fungal infections',\n",
       " 'taeniasis',\n",
       " 'polyarthritis',\n",
       " 'keratoconjunctivitis',\n",
       " 'rheumatology',\n",
       " 'cefuroxime',\n",
       " 'gastrointestinal tracts',\n",
       " 'thymectomy',\n",
       " 'Skin Diseases',\n",
       " 'mycoses',\n",
       " 'Child Care',\n",
       " 'surgical',\n",
       " 'pharyngitis',\n",
       " 'post-exposure prophylaxis',\n",
       " 'sarcoma',\n",
       " 'fibrinogen',\n",
       " 'coronaviridae infections',\n",
       " 'Neuroendoscopy',\n",
       " 'neoplasms',\n",
       " 'antilymphocyte serum',\n",
       " 'non-steroidal anti-inflammatory drug',\n",
       " 'follicular lymphoma',\n",
       " 'fetal anomalies',\n",
       " 'Inborn',\n",
       " 'hematuria',\n",
       " 'Hepatology',\n",
       " 'alagille syndrome',\n",
       " 'palatine tonsil',\n",
       " 'Multiple Myeloma',\n",
       " 'pulmonary disease',\n",
       " 'Congenital Heart Defect',\n",
       " 'Tachycardia',\n",
       " 'fatigue',\n",
       " 'Neuropathologists',\n",
       " 'Insomnia',\n",
       " 'ketorolac',\n",
       " 'intensive care',\n",
       " 'Intestine',\n",
       " 'hepatitis c',\n",
       " 'Follicular Lymphoma',\n",
       " 'vertebral artery dissection',\n",
       " 'Premedication',\n",
       " 'activities of daily living',\n",
       " 'Pharynx',\n",
       " 'tularemia',\n",
       " 'retinal vein occlusion',\n",
       " 'aortic aneurysm',\n",
       " 'ipilimumab',\n",
       " 'maxillary sinus',\n",
       " 'pharmacist',\n",
       " 'croup',\n",
       " 'endophthalmitis',\n",
       " 'eliquis',\n",
       " 'pulmonary arterial hypertension',\n",
       " 'intratracheal',\n",
       " 'hormone-dependent',\n",
       " 'skin rash',\n",
       " 'persistent left superior vena cava',\n",
       " 'hematologic diseases',\n",
       " 'Intrinsic',\n",
       " 'Asthma',\n",
       " 'surgical pathology',\n",
       " 'protein s deficiency',\n",
       " 'venous thrombosis',\n",
       " 'oral anticoagulants',\n",
       " 'actinomycosis',\n",
       " 'Starvation',\n",
       " 'chemoprophylaxis',\n",
       " 'Organ Transplantation',\n",
       " 'tinea cruris',\n",
       " 'sleep disorders',\n",
       " 'babesiosis',\n",
       " 'Cardiac',\n",
       " 'cryptococcosis',\n",
       " 'technology',\n",
       " 'coronary heart disease',\n",
       " 'neuromyelitis optica',\n",
       " 'purple',\n",
       " 'latent tuberculosis',\n",
       " 'Lung Diseases',\n",
       " 'immunosuppressed',\n",
       " 'kidney transplantation',\n",
       " 'molluscum contagiosum',\n",
       " 'arthropathy',\n",
       " 'Molecular',\n",
       " 'myeloproliferative neoplasms',\n",
       " 'endurance training',\n",
       " 'triple therapy',\n",
       " 'Emergency Medicine',\n",
       " 'zostavax',\n",
       " 'activated protein c resistance',\n",
       " 'Thrombocytopenia',\n",
       " 'liver transplantation',\n",
       " 'Thyroid Cancer',\n",
       " 'cardiometabolic risk',\n",
       " 'emotions',\n",
       " 'light sensitivity',\n",
       " 'autoimmune',\n",
       " 'vte',\n",
       " 'tachypnea',\n",
       " 'Bradycardia',\n",
       " 'furunculosis',\n",
       " 'nasopharynx',\n",
       " 'plague',\n",
       " 'Motor Neuron Disease',\n",
       " 'salivary gland',\n",
       " 'Endocrine',\n",
       " 'Thymectomy',\n",
       " 'Protein C Deficiency',\n",
       " 'Epidural Analgesia',\n",
       " 'Bariatric Surgery',\n",
       " 'staphylococcus aureus',\n",
       " 'Gastrointestinal Tract',\n",
       " 'proton therapy',\n",
       " 'bacteremia',\n",
       " 'Embolism',\n",
       " 'pain',\n",
       " 'bone marrow',\n",
       " 'leukoencephalopathy',\n",
       " 'Intracranial Aneurysm',\n",
       " 'capillary leak syndrome',\n",
       " 'Elderly',\n",
       " 'Airway Management',\n",
       " 'clindamycin',\n",
       " 'Exsanguination',\n",
       " 'tingling',\n",
       " 'Hemothorax',\n",
       " 'total',\n",
       " 'educational',\n",
       " 'abatacept',\n",
       " 'Total',\n",
       " 'Atrial Fibrillation',\n",
       " 'Dissecting',\n",
       " 'ischemic heart disease',\n",
       " 'skin diseases',\n",
       " 'angiotensin',\n",
       " 'Analgesia',\n",
       " 'Neurosurgical Procedures',\n",
       " 'Curettage',\n",
       " 'automation',\n",
       " 'Lymphopenia',\n",
       " 'laser therapy',\n",
       " 'physicians',\n",
       " 'naproxen',\n",
       " 'thyrotoxicosis',\n",
       " 'granulomatosis with polyangiitis',\n",
       " 'specialties',\n",
       " 'Dietetics',\n",
       " 'cystitis',\n",
       " 'immunization',\n",
       " 'non-high-density lipoprotein cholesterol',\n",
       " 'Sports Medicine',\n",
       " 'skeleton',\n",
       " 'Apnea',\n",
       " 'painkiller',\n",
       " 'polycythemia',\n",
       " 'conduct disorders',\n",
       " 'Abscess',\n",
       " 'laser',\n",
       " 'obstetric',\n",
       " 'Obstetrical',\n",
       " 'Hypospadias',\n",
       " 'hepatitis',\n",
       " 'Dermatitis',\n",
       " 'conjunctivitis',\n",
       " 'covid-19',\n",
       " 'disease',\n",
       " 'granuloma annulare',\n",
       " 'medical oncology',\n",
       " 'acquired immunodeficiency syndrome',\n",
       " 'mental health',\n",
       " 'Lung',\n",
       " 'bariatric surgery',\n",
       " 'foot deformities',\n",
       " 'neurological disorders',\n",
       " 'biopsy',\n",
       " 'Liver Diseases',\n",
       " 'pe',\n",
       " 'debridement',\n",
       " 'Rheumatic Diseases',\n",
       " 'Specialties',\n",
       " 'anticoagulants',\n",
       " 'psychological',\n",
       " 'respiratory tract infections',\n",
       " 'cerebrovascular accident',\n",
       " 'immunity',\n",
       " 'acetaminophen',\n",
       " 'bartonella infections',\n",
       " 'osteomyelitis',\n",
       " 'small cell lung carcinoma',\n",
       " 'cefixime',\n",
       " 'Early Goal-Directed Therapy',\n",
       " 'Induced',\n",
       " 'back pain',\n",
       " 'pathology',\n",
       " 'pharynx',\n",
       " 'vocal cords',\n",
       " 'prion diseases',\n",
       " 'epidural',\n",
       " 'Polyarthritis',\n",
       " 'lung transplantation',\n",
       " 'chronic obstructive pulmonary disease',\n",
       " 'loss of appetite',\n",
       " 'Anesthesia',\n",
       " 'cidofovir',\n",
       " 'hypereosinophilic syndrome',\n",
       " 'congestive heart failure',\n",
       " 'Endocrine Cancers',\n",
       " 'drug discovery',\n",
       " 'impetigo',\n",
       " 'frontal sinus',\n",
       " 'keratosis',\n",
       " 'refractory',\n",
       " 'new',\n",
       " 'vocal cord paralysis',\n",
       " 'hemostatic disorders',\n",
       " 'tinea capitis',\n",
       " 'oseltamivir',\n",
       " 'levofloxacin',\n",
       " 'neurogenic',\n",
       " 'Temporomandibular Joint',\n",
       " 'therapeutic effects',\n",
       " 'rehabilitation',\n",
       " 'Limb Salvage',\n",
       " 'shingles',\n",
       " 'pericardial effusion',\n",
       " 'blood protein disorders',\n",
       " 'proteomics',\n",
       " 'epistaxis',\n",
       " 'eye infections',\n",
       " 'Aortic Aneurysm',\n",
       " 'Chemoprevention',\n",
       " 'pneumothorax',\n",
       " 'Graves',\n",
       " 'psoriasis',\n",
       " 'ramsay hunt syndrome',\n",
       " 'dentist',\n",
       " 'atrial flutter',\n",
       " 'thyroid carcinoma',\n",
       " 'mandibular advancement',\n",
       " 'synthetic',\n",
       " 'bacteriuria',\n",
       " 'mitral valve prolapse',\n",
       " 'heart defects',\n",
       " 'Antigens',\n",
       " 'diabetes mellitus, type 2',\n",
       " 'Hypotension',\n",
       " 'ureaplasma infections',\n",
       " 'Angioplasty',\n",
       " 'Antibiotic Prophylaxis',\n",
       " 'polycythemia vera',\n",
       " 'reticuloendotheliosis',\n",
       " 'Post-Traumatic',\n",
       " 'genetic',\n",
       " 'transplantation',\n",
       " 'rheumatoid arthritis',\n",
       " 'Cardiac Rehabilitation',\n",
       " 'Neurogenic',\n",
       " 'intracranial aneurysm',\n",
       " 'Cardiology',\n",
       " 'pneumonia, viral',\n",
       " 'hypoalbuminemia',\n",
       " 'balloon',\n",
       " 'vzv',\n",
       " 'measles',\n",
       " 'patient care',\n",
       " 'dyssomnias',\n",
       " 'drug combinations',\n",
       " 'otitis media',\n",
       " 'nephrology',\n",
       " 'Central Nervous System Infections',\n",
       " 'vocal cord dysfunction',\n",
       " 'vomiting',\n",
       " 'antibody deficiency',\n",
       " 'Vascular Diseases',\n",
       " 'sitagliptin',\n",
       " 'protein folding',\n",
       " 'left ventricular ejection fraction',\n",
       " 'herpangina',\n",
       " 'escherichia coli',\n",
       " 'Stomach',\n",
       " 'Myocardial Reperfusion',\n",
       " 'sinus',\n",
       " 'quadrivalent influenza vaccine',\n",
       " 'typhus',\n",
       " 'rhinitis',\n",
       " 'congenital abnormalities',\n",
       " 'gastroenterology',\n",
       " 'muscle relaxation',\n",
       " 'Sleep Medicine',\n",
       " 'immunoglobulin a',\n",
       " 'Phagocytes',\n",
       " 'vaccine-preventable diseases',\n",
       " 'arteriovenous malformations',\n",
       " 'Hypervitaminosis A',\n",
       " 'liver abscess',\n",
       " 'strokes',\n",
       " 'dermatitis',\n",
       " 'Secondary Prevention',\n",
       " 'Behavior',\n",
       " 'monocytes',\n",
       " 'monkeypox',\n",
       " 'orthodontics',\n",
       " 'Metabolic Diseases',\n",
       " 'Angioedema',\n",
       " 'diabetic retinopathy',\n",
       " 'herpes simplex',\n",
       " 'Macrophages',\n",
       " 'hyperglycemia',\n",
       " 'drainage',\n",
       " 'mepolizumab',\n",
       " 'glucose intolerance',\n",
       " 'salivary glands',\n",
       " 'disseminated intravascular coagulation',\n",
       " 'Anti-Inflammatory Agents',\n",
       " 'tract neoplasms',\n",
       " 'amebiasis',\n",
       " 'Dissection',\n",
       " 'Ablation Techniques',\n",
       " 'urogenital',\n",
       " 'multiple myeloma',\n",
       " 'dehydroepiandrosterone',\n",
       " 'mucocele',\n",
       " 'denosumab',\n",
       " 'rubella syndrome',\n",
       " 'heparin',\n",
       " 'cardiovascular',\n",
       " 'gfr',\n",
       " 'oligoclonal bands',\n",
       " 'flu vaccine',\n",
       " 'organ transplant',\n",
       " 'spinal cord ischemia',\n",
       " 'Myelodysplastic Syndromes',\n",
       " 'sports medicine',\n",
       " 'FIBRINOGEN',\n",
       " 'Lymphoid Tissue',\n",
       " 'herpes',\n",
       " 'endocrinology',\n",
       " 'erysipeloid',\n",
       " 'cardiac rehabilitation',\n",
       " 'forensic',\n",
       " 'operative',\n",
       " 'Human Development',\n",
       " 'antithrombin iii deficiency',\n",
       " 'herpes zoster ophthalmicus',\n",
       " 'filariasis',\n",
       " 'ruxolitinib',\n",
       " 'Pulmonary Medicine',\n",
       " 'herpes zoster virus',\n",
       " 'central nervous system diseases',\n",
       " 'varicella-zoster',\n",
       " 'Minimally Invasive Surgical Procedures',\n",
       " 'blistering',\n",
       " 'Aged',\n",
       " 'Nervous System',\n",
       " 'cough',\n",
       " 'impulse control',\n",
       " 'perennial',\n",
       " 'child care',\n",
       " 'congenital',\n",
       " 'Conduct Disorders',\n",
       " 'coccidioidomycosis',\n",
       " 'autoimmune diseases',\n",
       " 'erythrasma',\n",
       " 'Surgical',\n",
       " 'Cellulitis',\n",
       " 'Biliary Tract',\n",
       " 'glucagon secretion',\n",
       " 'pulmonary medicine',\n",
       " 'inflammation',\n",
       " 'genomics',\n",
       " 'infected',\n",
       " 'smoldering multiple myeloma',\n",
       " 'ecchymosis',\n",
       " 'uveitis',\n",
       " 'Conscious Sedation',\n",
       " 'eosinophilic granuloma',\n",
       " 'rubella',\n",
       " 'cysticercosis',\n",
       " 'Ustekinumab',\n",
       " 'Thyroid Diseases',\n",
       " 'soft tissue infections',\n",
       " 'Hematology & Oncology',\n",
       " 'tracheitis',\n",
       " 'superinfection',\n",
       " 'Health Personnel',\n",
       " 'epidural abscess',\n",
       " 'herpes zoster oticus',\n",
       " 'Eosinophilic Esophagitis',\n",
       " 'embolic stroke',\n",
       " 'Dermatomyositis',\n",
       " 'Obstetrics & Gynecology',\n",
       " 'rheumatoid',\n",
       " 'immune reconstitution inflammatory syndrome',\n",
       " 'Esophagus',\n",
       " 'intracranial hemorrhages',\n",
       " 'fasting plasma glucose',\n",
       " 'patient-controlled',\n",
       " 'hypoglycemia',\n",
       " 'atherosclerosis',\n",
       " 'Perfusion',\n",
       " 'hypotension',\n",
       " 'Immune Reconstitution Inflammatory Syndrome',\n",
       " 'Antithrombin III Deficiency',\n",
       " 'moderate pain',\n",
       " 'trastuzumab',\n",
       " 'idiopathic pulmonary fibrosis',\n",
       " 'immunoglobulin m',\n",
       " 'scabies',\n",
       " 'obstetrical',\n",
       " 'pleural effusion',\n",
       " 'shingrix',\n",
       " 'leukemia',\n",
       " 'common cold',\n",
       " 'Neonatology',\n",
       " 'bipolar and related disorders',\n",
       " 'laryngitis',\n",
       " 'asthma',\n",
       " 'cor triatriatum',\n",
       " 'endocrine cancers',\n",
       " 'Minimally Invasive Surgeries',\n",
       " 'botulism',\n",
       " 'Ankylosis',\n",
       " 'child health',\n",
       " 'Hematopoietic System',\n",
       " 'smallpox',\n",
       " 'Altitude Sickness',\n",
       " 'Arrhythmias',\n",
       " 'Anticoagulants',\n",
       " 'Respiratory Tract',\n",
       " 'diagnostic molecular',\n",
       " 'Birth Cohort',\n",
       " 'Pediatrics',\n",
       " 'lung',\n",
       " 'Anus',\n",
       " 'dendritic cells',\n",
       " 'meningitis',\n",
       " 'daclizumab',\n",
       " 'urology',\n",
       " 'tract diseases',\n",
       " 'Biliary Atresia',\n",
       " 'Biliary Fistula',\n",
       " 'thromboplastin',\n",
       " 'neurodegenerative diseases',\n",
       " 'virus diseases',\n",
       " 'dissecting',\n",
       " 'noonan syndrome',\n",
       " 'thrombosis',\n",
       " 'hemophilia a',\n",
       " 'formulation',\n",
       " 'Acid-Base Imbalance',\n",
       " 'ustekinumab',\n",
       " 'pain management',\n",
       " 'transplant',\n",
       " 'Neoplastic',\n",
       " 'rheumatic heart disease',\n",
       " 'Neonatal Intensive Care',\n",
       " 'high-dose trivalent influenza vaccine',\n",
       " 'child well-being',\n",
       " 'radiosurgery',\n",
       " 'plus',\n",
       " 'seasonal',\n",
       " 'Brugada Syndrome',\n",
       " 'venous insufficiency',\n",
       " 'candidemia',\n",
       " 'tabes dorsalis',\n",
       " 'varicose veins',\n",
       " 'Anxiety Disorders',\n",
       " 'Health',\n",
       " 'Laser Therapy',\n",
       " 'toothache',\n",
       " 'fanconi anemia',\n",
       " 'resuscitation',\n",
       " 'hearing disorders',\n",
       " 'Extravascular Lung Water',\n",
       " 'Keratosis',\n",
       " 'ascariasis',\n",
       " 'plastic surgery',\n",
       " 'preschool',\n",
       " 'clinical',\n",
       " 'Bursitis',\n",
       " 'general practitioners',\n",
       " 'cystic fibrosis',\n",
       " 'social withdrawal',\n",
       " 'diabetic kidney disease',\n",
       " 'vka',\n",
       " 'bradycardia',\n",
       " 'hepatitis e',\n",
       " 'Bone Marrow Failure Disorders',\n",
       " 'urinary tract infections',\n",
       " 'cholera',\n",
       " 'Gastrointestinal Tracts',\n",
       " \"alzheimer's disease\",\n",
       " 'glycated hemoglobin',\n",
       " 'respiratory tract neoplasms',\n",
       " 'respiratory tract',\n",
       " 'rabies',\n",
       " 'General Practitioners',\n",
       " 'enuresis',\n",
       " 'clinical pathology',\n",
       " 'overweight',\n",
       " 'aortic rupture',\n",
       " 'shock',\n",
       " 'chemoprevention',\n",
       " 'nocardia infections',\n",
       " 'wes',\n",
       " 'Hereditary',\n",
       " 'gram-negative bacterial infections',\n",
       " 'anthrax',\n",
       " 'nasal obstruction',\n",
       " 'ventricular outflow obstruction',\n",
       " 'bile ducts',\n",
       " 'Long QT Syndrome',\n",
       " 'starvation',\n",
       " 'ileum',\n",
       " 'varicella',\n",
       " 'lupus',\n",
       " 'myocardial infarction',\n",
       " 'temporomandibular joint',\n",
       " 'Refractory',\n",
       " 'geroscience',\n",
       " 'bone marrow transplant',\n",
       " 'connective tissue diseases',\n",
       " 'Dissociative Disorders',\n",
       " 'decompression',\n",
       " 'stomach',\n",
       " 'motor neuron disease',\n",
       " 'Sternotomy',\n",
       " 'critical care',\n",
       " 'Primary Immunodeficiency Diseases',\n",
       " 'itching',\n",
       " 'diabetes',\n",
       " 'Oncology',\n",
       " 'paraneoplastic syndromes',\n",
       " 'Heart Block',\n",
       " 'radiotherapy',\n",
       " 'analgesic',\n",
       " 'hamartoma',\n",
       " 'controlled',\n",
       " 'nuclear medicine',\n",
       " 'CNS Infections',\n",
       " 'escitalopram',\n",
       " 'oncology',\n",
       " 'covid',\n",
       " 'scurvy',\n",
       " 'Perinatology',\n",
       " 'Congestive Heart Failure',\n",
       " 'Adrenal Hyperplasia',\n",
       " 'laryngopharyngeal reflux',\n",
       " 'Overweight',\n",
       " 'General',\n",
       " 'persistent infection',\n",
       " 'rituximab',\n",
       " 'penciclovir',\n",
       " 'Psychiatric Rehabilitation',\n",
       " 'glucocorticoids',\n",
       " 'intrinsic',\n",
       " 'radiography',\n",
       " 'tablet',\n",
       " 'Ischemic Heart Disease',\n",
       " 'dermabrasion',\n",
       " 'Infected',\n",
       " 'newborn',\n",
       " 'Disseminated Intravascular Coagulation',\n",
       " 'hyperemia',\n",
       " 'dwarfism',\n",
       " 'Operative',\n",
       " 'thalassemia',\n",
       " 'Cardiomyopathy',\n",
       " 'renal transplantation',\n",
       " 'central nervous system infections',\n",
       " 'paracetamol',\n",
       " 'Therapeutic Effects',\n",
       " 'laryngeal muscles',\n",
       " 'immunoglobulin e',\n",
       " 'physiology',\n",
       " 'Maze Procedure',\n",
       " 'sickle cell trait',\n",
       " 'lyme disease',\n",
       " 'staphylococcal scalded skin syndrome',\n",
       " 'Self-Management',\n",
       " 'eosinophilic esophagitis',\n",
       " 'nasopharyngitis',\n",
       " 'urologic diseases',\n",
       " 'Resuscitation',\n",
       " 'adrenal hyperplasia',\n",
       " 'malnutrition',\n",
       " 'Granulomatous Disease',\n",
       " 'cardiotoxicity',\n",
       " 'autoantibodies',\n",
       " 'inhaled corticosteroids',\n",
       " 'Hematology',\n",
       " 'scrub typhus',\n",
       " 'infectious mononucleosis',\n",
       " 'hematology & oncology',\n",
       " 'neuropathology',\n",
       " 'glucose metabolism disorders',\n",
       " 'whole genome sequencing',\n",
       " 'pilonidal sinus',\n",
       " 'postherpetic neuralgia',\n",
       " 'herbal medicine',\n",
       " 'Pathology',\n",
       " 'Breathing Exercises',\n",
       " 'False',\n",
       " 'Muscle Denervation',\n",
       " 'Ruptured',\n",
       " 'Pulmonologists',\n",
       " 'bacterial infections and mycoses',\n",
       " 'translocation',\n",
       " 'celecoxib',\n",
       " 'biliary tract',\n",
       " 'aplastic',\n",
       " 'Lipomatosis',\n",
       " 'brain diseases',\n",
       " 'grandparents',\n",
       " 'critical care medicine',\n",
       " 'Antibody Deficiency',\n",
       " 'Fasciitis',\n",
       " 'vaginosis',\n",
       " 'psychiatry',\n",
       " 'Drainage',\n",
       " 'severe asthma',\n",
       " 'bisphosphonate-associated osteonecrosis of the jaw',\n",
       " 'ribavirin',\n",
       " 'blood pressure control',\n",
       " 'scarlet fever',\n",
       " 'Endodontics',\n",
       " 'weil disease',\n",
       " 'letrozole',\n",
       " 'Seizures',\n",
       " 'vasculitis',\n",
       " 'Earache',\n",
       " 'Heart Diseases',\n",
       " 'Congenital Abnormalities',\n",
       " 'alemtuzumab',\n",
       " 'infertility',\n",
       " 'respiratory distress syndrome',\n",
       " 'cheyne-stokes respiration',\n",
       " 'neoplasms by histologic type',\n",
       " 'Neurological Surgery',\n",
       " 'dermatology',\n",
       " 'hospitals',\n",
       " 'Cardiac Output',\n",
       " 'Urogenital',\n",
       " 'inotuzumab ozogamicin',\n",
       " 'azithromycin',\n",
       " 'Pregnancy Complications',\n",
       " 'severe acute respiratory syndrome',\n",
       " 'Methylprednisolone',\n",
       " 'immunoglobulin d',\n",
       " 'drug development',\n",
       " 'interventional cardiology',\n",
       " 'atrial fibrillation',\n",
       " 'neurosyphilis',\n",
       " 'Mental Processes',\n",
       " 'Infant',\n",
       " 'Dental',\n",
       " 'Health Services',\n",
       " 'elephantiasis',\n",
       " 'Protein S Deficiency',\n",
       " 'pelvic inflammatory disease',\n",
       " 'Surgical Pathology',\n",
       " 'redness',\n",
       " 'Shock',\n",
       " 'protein c deficiency',\n",
       " 'Intracranial Hemorrhages',\n",
       " 'preventive',\n",
       " 'pregnancy complications',\n",
       " 'Radiography',\n",
       " 'epiglottitis',\n",
       " 'skin infections',\n",
       " 'lung injury',\n",
       " 'Neoplasms',\n",
       " 'Hemolytic',\n",
       " 'Eye Infections',\n",
       " 'Synthetic',\n",
       " 'Heart Rupture',\n",
       " 'anesthesia',\n",
       " 'beta-Thalassemia',\n",
       " 'livedo reticularis',\n",
       " 'chronic limb-threatening ischemia',\n",
       " 'Medicine',\n",
       " 'rheumatic diseases',\n",
       " 'biliary atresia',\n",
       " 'defense mechanisms',\n",
       " 'birth cohort',\n",
       " 'patient',\n",
       " 'vaccination coverage',\n",
       " 'peripheral nervous system diseases',\n",
       " 'diarrhea',\n",
       " 'beta-thalassemia',\n",
       " 'Orthodontics',\n",
       " 'cardiovascular risk reduction',\n",
       " 'Liver Dysfunction',\n",
       " 'arrhythmias',\n",
       " 'Blunt Cardiac Injury',\n",
       " 'Ventricular Fibrillation',\n",
       " 'Keratoacanthoma',\n",
       " 'Geriatric Medicine',\n",
       " ...]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#getting all the words to list\n",
    "all_keywords = train_df['KEYWORDS'].str.cat(sep='|')\n",
    "word_list = list(set(all_keywords.split('|')))\n",
    "\n",
    "# Remove empty value from word_list\n",
    "word_list = list(filter(None, word_list))\n",
    "\n",
    "\n",
    "list_size = len(word_list)\n",
    "print(\"Size of word_list:\", list_size)\n",
    "word_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### One hot Encoding for the KEYWORDS column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "for word in word_list:\n",
    "    train_df[word] = train_df['KEYWORDS'].apply(lambda x: 1 if word in x else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BIDREQUESTIP</th>\n",
       "      <th>USERPLATFORMUID</th>\n",
       "      <th>USERCITY</th>\n",
       "      <th>USERZIPCODE</th>\n",
       "      <th>PLATFORMTYPE</th>\n",
       "      <th>URL</th>\n",
       "      <th>KEYWORDS</th>\n",
       "      <th>TAXONOMY</th>\n",
       "      <th>IS_HCP</th>\n",
       "      <th>Endotracheal Intubation</th>\n",
       "      <th>...</th>\n",
       "      <th>Biomarkers</th>\n",
       "      <th>epidural analgesia</th>\n",
       "      <th>gastroenteritis</th>\n",
       "      <th>coronary disease</th>\n",
       "      <th>h1n1 vaccine</th>\n",
       "      <th>hemothorax</th>\n",
       "      <th>hoarseness</th>\n",
       "      <th>Balloon</th>\n",
       "      <th>drug therapy</th>\n",
       "      <th>listeriosis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>170.173.0.22</td>\n",
       "      <td>6974dcaa-f932-480e-9fb5-c52e20e1393a</td>\n",
       "      <td>Portland</td>\n",
       "      <td>97206.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.cancertherapyadvisor.com/home/canc...</td>\n",
       "      <td>General|Endocrine|False|Medicine|Surgery|Urolo...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>65.216.253.25</td>\n",
       "      <td>c12f3f8f-8fcf-484a-90e1-1ac04db8cdcf</td>\n",
       "      <td>Arlington</td>\n",
       "      <td>22202.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.cancertherapyadvisor.com/home/deci...</td>\n",
       "      <td>Bone Marrow|Radiography|Chronic|Oncology|Psych...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>66.232.79.22</td>\n",
       "      <td>a698de4b-e200-46dd-b5fb-40402175ae18</td>\n",
       "      <td>New Meadows</td>\n",
       "      <td>83654.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.cancertherapyadvisor.com/home/canc...</td>\n",
       "      <td>General|Endocrine|False|Medicine|Surgery|Urolo...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>137.54.125.246</td>\n",
       "      <td>45967533-75c8-4fbd-a00c-e6ff20447aaa</td>\n",
       "      <td>None</td>\n",
       "      <td>229114624.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://globalrph.com/medcalcs/warfarin-mainte...</td>\n",
       "      <td>Dental|Total|Clinical|Pharmacology|Physicians|...</td>\n",
       "      <td>2084P0800X</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>174.202.231.99</td>\n",
       "      <td>a17e25be-532d-4cf5-b916-9308c8c3961f</td>\n",
       "      <td>Houston</td>\n",
       "      <td>77008.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.cureus.com/articles/58184-a-review...</td>\n",
       "      <td>Critical Care|Emergency Medicine|General Pract...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>24.59.163.238</td>\n",
       "      <td>3b644d8f-b225-4ea3-b370-160ff7858749</td>\n",
       "      <td>Syracuse</td>\n",
       "      <td>13215.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.cureus.com/articles/131022-the-imp...</td>\n",
       "      <td>small|covid|nephrology|radiology|preventive me...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>67.203.36.94</td>\n",
       "      <td>456631b8-ca59-4e7a-96a0-53bcbf85eaef</td>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>90060.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.gastroenterologyadvisor.com/</td>\n",
       "      <td>Transplantation|Technology|Chronic|Neoplasms|U...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>24.28.40.219</td>\n",
       "      <td>1e84b963-2a2a-4e1a-a767-33867aa89bd4</td>\n",
       "      <td>Dallas</td>\n",
       "      <td>75204.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.clinicaladvisor.com/home/features/...</td>\n",
       "      <td>Family Practice|Drainage|Clinical|Dermatology|...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>104.172.116.202</td>\n",
       "      <td>be61a8f7-0f00-487f-8a56-679c819fa01a</td>\n",
       "      <td>Granada Hills</td>\n",
       "      <td>91344.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.cancertherapyadvisor.com/home/deci...</td>\n",
       "      <td>Intensive Care|Shock|Ischemic Heart Disease|He...</td>\n",
       "      <td>2084P0804X</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>170.178.236.25</td>\n",
       "      <td>ddbea19a-fa49-4d3d-965a-bee29cbe92da</td>\n",
       "      <td>Davenport</td>\n",
       "      <td>52807.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://fpnotebook.com/Endo/Pharm/SgltInhbtr.htm</td>\n",
       "      <td>chronic kidney disease|nephrology|radiology|da...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 1622 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      BIDREQUESTIP                       USERPLATFORMUID       USERCITY  \\\n",
       "0     170.173.0.22  6974dcaa-f932-480e-9fb5-c52e20e1393a       Portland   \n",
       "1    65.216.253.25  c12f3f8f-8fcf-484a-90e1-1ac04db8cdcf      Arlington   \n",
       "2     66.232.79.22  a698de4b-e200-46dd-b5fb-40402175ae18    New Meadows   \n",
       "3   137.54.125.246  45967533-75c8-4fbd-a00c-e6ff20447aaa           None   \n",
       "4   174.202.231.99  a17e25be-532d-4cf5-b916-9308c8c3961f        Houston   \n",
       "5    24.59.163.238  3b644d8f-b225-4ea3-b370-160ff7858749       Syracuse   \n",
       "6     67.203.36.94  456631b8-ca59-4e7a-96a0-53bcbf85eaef    Los Angeles   \n",
       "7     24.28.40.219  1e84b963-2a2a-4e1a-a767-33867aa89bd4         Dallas   \n",
       "8  104.172.116.202  be61a8f7-0f00-487f-8a56-679c819fa01a  Granada Hills   \n",
       "9   170.178.236.25  ddbea19a-fa49-4d3d-965a-bee29cbe92da      Davenport   \n",
       "\n",
       "   USERZIPCODE            PLATFORMTYPE  \\\n",
       "0      97206.0  Online Medical Journal   \n",
       "1      22202.0  Online Medical Journal   \n",
       "2      83654.0  Online Medical Journal   \n",
       "3  229114624.0  Online Medical Journal   \n",
       "4      77008.0  Online Medical Journal   \n",
       "5      13215.0  Online Medical Journal   \n",
       "6      90060.0  Online Medical Journal   \n",
       "7      75204.0  Online Medical Journal   \n",
       "8      91344.0  Online Medical Journal   \n",
       "9      52807.0  Online Medical Journal   \n",
       "\n",
       "                                                 URL  \\\n",
       "0  https://www.cancertherapyadvisor.com/home/canc...   \n",
       "1  https://www.cancertherapyadvisor.com/home/deci...   \n",
       "2  https://www.cancertherapyadvisor.com/home/canc...   \n",
       "3  https://globalrph.com/medcalcs/warfarin-mainte...   \n",
       "4  https://www.cureus.com/articles/58184-a-review...   \n",
       "5  https://www.cureus.com/articles/131022-the-imp...   \n",
       "6           https://www.gastroenterologyadvisor.com/   \n",
       "7  https://www.clinicaladvisor.com/home/features/...   \n",
       "8  https://www.cancertherapyadvisor.com/home/deci...   \n",
       "9   https://fpnotebook.com/Endo/Pharm/SgltInhbtr.htm   \n",
       "\n",
       "                                            KEYWORDS    TAXONOMY  IS_HCP  \\\n",
       "0  General|Endocrine|False|Medicine|Surgery|Urolo...         NaN       0   \n",
       "1  Bone Marrow|Radiography|Chronic|Oncology|Psych...         NaN       0   \n",
       "2  General|Endocrine|False|Medicine|Surgery|Urolo...         NaN       0   \n",
       "3  Dental|Total|Clinical|Pharmacology|Physicians|...  2084P0800X       1   \n",
       "4  Critical Care|Emergency Medicine|General Pract...         NaN       0   \n",
       "5  small|covid|nephrology|radiology|preventive me...         NaN       0   \n",
       "6  Transplantation|Technology|Chronic|Neoplasms|U...         NaN       0   \n",
       "7  Family Practice|Drainage|Clinical|Dermatology|...         NaN       0   \n",
       "8  Intensive Care|Shock|Ischemic Heart Disease|He...  2084P0804X       1   \n",
       "9  chronic kidney disease|nephrology|radiology|da...         NaN       0   \n",
       "\n",
       "   Endotracheal Intubation  ...  Biomarkers  epidural analgesia  \\\n",
       "0                        0  ...           0                   0   \n",
       "1                        0  ...           0                   0   \n",
       "2                        0  ...           0                   0   \n",
       "3                        0  ...           0                   0   \n",
       "4                        0  ...           0                   0   \n",
       "5                        0  ...           0                   0   \n",
       "6                        0  ...           0                   0   \n",
       "7                        0  ...           0                   0   \n",
       "8                        0  ...           0                   0   \n",
       "9                        0  ...           0                   0   \n",
       "\n",
       "   gastroenteritis  coronary disease  h1n1 vaccine  hemothorax  hoarseness  \\\n",
       "0                0                 0             0           0           0   \n",
       "1                0                 0             0           0           0   \n",
       "2                0                 0             0           0           0   \n",
       "3                0                 0             0           0           0   \n",
       "4                0                 0             0           0           0   \n",
       "5                0                 0             0           0           0   \n",
       "6                0                 0             0           0           0   \n",
       "7                0                 0             0           0           0   \n",
       "8                0                 0             0           0           0   \n",
       "9                0                 0             0           0           0   \n",
       "\n",
       "   Balloon  drug therapy  listeriosis  \n",
       "0        0             0            0  \n",
       "1        0             0            0  \n",
       "2        0             0            0  \n",
       "3        0             0            0  \n",
       "4        0             0            0  \n",
       "5        0             0            0  \n",
       "6        1             0            0  \n",
       "7        0             0            0  \n",
       "8        0             0            0  \n",
       "9        0             0            0  \n",
       "\n",
       "[10 rows x 1622 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Performing One Hot Encoding on PLATFORMTYPE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          BIDREQUESTIP                       USERPLATFORMUID      USERCITY  \\\n",
      "0         170.173.0.22  6974dcaa-f932-480e-9fb5-c52e20e1393a      Portland   \n",
      "1        65.216.253.25  c12f3f8f-8fcf-484a-90e1-1ac04db8cdcf     Arlington   \n",
      "2         66.232.79.22  a698de4b-e200-46dd-b5fb-40402175ae18   New Meadows   \n",
      "3       137.54.125.246  45967533-75c8-4fbd-a00c-e6ff20447aaa          None   \n",
      "4       174.202.231.99  a17e25be-532d-4cf5-b916-9308c8c3961f       Houston   \n",
      "...                ...                                   ...           ...   \n",
      "113936  108.41.233.175  d26887c5-15f9-4b72-80dc-728039529e60  White Plains   \n",
      "59158              NaN                                   NaN           NaN   \n",
      "80923              NaN                                   NaN           NaN   \n",
      "99416              NaN                                   NaN           NaN   \n",
      "102512             NaN                                   NaN           NaN   \n",
      "\n",
      "        USERZIPCODE            PLATFORMTYPE  \\\n",
      "0           97206.0  Online Medical Journal   \n",
      "1           22202.0  Online Medical Journal   \n",
      "2           83654.0  Online Medical Journal   \n",
      "3       229114624.0  Online Medical Journal   \n",
      "4           77008.0  Online Medical Journal   \n",
      "...             ...                     ...   \n",
      "113936      10605.0  Online Medical Journal   \n",
      "59158           NaN                     NaN   \n",
      "80923           NaN                     NaN   \n",
      "99416           NaN                     NaN   \n",
      "102512          NaN                     NaN   \n",
      "\n",
      "                                                      URL  \\\n",
      "0       https://www.cancertherapyadvisor.com/home/canc...   \n",
      "1       https://www.cancertherapyadvisor.com/home/deci...   \n",
      "2       https://www.cancertherapyadvisor.com/home/canc...   \n",
      "3       https://globalrph.com/medcalcs/warfarin-mainte...   \n",
      "4       https://www.cureus.com/articles/58184-a-review...   \n",
      "...                                                   ...   \n",
      "113936                        https://www.medicalbag.com/   \n",
      "59158                                                 NaN   \n",
      "80923                                                 NaN   \n",
      "99416                                                 NaN   \n",
      "102512                                                NaN   \n",
      "\n",
      "                                                 KEYWORDS    TAXONOMY  IS_HCP  \\\n",
      "0       General|Endocrine|False|Medicine|Surgery|Urolo...         NaN     0.0   \n",
      "1       Bone Marrow|Radiography|Chronic|Oncology|Psych...         NaN     0.0   \n",
      "2       General|Endocrine|False|Medicine|Surgery|Urolo...         NaN     0.0   \n",
      "3       Dental|Total|Clinical|Pharmacology|Physicians|...  2084P0800X     1.0   \n",
      "4       Critical Care|Emergency Medicine|General Pract...         NaN     0.0   \n",
      "...                                                   ...         ...     ...   \n",
      "113936  Genetic|Large|Autoimmune|Health|False|Abortion...  2084P0800X     1.0   \n",
      "59158                                                 NaN         NaN     NaN   \n",
      "80923                                                 NaN         NaN     NaN   \n",
      "99416                                                 NaN         NaN     NaN   \n",
      "102512                                                NaN         NaN     NaN   \n",
      "\n",
      "        Endotracheal Intubation  ...  hemothorax  hoarseness  Balloon  \\\n",
      "0                           0.0  ...         0.0         0.0      0.0   \n",
      "1                           0.0  ...         0.0         0.0      0.0   \n",
      "2                           0.0  ...         0.0         0.0      0.0   \n",
      "3                           0.0  ...         0.0         0.0      0.0   \n",
      "4                           0.0  ...         0.0         0.0      0.0   \n",
      "...                         ...  ...         ...         ...      ...   \n",
      "113936                      0.0  ...         0.0         0.0      0.0   \n",
      "59158                       NaN  ...         NaN         NaN      NaN   \n",
      "80923                       NaN  ...         NaN         NaN      NaN   \n",
      "99416                       NaN  ...         NaN         NaN      NaN   \n",
      "102512                      NaN  ...         NaN         NaN      NaN   \n",
      "\n",
      "        drug therapy  listeriosis  Electronic Health Record  \\\n",
      "0                0.0          0.0                       0.0   \n",
      "1                0.0          0.0                       0.0   \n",
      "2                0.0          0.0                       0.0   \n",
      "3                0.0          0.0                       0.0   \n",
      "4                0.0          0.0                       0.0   \n",
      "...              ...          ...                       ...   \n",
      "113936           0.0          0.0                       NaN   \n",
      "59158            NaN          NaN                       0.0   \n",
      "80923            NaN          NaN                       0.0   \n",
      "99416            NaN          NaN                       0.0   \n",
      "102512           NaN          NaN                       0.0   \n",
      "\n",
      "        Medical News Platform  Online Learning Portal  Online Medical Journal  \\\n",
      "0                         0.0                     0.0                     1.0   \n",
      "1                         0.0                     0.0                     1.0   \n",
      "2                         0.0                     0.0                     1.0   \n",
      "3                         0.0                     0.0                     1.0   \n",
      "4                         0.0                     0.0                     1.0   \n",
      "...                       ...                     ...                     ...   \n",
      "113936                    NaN                     NaN                     NaN   \n",
      "59158                     0.0                     0.0                     1.0   \n",
      "80923                     0.0                     0.0                     1.0   \n",
      "99416                     0.0                     0.0                     1.0   \n",
      "102512                    0.0                     0.0                     1.0   \n",
      "\n",
      "        Tele-medicine Platform  \n",
      "0                          0.0  \n",
      "1                          0.0  \n",
      "2                          0.0  \n",
      "3                          0.0  \n",
      "4                          0.0  \n",
      "...                        ...  \n",
      "113936                     NaN  \n",
      "59158                      0.0  \n",
      "80923                      0.0  \n",
      "99416                      0.0  \n",
      "102512                     0.0  \n",
      "\n",
      "[113937 rows x 1627 columns]\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "# One-hot encoding\n",
    "encoder = OneHotEncoder(sparse=False)\n",
    "platform_type = train_df['PLATFORMTYPE'].values.reshape(-1, 1)\n",
    "encoded_features = encoder.fit_transform(platform_type)\n",
    "encoded_df = pd.DataFrame(encoded_features, columns=encoder.categories_[0])\n",
    "\n",
    "# Concatenate encoded DataFrame with the original DataFrame\n",
    "df_encoded = pd.concat([train_df, encoded_df], axis=1)\n",
    "\n",
    "# Print the encoded DataFrame\n",
    "print(df_encoded)\n",
    "train_df = df_encoded"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dropping the original as Encoded features are appended to the dataframe."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.drop('PLATFORMTYPE', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(113937, 1626)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['BIDREQUESTIP', 'USERPLATFORMUID', 'USERCITY', 'USERZIPCODE', 'URL',\n",
       "       'KEYWORDS', 'TAXONOMY', 'IS_HCP', 'Endotracheal Intubation',\n",
       "       'congenital heart defect',\n",
       "       ...\n",
       "       'hemothorax', 'hoarseness', 'Balloon', 'drug therapy', 'listeriosis',\n",
       "       'Electronic Health Record', 'Medical News Platform',\n",
       "       'Online Learning Portal', 'Online Medical Journal',\n",
       "       'Tele-medicine Platform'],\n",
       "      dtype='object', length=1626)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5929"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df['USERCITY'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4596"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df['USERZIPCODE'].isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dropping the rows with null USERCITY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.dropna(subset=['USERCITY'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(108008, 1626)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dropping the rows with null USERZIPCODE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.dropna(subset=['USERZIPCODE'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(108006, 1626)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_to_drop = ['BIDREQUESTIP', 'USERPLATFORMUID','URL','KEYWORDS']\n",
    "train_df.drop(columns_to_drop, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>USERCITY</th>\n",
       "      <th>USERZIPCODE</th>\n",
       "      <th>TAXONOMY</th>\n",
       "      <th>IS_HCP</th>\n",
       "      <th>Endotracheal Intubation</th>\n",
       "      <th>congenital heart defect</th>\n",
       "      <th>coronavirus infections</th>\n",
       "      <th>digestive system neoplasms</th>\n",
       "      <th>Age Groups</th>\n",
       "      <th>Lung Abscess</th>\n",
       "      <th>...</th>\n",
       "      <th>hemothorax</th>\n",
       "      <th>hoarseness</th>\n",
       "      <th>Balloon</th>\n",
       "      <th>drug therapy</th>\n",
       "      <th>listeriosis</th>\n",
       "      <th>Electronic Health Record</th>\n",
       "      <th>Medical News Platform</th>\n",
       "      <th>Online Learning Portal</th>\n",
       "      <th>Online Medical Journal</th>\n",
       "      <th>Tele-medicine Platform</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Portland</td>\n",
       "      <td>97206.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Arlington</td>\n",
       "      <td>22202.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>New Meadows</td>\n",
       "      <td>83654.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Houston</td>\n",
       "      <td>77008.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Syracuse</td>\n",
       "      <td>13215.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1622 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      USERCITY  USERZIPCODE TAXONOMY  IS_HCP  Endotracheal Intubation  \\\n",
       "0     Portland      97206.0      NaN     0.0                      0.0   \n",
       "1    Arlington      22202.0      NaN     0.0                      0.0   \n",
       "2  New Meadows      83654.0      NaN     0.0                      0.0   \n",
       "4      Houston      77008.0      NaN     0.0                      0.0   \n",
       "5     Syracuse      13215.0      NaN     0.0                      0.0   \n",
       "\n",
       "   congenital heart defect  coronavirus infections  \\\n",
       "0                      0.0                     0.0   \n",
       "1                      0.0                     0.0   \n",
       "2                      0.0                     0.0   \n",
       "4                      0.0                     0.0   \n",
       "5                      0.0                     0.0   \n",
       "\n",
       "   digestive system neoplasms  Age Groups  Lung Abscess  ...  hemothorax  \\\n",
       "0                         0.0         0.0           0.0  ...         0.0   \n",
       "1                         0.0         0.0           0.0  ...         0.0   \n",
       "2                         0.0         0.0           0.0  ...         0.0   \n",
       "4                         0.0         0.0           0.0  ...         0.0   \n",
       "5                         0.0         0.0           0.0  ...         0.0   \n",
       "\n",
       "   hoarseness  Balloon  drug therapy  listeriosis  Electronic Health Record  \\\n",
       "0         0.0      0.0           0.0          0.0                       0.0   \n",
       "1         0.0      0.0           0.0          0.0                       0.0   \n",
       "2         0.0      0.0           0.0          0.0                       0.0   \n",
       "4         0.0      0.0           0.0          0.0                       0.0   \n",
       "5         0.0      0.0           0.0          0.0                       0.0   \n",
       "\n",
       "   Medical News Platform  Online Learning Portal  Online Medical Journal  \\\n",
       "0                    0.0                     0.0                     1.0   \n",
       "1                    0.0                     0.0                     1.0   \n",
       "2                    0.0                     0.0                     1.0   \n",
       "4                    0.0                     0.0                     1.0   \n",
       "5                    0.0                     0.0                     1.0   \n",
       "\n",
       "   Tele-medicine Platform  \n",
       "0                     0.0  \n",
       "1                     0.0  \n",
       "2                     0.0  \n",
       "4                     0.0  \n",
       "5                     0.0  \n",
       "\n",
       "[5 rows x 1622 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Performing Label Encoding on the USERCITY AND USERZIPCODE column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>USERCITY</th>\n",
       "      <th>USERZIPCODE</th>\n",
       "      <th>TAXONOMY</th>\n",
       "      <th>IS_HCP</th>\n",
       "      <th>Endotracheal Intubation</th>\n",
       "      <th>congenital heart defect</th>\n",
       "      <th>coronavirus infections</th>\n",
       "      <th>digestive system neoplasms</th>\n",
       "      <th>Age Groups</th>\n",
       "      <th>Lung Abscess</th>\n",
       "      <th>...</th>\n",
       "      <th>hemothorax</th>\n",
       "      <th>hoarseness</th>\n",
       "      <th>Balloon</th>\n",
       "      <th>drug therapy</th>\n",
       "      <th>listeriosis</th>\n",
       "      <th>Electronic Health Record</th>\n",
       "      <th>Medical News Platform</th>\n",
       "      <th>Online Learning Portal</th>\n",
       "      <th>Online Medical Journal</th>\n",
       "      <th>Tele-medicine Platform</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3176</td>\n",
       "      <td>8934</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>130</td>\n",
       "      <td>2207</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2700</td>\n",
       "      <td>7585</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1766</td>\n",
       "      <td>7018</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3877</td>\n",
       "      <td>1351</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1622 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   USERCITY  USERZIPCODE TAXONOMY  IS_HCP  Endotracheal Intubation  \\\n",
       "0      3176         8934      NaN     0.0                      0.0   \n",
       "1       130         2207      NaN     0.0                      0.0   \n",
       "2      2700         7585      NaN     0.0                      0.0   \n",
       "4      1766         7018      NaN     0.0                      0.0   \n",
       "5      3877         1351      NaN     0.0                      0.0   \n",
       "\n",
       "   congenital heart defect  coronavirus infections  \\\n",
       "0                      0.0                     0.0   \n",
       "1                      0.0                     0.0   \n",
       "2                      0.0                     0.0   \n",
       "4                      0.0                     0.0   \n",
       "5                      0.0                     0.0   \n",
       "\n",
       "   digestive system neoplasms  Age Groups  Lung Abscess  ...  hemothorax  \\\n",
       "0                         0.0         0.0           0.0  ...         0.0   \n",
       "1                         0.0         0.0           0.0  ...         0.0   \n",
       "2                         0.0         0.0           0.0  ...         0.0   \n",
       "4                         0.0         0.0           0.0  ...         0.0   \n",
       "5                         0.0         0.0           0.0  ...         0.0   \n",
       "\n",
       "   hoarseness  Balloon  drug therapy  listeriosis  Electronic Health Record  \\\n",
       "0         0.0      0.0           0.0          0.0                       0.0   \n",
       "1         0.0      0.0           0.0          0.0                       0.0   \n",
       "2         0.0      0.0           0.0          0.0                       0.0   \n",
       "4         0.0      0.0           0.0          0.0                       0.0   \n",
       "5         0.0      0.0           0.0          0.0                       0.0   \n",
       "\n",
       "   Medical News Platform  Online Learning Portal  Online Medical Journal  \\\n",
       "0                    0.0                     0.0                     1.0   \n",
       "1                    0.0                     0.0                     1.0   \n",
       "2                    0.0                     0.0                     1.0   \n",
       "4                    0.0                     0.0                     1.0   \n",
       "5                    0.0                     0.0                     1.0   \n",
       "\n",
       "   Tele-medicine Platform  \n",
       "0                     0.0  \n",
       "1                     0.0  \n",
       "2                     0.0  \n",
       "4                     0.0  \n",
       "5                     0.0  \n",
       "\n",
       "[5 rows x 1622 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "city_encoder = LabelEncoder()\n",
    "train_df['USERCITY'] = city_encoder.fit_transform(train_df['USERCITY'])\n",
    "\n",
    "# Apply label encoding to the 'USERZIPCODE' column\n",
    "zipcode_encoder = LabelEncoder()\n",
    "train_df['USERZIPCODE'] = zipcode_encoder.fit_transform(train_df['USERZIPCODE'])\n",
    "\n",
    "# Print the updated DataFrame\n",
    "train_df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Performing the Same operations on Test Dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import OneHotEncoder, LabelEncoder\n",
    "\n",
    "# Define columns to drop\n",
    "columns_to_drop = ['DEVICETYPE', 'PLATFORM_ID', 'CHANNELTYPE', 'USERAGENT',\n",
    "                   'BIDREQUESTIP', 'USERPLATFORMUID', 'URL']\n",
    "\n",
    "# Drop specified columns\n",
    "test_df.drop(columns=columns_to_drop, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       ID  USERCITY  USERZIPCODE  Endotracheal Intubation  \\\n",
      "0  115501       857         1827                        0   \n",
      "1  115502       559         1091                        0   \n",
      "2  115503       592         5481                        0   \n",
      "3  115504       323          739                        0   \n",
      "4  115505      1609         2008                        0   \n",
      "\n",
      "   congenital heart defect  coronavirus infections  \\\n",
      "0                        0                       0   \n",
      "1                        0                       0   \n",
      "2                        0                       0   \n",
      "3                        0                       0   \n",
      "4                        0                       0   \n",
      "\n",
      "   digestive system neoplasms  Age Groups  Lung Abscess  Thoracotomy  ...  \\\n",
      "0                           0           0             0            0  ...   \n",
      "1                           0           0             0            0  ...   \n",
      "2                           0           0             0            0  ...   \n",
      "3                           0           0             0            0  ...   \n",
      "4                           0           0             0            0  ...   \n",
      "\n",
      "   coronary disease  hemothorax  hoarseness  Balloon  drug therapy  \\\n",
      "0                 0           0           0        0             0   \n",
      "1                 0           0           0        0             0   \n",
      "2                 0           0           0        0             0   \n",
      "3                 0           0           0        0             0   \n",
      "4                 0           0           0        0             0   \n",
      "\n",
      "   listeriosis  Electronic Health Record  Medical News Platform  \\\n",
      "0            0                       0.0                    0.0   \n",
      "1            0                       0.0                    0.0   \n",
      "2            0                       0.0                    0.0   \n",
      "3            0                       0.0                    0.0   \n",
      "4            0                       0.0                    0.0   \n",
      "\n",
      "   Online Learning Portal  Online Medical Journal  \n",
      "0                     0.0                     1.0  \n",
      "1                     0.0                     1.0  \n",
      "2                     0.0                     1.0  \n",
      "3                     0.0                     1.0  \n",
      "4                     0.0                     1.0  \n",
      "\n",
      "[5 rows x 1433 columns]\n"
     ]
    }
   ],
   "source": [
    "test_df['USERCITY'] = test_df['USERCITY'].fillna(test_df.groupby('USERZIPCODE')['USERCITY'].transform('first'))\n",
    "\n",
    "\n",
    "all_keywords = test_df['KEYWORDS'].str.cat(sep='|')\n",
    "word_list = list(set(all_keywords.split('|')))\n",
    "\n",
    "\n",
    "word_list = list(filter(None, word_list))\n",
    "\n",
    "\n",
    "for word in word_list:\n",
    "    test_df[word] = test_df['KEYWORDS'].apply(lambda x: 1 if word in x else 0)\n",
    "\n",
    "\n",
    "columns_to_drop = ['KEYWORDS']\n",
    "test_df.drop(columns_to_drop, axis=1, inplace=True)\n",
    "\n",
    "\n",
    "encoder = OneHotEncoder(sparse=False)\n",
    "platform_type = test_df['PLATFORMTYPE'].values.reshape(-1, 1)\n",
    "encoded_features = encoder.fit_transform(platform_type)\n",
    "encoded_df = pd.DataFrame(encoded_features, columns=encoder.categories_[0])\n",
    "\n",
    "\n",
    "test_df = pd.concat([test_df, encoded_df], axis=1)\n",
    "\n",
    "\n",
    "test_df.drop('PLATFORMTYPE', axis=1, inplace=True)\n",
    "\n",
    "\n",
    "test_df.dropna(subset=['USERCITY', 'USERZIPCODE'], inplace=True)\n",
    "\n",
    "\n",
    "city_encoder = LabelEncoder()\n",
    "test_df['USERCITY'] = city_encoder.fit_transform(test_df['USERCITY'])\n",
    "\n",
    "zipcode_encoder = LabelEncoder()\n",
    "test_df['USERZIPCODE'] = zipcode_encoder.fit_transform(test_df['USERZIPCODE'])\n",
    "\n",
    "print(test_df.head(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>USERCITY</th>\n",
       "      <th>USERZIPCODE</th>\n",
       "      <th>Endotracheal Intubation</th>\n",
       "      <th>congenital heart defect</th>\n",
       "      <th>coronavirus infections</th>\n",
       "      <th>digestive system neoplasms</th>\n",
       "      <th>Age Groups</th>\n",
       "      <th>Lung Abscess</th>\n",
       "      <th>Thoracotomy</th>\n",
       "      <th>...</th>\n",
       "      <th>coronary disease</th>\n",
       "      <th>hemothorax</th>\n",
       "      <th>hoarseness</th>\n",
       "      <th>Balloon</th>\n",
       "      <th>drug therapy</th>\n",
       "      <th>listeriosis</th>\n",
       "      <th>Electronic Health Record</th>\n",
       "      <th>Medical News Platform</th>\n",
       "      <th>Online Learning Portal</th>\n",
       "      <th>Online Medical Journal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>115501</td>\n",
       "      <td>857</td>\n",
       "      <td>1827</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>115502</td>\n",
       "      <td>559</td>\n",
       "      <td>1091</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>115503</td>\n",
       "      <td>592</td>\n",
       "      <td>5481</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>115504</td>\n",
       "      <td>323</td>\n",
       "      <td>739</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>115505</td>\n",
       "      <td>1609</td>\n",
       "      <td>2008</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28487</th>\n",
       "      <td>143988</td>\n",
       "      <td>2436</td>\n",
       "      <td>5898</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28488</th>\n",
       "      <td>143989</td>\n",
       "      <td>323</td>\n",
       "      <td>730</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28489</th>\n",
       "      <td>143990</td>\n",
       "      <td>1693</td>\n",
       "      <td>2340</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28490</th>\n",
       "      <td>143991</td>\n",
       "      <td>903</td>\n",
       "      <td>2355</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28492</th>\n",
       "      <td>143993</td>\n",
       "      <td>2987</td>\n",
       "      <td>1844</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>27008 rows × 1433 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           ID  USERCITY  USERZIPCODE  Endotracheal Intubation  \\\n",
       "0      115501       857         1827                        0   \n",
       "1      115502       559         1091                        0   \n",
       "2      115503       592         5481                        0   \n",
       "3      115504       323          739                        0   \n",
       "4      115505      1609         2008                        0   \n",
       "...       ...       ...          ...                      ...   \n",
       "28487  143988      2436         5898                        0   \n",
       "28488  143989       323          730                        0   \n",
       "28489  143990      1693         2340                        0   \n",
       "28490  143991       903         2355                        0   \n",
       "28492  143993      2987         1844                        0   \n",
       "\n",
       "       congenital heart defect  coronavirus infections  \\\n",
       "0                            0                       0   \n",
       "1                            0                       0   \n",
       "2                            0                       0   \n",
       "3                            0                       0   \n",
       "4                            0                       0   \n",
       "...                        ...                     ...   \n",
       "28487                        0                       0   \n",
       "28488                        0                       0   \n",
       "28489                        0                       0   \n",
       "28490                        0                       0   \n",
       "28492                        0                       0   \n",
       "\n",
       "       digestive system neoplasms  Age Groups  Lung Abscess  Thoracotomy  ...  \\\n",
       "0                               0           0             0            0  ...   \n",
       "1                               0           0             0            0  ...   \n",
       "2                               0           0             0            0  ...   \n",
       "3                               0           0             0            0  ...   \n",
       "4                               0           0             0            0  ...   \n",
       "...                           ...         ...           ...          ...  ...   \n",
       "28487                           0           0             0            0  ...   \n",
       "28488                           0           0             0            0  ...   \n",
       "28489                           0           0             0            0  ...   \n",
       "28490                           0           0             0            0  ...   \n",
       "28492                           0           0             0            0  ...   \n",
       "\n",
       "       coronary disease  hemothorax  hoarseness  Balloon  drug therapy  \\\n",
       "0                     0           0           0        0             0   \n",
       "1                     0           0           0        0             0   \n",
       "2                     0           0           0        0             0   \n",
       "3                     0           0           0        0             0   \n",
       "4                     0           0           0        0             0   \n",
       "...                 ...         ...         ...      ...           ...   \n",
       "28487                 0           0           0        0             0   \n",
       "28488                 0           0           0        0             0   \n",
       "28489                 0           0           0        0             0   \n",
       "28490                 0           0           0        0             0   \n",
       "28492                 0           0           0        0             0   \n",
       "\n",
       "       listeriosis  Electronic Health Record  Medical News Platform  \\\n",
       "0                0                       0.0                    0.0   \n",
       "1                0                       0.0                    0.0   \n",
       "2                0                       0.0                    0.0   \n",
       "3                0                       0.0                    0.0   \n",
       "4                0                       0.0                    0.0   \n",
       "...            ...                       ...                    ...   \n",
       "28487            0                       0.0                    0.0   \n",
       "28488            0                       0.0                    0.0   \n",
       "28489            0                       0.0                    0.0   \n",
       "28490            0                       0.0                    0.0   \n",
       "28492            0                       0.0                    0.0   \n",
       "\n",
       "       Online Learning Portal  Online Medical Journal  \n",
       "0                         0.0                     1.0  \n",
       "1                         0.0                     1.0  \n",
       "2                         0.0                     1.0  \n",
       "3                         0.0                     1.0  \n",
       "4                         0.0                     1.0  \n",
       "...                       ...                     ...  \n",
       "28487                     0.0                     1.0  \n",
       "28488                     0.0                     1.0  \n",
       "28489                     0.0                     1.0  \n",
       "28490                     0.0                     1.0  \n",
       "28492                     0.0                     1.0  \n",
       "\n",
       "[27008 rows x 1433 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>USERCITY</th>\n",
       "      <th>USERZIPCODE</th>\n",
       "      <th>TAXONOMY</th>\n",
       "      <th>IS_HCP</th>\n",
       "      <th>Endotracheal Intubation</th>\n",
       "      <th>congenital heart defect</th>\n",
       "      <th>coronavirus infections</th>\n",
       "      <th>digestive system neoplasms</th>\n",
       "      <th>Age Groups</th>\n",
       "      <th>Lung Abscess</th>\n",
       "      <th>...</th>\n",
       "      <th>hemothorax</th>\n",
       "      <th>hoarseness</th>\n",
       "      <th>Balloon</th>\n",
       "      <th>drug therapy</th>\n",
       "      <th>listeriosis</th>\n",
       "      <th>Electronic Health Record</th>\n",
       "      <th>Medical News Platform</th>\n",
       "      <th>Online Learning Portal</th>\n",
       "      <th>Online Medical Journal</th>\n",
       "      <th>Tele-medicine Platform</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3176</td>\n",
       "      <td>8934</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>130</td>\n",
       "      <td>2207</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2700</td>\n",
       "      <td>7585</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1766</td>\n",
       "      <td>7018</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3877</td>\n",
       "      <td>1351</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113932</th>\n",
       "      <td>3066</td>\n",
       "      <td>1837</td>\n",
       "      <td>207RP1001X</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113933</th>\n",
       "      <td>4042</td>\n",
       "      <td>10318</td>\n",
       "      <td>2084P0800X</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113934</th>\n",
       "      <td>3569</td>\n",
       "      <td>9052</td>\n",
       "      <td>208000000X</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113935</th>\n",
       "      <td>4321</td>\n",
       "      <td>1932</td>\n",
       "      <td>2084P0800X</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113936</th>\n",
       "      <td>4278</td>\n",
       "      <td>994</td>\n",
       "      <td>2084P0800X</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>108006 rows × 1622 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        USERCITY  USERZIPCODE    TAXONOMY  IS_HCP  Endotracheal Intubation  \\\n",
       "0           3176         8934         NaN     0.0                      0.0   \n",
       "1            130         2207         NaN     0.0                      0.0   \n",
       "2           2700         7585         NaN     0.0                      0.0   \n",
       "4           1766         7018         NaN     0.0                      0.0   \n",
       "5           3877         1351         NaN     0.0                      0.0   \n",
       "...          ...          ...         ...     ...                      ...   \n",
       "113932      3066         1837  207RP1001X     1.0                      0.0   \n",
       "113933      4042        10318  2084P0800X     1.0                      0.0   \n",
       "113934      3569         9052  208000000X     1.0                      0.0   \n",
       "113935      4321         1932  2084P0800X     1.0                      0.0   \n",
       "113936      4278          994  2084P0800X     1.0                      0.0   \n",
       "\n",
       "        congenital heart defect  coronavirus infections  \\\n",
       "0                           0.0                     0.0   \n",
       "1                           0.0                     0.0   \n",
       "2                           0.0                     0.0   \n",
       "4                           0.0                     0.0   \n",
       "5                           0.0                     0.0   \n",
       "...                         ...                     ...   \n",
       "113932                      0.0                     0.0   \n",
       "113933                      0.0                     0.0   \n",
       "113934                      0.0                     0.0   \n",
       "113935                      0.0                     0.0   \n",
       "113936                      0.0                     0.0   \n",
       "\n",
       "        digestive system neoplasms  Age Groups  Lung Abscess  ...  hemothorax  \\\n",
       "0                              0.0         0.0           0.0  ...         0.0   \n",
       "1                              0.0         0.0           0.0  ...         0.0   \n",
       "2                              0.0         0.0           0.0  ...         0.0   \n",
       "4                              0.0         0.0           0.0  ...         0.0   \n",
       "5                              0.0         0.0           0.0  ...         0.0   \n",
       "...                            ...         ...           ...  ...         ...   \n",
       "113932                         0.0         0.0           0.0  ...         0.0   \n",
       "113933                         0.0         0.0           0.0  ...         0.0   \n",
       "113934                         0.0         0.0           0.0  ...         0.0   \n",
       "113935                         0.0         0.0           0.0  ...         0.0   \n",
       "113936                         0.0         0.0           1.0  ...         0.0   \n",
       "\n",
       "        hoarseness  Balloon  drug therapy  listeriosis  \\\n",
       "0              0.0      0.0           0.0          0.0   \n",
       "1              0.0      0.0           0.0          0.0   \n",
       "2              0.0      0.0           0.0          0.0   \n",
       "4              0.0      0.0           0.0          0.0   \n",
       "5              0.0      0.0           0.0          0.0   \n",
       "...            ...      ...           ...          ...   \n",
       "113932         0.0      0.0           0.0          0.0   \n",
       "113933         0.0      0.0           0.0          0.0   \n",
       "113934         0.0      0.0           0.0          0.0   \n",
       "113935         0.0      0.0           0.0          0.0   \n",
       "113936         0.0      0.0           0.0          0.0   \n",
       "\n",
       "        Electronic Health Record  Medical News Platform  \\\n",
       "0                            0.0                    0.0   \n",
       "1                            0.0                    0.0   \n",
       "2                            0.0                    0.0   \n",
       "4                            0.0                    0.0   \n",
       "5                            0.0                    0.0   \n",
       "...                          ...                    ...   \n",
       "113932                       0.0                    0.0   \n",
       "113933                       NaN                    NaN   \n",
       "113934                       NaN                    NaN   \n",
       "113935                       NaN                    NaN   \n",
       "113936                       NaN                    NaN   \n",
       "\n",
       "        Online Learning Portal  Online Medical Journal  Tele-medicine Platform  \n",
       "0                          0.0                     1.0                     0.0  \n",
       "1                          0.0                     1.0                     0.0  \n",
       "2                          0.0                     1.0                     0.0  \n",
       "4                          0.0                     1.0                     0.0  \n",
       "5                          0.0                     1.0                     0.0  \n",
       "...                        ...                     ...                     ...  \n",
       "113932                     0.0                     1.0                     0.0  \n",
       "113933                     NaN                     NaN                     NaN  \n",
       "113934                     NaN                     NaN                     NaN  \n",
       "113935                     NaN                     NaN                     NaN  \n",
       "113936                     NaN                     NaN                     NaN  \n",
       "\n",
       "[108006 rows x 1622 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Handling the dimension mismatch Problem between train_df and test_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking for missing columns in test_df that are present in train_df and including them in with prop value 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_columns = set(train_df.columns)\n",
    "test_columns = set(test_df.columns)\n",
    "\n",
    "# Find the columns that are in train_df but not in test_df\n",
    "missing_columns = train_columns - test_columns\n",
    "\n",
    "# Add missing columns to test_df and initialize them with 0\n",
    "for column in missing_columns:\n",
    "    test_df[column] = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking for missing columns in train_df that are present in test_df and including them in with prop value 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_columns = set(train_df.columns)\n",
    "test_columns = set(test_df.columns)\n",
    "\n",
    "# Find the columns that are in train_df but not in test_df\n",
    "missing_columns = test_columns - train_columns\n",
    "\n",
    "# Add missing columns to test_df and initialize them with 0\n",
    "for column in missing_columns:\n",
    "    train_df[column] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(27008, 1629)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(108006, 1629)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Matching the Order of Features in Test_df and Train_df, so as to predict values from the model without error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_column_order = train_df.columns\n",
    "\n",
    "test_df = test_df.reindex(columns=train_column_order)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pandas as pd\n",
    "# import xgboost as xgb\n",
    "# from sklearn.model_selection import cross_val_score\n",
    "\n",
    "# # Split the dataset into features and target variable\n",
    "# X = train_df.drop('IS_HCP', axis=1).drop('TAXONOMY',axis =1)\n",
    "# y = train_df['IS_HCP']\n",
    "\n",
    "# # Define the XGBoost classifier\n",
    "# model = xgb.XGBClassifier()\n",
    "\n",
    "# # Perform cross-validation and calculate accuracy\n",
    "# scores = cross_val_score(model, X, y, cv=5)  # Use 5-fold cross-validation\n",
    "# accuracy = scores.mean()\n",
    "\n",
    "# print(\"Accuracy: {:.2f}%\".format(accuracy * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 1: Predicting IS_HCP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training the xgBoost Model to predict the former."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 97.39%\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import xgboost as xgb\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "\n",
    "X = train_df.drop('IS_HCP', axis=1).drop('TAXONOMY',axis =1).drop('ID',axis =1)\n",
    "y = train_df['IS_HCP']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)\n",
    "\n",
    "model = xgb.XGBClassifier()\n",
    "\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "y_pred = model.predict(X_test)\n",
    "\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "print(\"Accuracy: {:.2f}%\".format(accuracy * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking MSE and RMSE errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Squared Error (MSE): 0.03\n",
      "Root Mean Squared Error (RMSE): 0.16\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "from math import sqrt\n",
    "mse = mean_squared_error(y_test, y_pred)\n",
    "print(\"Mean Squared Error (MSE): {:.2f}\".format(mse))\n",
    "\n",
    "\n",
    "rmse = sqrt(mse)\n",
    "print(\"Root Mean Squared Error (RMSE): {:.2f}\".format(rmse))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking the Metrics to know the efficacy of the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 97.39%\n",
      "Precision: 0.97\n",
      "Recall: 0.96\n",
      "F1 Score: 0.97\n",
      "Confusion Matrix: \n",
      "[[22352   278]\n",
      " [  568  9204]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "precision = precision_score(y_test, y_pred, average='macro')\n",
    "recall = recall_score(y_test, y_pred, average='macro')\n",
    "f1 = f1_score(y_test, y_pred, average='macro')\n",
    "confusion = confusion_matrix(y_test, y_pred)\n",
    "\n",
    "print(\"Accuracy: {:.2f}%\".format(accuracy * 100))\n",
    "print(\"Precision: {:.2f}\".format(precision))\n",
    "print(\"Recall: {:.2f}\".format(recall))\n",
    "print(\"F1 Score: {:.2f}\".format(f1))\n",
    "print(\"Confusion Matrix: \")\n",
    "print(confusion)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predicting the IS_HCP for test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.fit(X, y)\n",
    "X_test = test_df.drop(['IS_HCP', 'TAXONOMY','ID'], axis=1)\n",
    "y_pred = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 0, ..., 0, 0, 1])"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Saving the results in another result dataframe with respective ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_df = pd.DataFrame({'ID': test_df['ID'], 'IS_HCP': y_pred})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df['IS_HCP'] = result_df['IS_HCP']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27008"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 2: Predicting the Taxonomy of Health care Professionals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Taking the subset of dataset where the subject is health care professional"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_train_df = train_df[train_df['IS_HCP'] == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>USERCITY</th>\n",
       "      <th>USERZIPCODE</th>\n",
       "      <th>TAXONOMY</th>\n",
       "      <th>IS_HCP</th>\n",
       "      <th>Endotracheal Intubation</th>\n",
       "      <th>congenital heart defect</th>\n",
       "      <th>coronavirus infections</th>\n",
       "      <th>digestive system neoplasms</th>\n",
       "      <th>Age Groups</th>\n",
       "      <th>Lung Abscess</th>\n",
       "      <th>...</th>\n",
       "      <th>Online Learning Portal</th>\n",
       "      <th>Online Medical Journal</th>\n",
       "      <th>Tele-medicine Platform</th>\n",
       "      <th>ID</th>\n",
       "      <th>transient ischemic attack</th>\n",
       "      <th>vascular malformations</th>\n",
       "      <th>restlessness</th>\n",
       "      <th>sleep deprivation</th>\n",
       "      <th>hemoperitoneum</th>\n",
       "      <th>cefpodoxime</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1479</td>\n",
       "      <td>8154</td>\n",
       "      <td>2084P0804X</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>487</td>\n",
       "      <td>8082</td>\n",
       "      <td>2084V0102X</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>3682</td>\n",
       "      <td>171</td>\n",
       "      <td>207RC0000X</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>586</td>\n",
       "      <td>585</td>\n",
       "      <td>2084P0804X</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>1066</td>\n",
       "      <td>9349</td>\n",
       "      <td>2084N0400X</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113932</th>\n",
       "      <td>3066</td>\n",
       "      <td>1837</td>\n",
       "      <td>207RP1001X</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113933</th>\n",
       "      <td>4042</td>\n",
       "      <td>10318</td>\n",
       "      <td>2084P0800X</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113934</th>\n",
       "      <td>3569</td>\n",
       "      <td>9052</td>\n",
       "      <td>208000000X</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113935</th>\n",
       "      <td>4321</td>\n",
       "      <td>1932</td>\n",
       "      <td>2084P0800X</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113936</th>\n",
       "      <td>4278</td>\n",
       "      <td>994</td>\n",
       "      <td>2084P0800X</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>32786 rows × 1629 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        USERCITY  USERZIPCODE    TAXONOMY  IS_HCP  Endotracheal Intubation  \\\n",
       "8           1479         8154  2084P0804X     1.0                      0.0   \n",
       "29           487         8082  2084V0102X     1.0                      0.0   \n",
       "35          3682          171  207RC0000X     1.0                      0.0   \n",
       "52           586          585  2084P0804X     1.0                      0.0   \n",
       "58          1066         9349  2084N0400X     1.0                      0.0   \n",
       "...          ...          ...         ...     ...                      ...   \n",
       "113932      3066         1837  207RP1001X     1.0                      0.0   \n",
       "113933      4042        10318  2084P0800X     1.0                      0.0   \n",
       "113934      3569         9052  208000000X     1.0                      0.0   \n",
       "113935      4321         1932  2084P0800X     1.0                      0.0   \n",
       "113936      4278          994  2084P0800X     1.0                      0.0   \n",
       "\n",
       "        congenital heart defect  coronavirus infections  \\\n",
       "8                           0.0                     0.0   \n",
       "29                          0.0                     0.0   \n",
       "35                          0.0                     0.0   \n",
       "52                          0.0                     0.0   \n",
       "58                          0.0                     0.0   \n",
       "...                         ...                     ...   \n",
       "113932                      0.0                     0.0   \n",
       "113933                      0.0                     0.0   \n",
       "113934                      0.0                     0.0   \n",
       "113935                      0.0                     0.0   \n",
       "113936                      0.0                     0.0   \n",
       "\n",
       "        digestive system neoplasms  Age Groups  Lung Abscess  ...  \\\n",
       "8                              0.0         0.0           0.0  ...   \n",
       "29                             0.0         0.0           0.0  ...   \n",
       "35                             0.0         0.0           0.0  ...   \n",
       "52                             0.0         0.0           0.0  ...   \n",
       "58                             0.0         0.0           0.0  ...   \n",
       "...                            ...         ...           ...  ...   \n",
       "113932                         0.0         0.0           0.0  ...   \n",
       "113933                         0.0         0.0           0.0  ...   \n",
       "113934                         0.0         0.0           0.0  ...   \n",
       "113935                         0.0         0.0           0.0  ...   \n",
       "113936                         0.0         0.0           1.0  ...   \n",
       "\n",
       "        Online Learning Portal  Online Medical Journal  \\\n",
       "8                          0.0                     1.0   \n",
       "29                         0.0                     1.0   \n",
       "35                         0.0                     1.0   \n",
       "52                         0.0                     1.0   \n",
       "58                         0.0                     1.0   \n",
       "...                        ...                     ...   \n",
       "113932                     0.0                     1.0   \n",
       "113933                     NaN                     NaN   \n",
       "113934                     NaN                     NaN   \n",
       "113935                     NaN                     NaN   \n",
       "113936                     NaN                     NaN   \n",
       "\n",
       "        Tele-medicine Platform  ID  transient ischemic attack  \\\n",
       "8                          0.0   0                          0   \n",
       "29                         0.0   0                          0   \n",
       "35                         0.0   0                          0   \n",
       "52                         0.0   0                          0   \n",
       "58                         0.0   0                          0   \n",
       "...                        ...  ..                        ...   \n",
       "113932                     0.0   0                          0   \n",
       "113933                     NaN   0                          0   \n",
       "113934                     NaN   0                          0   \n",
       "113935                     NaN   0                          0   \n",
       "113936                     NaN   0                          0   \n",
       "\n",
       "        vascular malformations  restlessness  sleep deprivation  \\\n",
       "8                            0             0                  0   \n",
       "29                           0             0                  0   \n",
       "35                           0             0                  0   \n",
       "52                           0             0                  0   \n",
       "58                           0             0                  0   \n",
       "...                        ...           ...                ...   \n",
       "113932                       0             0                  0   \n",
       "113933                       0             0                  0   \n",
       "113934                       0             0                  0   \n",
       "113935                       0             0                  0   \n",
       "113936                       0             0                  0   \n",
       "\n",
       "        hemoperitoneum  cefpodoxime  \n",
       "8                    0            0  \n",
       "29                   0            0  \n",
       "35                   0            0  \n",
       "52                   0            0  \n",
       "58                   0            0  \n",
       "...                ...          ...  \n",
       "113932               0            0  \n",
       "113933               0            0  \n",
       "113934               0            0  \n",
       "113935               0            0  \n",
       "113936               0            0  \n",
       "\n",
       "[32786 rows x 1629 columns]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32786, 1629)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_train_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1812"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_train_df['TAXONOMY'].isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Imputation will bring noise to the dataset, as no anchor is present to impute the value with."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_train_df = filtered_train_df.dropna()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Resetting the index after the dropna opeartion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_train_df = filtered_train_df.reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking for the missig values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_train_df['TAXONOMY'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "missing_values = filtered_train_df['TAXONOMY'].isnull()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Empty DataFrame\n",
      "Columns: [index, USERCITY, USERZIPCODE, TAXONOMY, IS_HCP, Endotracheal Intubation, congenital heart defect, coronavirus infections, digestive system neoplasms, Age Groups, Lung Abscess, brentuximab vedotin, Thoracotomy, cardiac output, retinal artery occlusion, Neuropharmacology, olfactory mucosa, Transplantation, Leukopenia, abortion, body fluids, macrolides, the, experimental, difficulty concentrating, Technology, ventricular dysfunction, Nerve Transfer, amyotrophic lateral sclerosis, bevacizumab, macrophages, Arthralgia, intubation, nonvalvular atrial fibrillation, gonorrhea, burkitt lymphoma, infliximab, Necrolytic Migratory Erythema, polyneuropathy, eosinophilic, advanced, Status Asthmaticus, hand, Orthopedics, paronychia, family practice, kidney diseases, panitumumab, nausea, catheterization, post-traumatic, Dermatology, Cannulation, encephalomyelitis, Circadian Rhythm, Vascular Malformations, Health Education, tetralogy of fallot, Skeleton, Neonates, Multiple Sclerosis, sglt2, phlebitis, coinfection, noac, thoracic outlet syndrome, Allergic, Heart Transplantation, carney complex, Clinical Pathology, molecular pathology, acanthamoeba keratitis, middle ear, Neonatal, Medical Specialties, curettage, Pelvic Inflammatory Disease, Polyneuropathy, blastomycosis, muscle weakness, pregnancy, elective surgical procedures, arrhythmogenic right ventricular dysplasia, molecular, thyroiditis, Thalassemia, Nephritis, gastrointestinal tract, mild to moderate, genital herpes, Drug Therapy, hepatology, Neurodegenerative Diseases, Preventive Medicine, Catheterization, Bernard-Soulier Syndrome, tonsillitis, respiratory insufficiency, Malabsorption Syndromes, Neutrophils, ...]\n",
      "Index: []\n",
      "\n",
      "[0 rows x 1630 columns]\n"
     ]
    }
   ],
   "source": [
    "print(filtered_train_df[missing_values])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30970, 1630)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_train_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        2084P0804X\n",
       "1        2084V0102X\n",
       "2        207RC0000X\n",
       "3        2084P0804X\n",
       "4        2084N0400X\n",
       "            ...    \n",
       "30965    207Q00000X\n",
       "30966    2084P0800X\n",
       "30967    207R00000X\n",
       "30968    2084N0400X\n",
       "30969    207RP1001X\n",
       "Name: TAXONOMY, Length: 30970, dtype: object"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_train_df['TAXONOMY']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2084P0800X    7830\n",
       "2084N0400X    6287\n",
       "207Q00000X    3936\n",
       "207R00000X    3097\n",
       "208000000X     802\n",
       "              ... \n",
       "207RS0010X       1\n",
       "208C00000X       1\n",
       "2081S0010X       1\n",
       "207ZF0201X       1\n",
       "207XX0801X       1\n",
       "Name: TAXONOMY, Length: 205, dtype: int64"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_train_df['TAXONOMY'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training a Random Forest Classification Model to predict Taxonomy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 68.94%\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "# Split the dataset into features and target variable\n",
    "X = filtered_train_df.drop(['IS_HCP', 'TAXONOMY'], axis=1).drop('ID',axis =1).drop('index', axis=1)\n",
    "y = filtered_train_df['TAXONOMY']\n",
    "\n",
    "# Split the data into training and testing sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size= 0.1, random_state = 0)\n",
    "\n",
    "# Define the Random Forest classifier\n",
    "model = RandomForestClassifier()\n",
    "\n",
    "# Train the model\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = model.predict(X_test)\n",
    "\n",
    "# Evaluate the model\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "print(\"Accuracy: {:.2f}%\".format(accuracy * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Performing Hyperparameter Tuning on the model to improve the Precison and Accuracy of the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import  GridSearchCV\n",
    "\n",
    "param_grid = {\n",
    "    'n_estimators': [100, 200, 300],\n",
    "    'max_depth': [None, 5, 10],\n",
    "    'min_samples_split': [2, 5, 10],\n",
    "    'min_samples_leaf': [1, 2, 4]\n",
    "}\n",
    "\n",
    "\n",
    "grid_search = GridSearchCV(model, param_grid, cv=5)\n",
    "grid_search.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "best_model = grid_search.best_estimator_\n",
    "best_params = grid_search.best_params_\n",
    "\n",
    "\n",
    "y_pred = best_model.predict(X_test)\n",
    "  \n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "print(\"Best Model Accuracy: {:.2f}%\".format(accuracy * 100))\n",
    "print(\"Best Model Parameters:\", best_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['2084N0400X', '2084P0800X', '207Q00000X', ..., '2084A0401X',\n",
       "       '2084N0600X', '2084N0400X'], dtype=object)"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking Metrics to analyse the Permance of the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 68.94%\n",
      "Precision: 0.60\n",
      "Recall: 0.56\n",
      "F1 Score: 0.56\n",
      "Confusion Matrix:\n",
      "[[2 0 0 ... 0 0 0]\n",
      " [0 1 0 ... 0 0 0]\n",
      " [0 0 2 ... 0 0 0]\n",
      " ...\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 1 0]\n",
      " [0 0 0 ... 0 0 5]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "precision = precision_score(y_test, y_pred, average='macro')\n",
    "recall = recall_score(y_test, y_pred, average='macro')\n",
    "f1 = f1_score(y_test, y_pred, average='macro')\n",
    "confusion = confusion_matrix(y_test, y_pred)\n",
    "\n",
    "print(\"Accuracy: {:.2f}%\".format(accuracy * 100))\n",
    "print(\"Precision: {:.2f}\".format(precision))\n",
    "print(\"Recall: {:.2f}\".format(recall))\n",
    "print(\"F1 Score: {:.2f}\".format(f1))\n",
    "print(\"Confusion Matrix:\")\n",
    "print(confusion)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predicting the Taxonomy of the subject in test dataset where IS_HCP is 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = test_df[test_df['IS_HCP'] == 1].drop('IS_HCP', axis=1)\n",
    "y_pred = best_model.predict(X_test.drop('TAXONOMY',axis =1).drop('ID',axis =1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Populating the predicted taxonomy value in result_df where IS_HCP is 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_df.loc[result_df['IS_HCP'] == 1, 'TAXONOMY'] = y_pred\n",
    "result_df.loc[result_df['IS_HCP'] != 1, 'TAXONOMY'] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['2084P0800X', '2084N0400X', '2084N0400X', ..., '2084N0400X',\n",
       "       '207L00000X', '207NS0135X'], dtype=object)"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7859"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Saving the results dataset as a csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_df.to_csv('predictions.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}