analysis/results.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "average-assumption",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "from IPython.display import Markdown as md\n",
    "from IPython.display import HTML, display\n",
    "from collections import Counter\n",
    "import math\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "pd.options.display.float_format = '{:.0f}'.format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "changed-duncan",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('../output/input.csv')\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "identical-spain",
   "metadata": {},
   "source": [
    "Emergency hospitalisations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "brilliant-smile",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_patients = len(df['patient_id'].unique())\n",
    "num_patients_hosp = len(df[df['hospital_admission'].notna()]['patient_id'].unique())\n",
    "num_patients_hosp_emergency = len(df[df['emergency_hospital_admission'].notna()]['patient_id'].unique())\n",
    "num_patients_hosp_prim_covid = len(df[df['primary_covid_hospital_admission'].notna()]['patient_id'].unique())\n",
    "num_patients_hosp_emergency_prim_covid = len(df[df['emergency_primary_covid_hospital_admission'].notna()]['patient_id'].unique())\n",
    "num_patients_hosp_covid = len(df[df['covid_hospital_admission'].notna()]['patient_id'].unique())\n",
    "num_patients_hosp_emergency_covid = len(df[df['emergency_covid_hospital_admission'].notna()]['patient_id'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "assigned-remark",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table><tr><th>Hospital Admission</th></tr><tr><th>Group</th><th>Total</th><th>%</th></tr><tr><td>Hospital Admission</td><td>2871</td><td>-</td></tr><tr><td>Emergency Admission</td><td>2831</td><td>98.60675722744688%</td></tr><tr><td>Admission with Primary Covid</td><td>2857</td><td>99.51%</td></tr><tr><td>Emergency Admission with Primary Covid</td><td>2845</td><td>99.09%</td></tr><tr><td>Admission with secondary Covid</td><td>2864</td><td>99.76%</td></tr><tr><td>Emergency Admission with secondary Covid</td><td>2838</td><td>98.85%</td></tr></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(HTML(f'<table><tr><th>Hospital Admission</th></tr><tr><th>Group</th><th>Total</th><th>%</th></tr><tr><td>Hospital Admission</td><td>{num_patients_hosp}</td><td>-</td></tr><tr><td>Emergency Admission</td><td>{num_patients_hosp_emergency}</td><td>{(num_patients_hosp_emergency/num_patients_hosp)*100}%</td></tr><tr><td>Admission with Primary Covid</td><td>{num_patients_hosp_prim_covid}</td><td>{(num_patients_hosp_prim_covid/num_patients_hosp)*100:.2f}%</td></tr><tr><td>Emergency Admission with Primary Covid</td><td>{num_patients_hosp_emergency_prim_covid}</td><td>{(num_patients_hosp_emergency_prim_covid/num_patients_hosp)*100:.2f}%</td></tr><tr><td>Admission with secondary Covid</td><td>{num_patients_hosp_covid}</td><td>{(num_patients_hosp_covid/num_patients_hosp)*100:.2f}%</td></tr><tr><td>Emergency Admission with secondary Covid</td><td>{num_patients_hosp_emergency_covid}</td><td>{(num_patients_hosp_emergency_covid/num_patients_hosp)*100:.2f}%</td></tr></table>'\n",
    "))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "finite-hughes",
   "metadata": {},
   "source": [
    "Emergency hospitalisations with primary covid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "exotic-fleece",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_patients_attended_ae = len(df[(df['primary_covid_hospital_admission'].notna() & df['ae_attendance_no_discharge']==1)]['patient_id'].unique())\n",
    "num_patients_attended_ae_with_discharge = len(df[(df['primary_covid_hospital_admission'].notna() & df['discharge_destination'].notna())]['patient_id'].unique())\n",
    "num_patients_attended_ae_cov = len(df[(df['primary_covid_hospital_admission'].notna() & df['ae_attendance_covid_status']==1)]['patient_id'].unique())\n",
    "num_patients_attended_ae_resp = len(df[(df['primary_covid_hospital_admission'].notna() & df['ae_attendance_respiratory_status']==1)]['patient_id'].unique())\n",
    "num_patients_attended_ae_cov_pc = len(df[(df['primary_covid_hospital_admission'].notna() & df['covid_primary_care_before_ae_attendance']==1)]['patient_id'].unique())\n",
    "num_patients_attended_ae_pos_test = len(df[(df['primary_covid_hospital_admission'].notna() & df['positive_covid_test_before_ae_attendance']==1)]['patient_id'].unique())\n",
    "num_patients_attended_ae_pos_test_month = len(df[(df['primary_covid_hospital_admission'].notna() & df['positive_covid_test_month_before_ae_attendance']==1)]['patient_id'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "arbitrary-survivor",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table><tr><th>Emergency Hospitalisation with primary Covid-19</th></tr><tr><th>Group</th><th>Total</th><th>%</th></tr><tr><td>Attended A&E</td><td>1120</td><td>39.20%</td></tr><tr><td>Attended A&E with discharge info</td><td>1098</td><td>38.43%</td></tr><tr><td>Attended A&E with COVID-19 diagnosis code</td><td>2560</td><td>89.60%</td></tr><tr><td>Attended A&E with respiratory-related diagnosis code</td><td>2587</td><td>90.55%</td></tr><tr><td>Attended A&E with recent COVID-19 in PC</td><td>1435</td><td>50.23%</td></tr><tr><td>Attended A&E with positive test in previous 2 weeks</td><td>1453</td><td>50.86%</td></tr><tr><td>Attended A&E with positive test in previous month</td><td>1411</td><td>49.39%</td></tr></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(HTML(f'<table><tr><th>Emergency Hospitalisation with primary Covid-19</th></tr><tr><th>Group</th><th>Total</th><th>%</th></tr><tr><td>Attended A&E</td><td>{num_patients_attended_ae}</td><td>{(num_patients_attended_ae/num_patients_hosp_prim_covid)*100:.2f}%</td></tr><tr><td>Attended A&E with discharge info</td><td>{num_patients_attended_ae_with_discharge}</td><td>{(num_patients_attended_ae_with_discharge/num_patients_hosp_prim_covid)*100:.2f}%</td></tr><tr><td>Attended A&E with COVID-19 diagnosis code</td><td>{num_patients_attended_ae_cov}</td><td>{(num_patients_attended_ae_cov/num_patients_hosp_prim_covid)*100:.2f}%</td></tr><tr><td>Attended A&E with respiratory-related diagnosis code</td><td>{num_patients_attended_ae_resp}</td><td>{(num_patients_attended_ae_resp/num_patients_hosp_prim_covid)*100:.2f}%</td></tr><tr><td>Attended A&E with recent COVID-19 in PC</td><td>{num_patients_attended_ae_cov_pc}</td><td>{(num_patients_attended_ae_cov_pc/num_patients_hosp_prim_covid)*100:.2f}%</td></tr><tr><td>Attended A&E with positive test in previous 2 weeks</td><td>{num_patients_attended_ae_pos_test}</td><td>{(num_patients_attended_ae_pos_test/num_patients_hosp_prim_covid)*100:.2f}%</td></tr><tr><td>Attended A&E with positive test in previous month</td><td>{num_patients_attended_ae_pos_test_month}</td><td>{(num_patients_attended_ae_pos_test_month/num_patients_hosp_prim_covid)*100:.2f}%</td></tr></table>'))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "colored-marriage",
   "metadata": {},
   "source": [
    "Discharge destination in patients with primary covid hospitalisation who previously attended A&E."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "quantitative-imaging",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Discharge Destination</th>\n",
       "      <th>Number</th>\n",
       "      <th>%</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ICU</td>\n",
       "      <td>239</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Ward</td>\n",
       "      <td>194</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>missing</td>\n",
       "      <td>687</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Discharge Destination  Number  %\n",
       "0                   ICU     239 21\n",
       "1                  Ward     194 17\n",
       "2               missing     687 61"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prim_cov_ae_discharge = df[(df['emergency_primary_covid_hospital_admission'].notna() & df['ae_attendance_no_discharge']==1)]['discharge_destination']\n",
    "missing = prim_cov_ae_discharge.isna().sum()\n",
    "destination_dict = Counter(prim_cov_ae_discharge[prim_cov_ae_discharge.notnull()])\n",
    "destination_dict['missing'] = missing\n",
    "\n",
    "discharge_dict = {1066341000000100:\"Ambulatory Emergency Care\", 19712007: \"Patient Transfer\", 183919006: \"Hospice\", 1066361000000104: \"High dependency unit\", 305398007: \"Mortuary\", 1066381000000108: \"Special baby care unit\", 1066331000000109: \"Emergency department short stay ward\", 306705005: \"Police Custody\", 306706006:\"Ward\", 306689006: \"Home\", 306694006: \"Residential or Nursing Home\", 306691003: \"Residential or Nursing Home\", 1066351000000102: \"Hospital at home\", 1066401000000108: \"Neonatal ICU\", 1066371000000106: \"Coronary Care Unit\", 50861005: \"Legal Custody\", 1066391000000105: \"ICU\", \"missing\": \"missing\"}\n",
    "percent_dict = {}\n",
    "data = []\n",
    "total = 0\n",
    "\n",
    "other_count=0\n",
    "drop_keys=[]\n",
    "#Drop dictionary pairs if value <10\n",
    "for key, value in destination_dict.items():\n",
    "    if value <10:\n",
    "        other_count +=value\n",
    "        drop_keys.append(key)\n",
    "        \n",
    "for key in drop_keys:\n",
    "    del destination_dict[key]\n",
    "    \n",
    "destination_dict[\"Other\"] = other_count\n",
    "\n",
    "for key, value in destination_dict.items():\n",
    "    total+=value\n",
    "\n",
    "for key, value in destination_dict.items():\n",
    "    if key==\"Other\":\n",
    "        pass\n",
    "    else:\n",
    "        percent = (value/total) * 100\n",
    "        row = [discharge_dict[key], value, percent]\n",
    "        data.append(row)\n",
    "\n",
    "discharge_destination_df = pd.DataFrame(data, columns=[\"Discharge Destination\", \"Number\", \"%\"])\n",
    "discharge_destination_df\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "lyric-lawyer",
   "metadata": {},
   "source": [
    "Test hospital admission subset.\n",
    "Hospital admission covers those in emergency short stay, ward or icu.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "utility-insert",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({0: 6000, 1: 4000})"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Counter(df['ae_attendance_hosp_discharge'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sublime-subscriber",
   "metadata": {},
   "source": [
    "Define models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "greenhouse-ceremony",
   "metadata": {},
   "outputs": [],
   "source": [
    "positive_covid_patients_sus = df[df['emergency_primary_covid_hospital_admission'].notna()]\n",
    "negative_covid_patients_sus = df[~df['emergency_primary_covid_hospital_admission'].notna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "passive-comfort",
   "metadata": {},
   "outputs": [],
   "source": [
    "# model_a\n",
    "\n",
    "positive_covid_patients_a = df[(df['ae_attendance_no_discharge']==1) & ((df['ae_attendance_covid_status']==1) | (df['positive_covid_test_before_ae_attendance'] ==1) | (df['covid_primary_care_before_ae_attendance'] ==1))]\n",
    "negative_covid_patients_a = df[(df['ae_attendance_no_discharge']==0) | ((df['ae_attendance_no_discharge']==1) & ((df['ae_attendance_covid_status']==0) & (df['positive_covid_test_before_ae_attendance'] ==0) & (df['covid_primary_care_before_ae_attendance'] ==0)))]\n",
    "\n",
    "\n",
    "sus_patients_positive = set(list(positive_covid_patients_sus['patient_id']))\n",
    "model_a_patients_positive = set(list(positive_covid_patients_a['patient_id']))\n",
    "\n",
    "sus_patients_negative = set(list(negative_covid_patients_sus['patient_id']))\n",
    "model_a_patients_negative = set(list(negative_covid_patients_a['patient_id']))\n",
    "\n",
    "\n",
    "sus_pos_ecds_pos = len(list(set(sus_patients_positive) & set(model_a_patients_positive)))\n",
    "sus_pos_ecds_neg = len(list(set(sus_patients_positive) & set(model_a_patients_negative)))\n",
    "sus_neg_ecds_pos = len(list(set(sus_patients_negative) & set(model_a_patients_positive)))\n",
    "sus_neg_ecds_neg = len(list(set(sus_patients_negative) & set(model_a_patients_negative)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "impaired-husband",
   "metadata": {},
   "outputs": [],
   "source": [
    "sensitivity_a = (sus_pos_ecds_pos/(sus_pos_ecds_pos + sus_pos_ecds_neg))*100\n",
    "specificity_a = (sus_neg_ecds_neg/(sus_neg_ecds_pos + sus_neg_ecds_neg))*100\n",
    "PPV_a = (sus_pos_ecds_pos/(sus_pos_ecds_pos + sus_neg_ecds_pos))*100\n",
    "NPV_a = (sus_neg_ecds_neg/(sus_neg_ecds_neg + sus_pos_ecds_neg))*100\n",
    "MCC_a = ((sus_pos_ecds_pos * sus_neg_ecds_neg)-(sus_neg_ecds_pos * sus_pos_ecds_neg))/math.sqrt((sus_pos_ecds_pos + sus_neg_ecds_pos)*(sus_pos_ecds_neg+sus_neg_ecds_neg)*(sus_pos_ecds_pos + sus_pos_ecds_neg)*(sus_neg_ecds_pos+sus_neg_ecds_neg))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "paperback-trailer",
   "metadata": {},
   "outputs": [],
   "source": [
    "# model_b\n",
    "hospital_discharge_destinations = [306706006, 1066331000000109, 1066391000000105]\n",
    "positive_covid_patients_b = df[(df['ae_attendance_hosp_discharge']==1) & ((df['ae_attendance_covid_status']==1) | (df['positive_covid_test_before_ae_attendance'] ==1) | (df['covid_primary_care_before_ae_attendance'] ==1))]\n",
    "negative_covid_patients_b = df[df['ae_attendance_hosp_discharge']==0 | ((df['ae_attendance_hosp_discharge']==1) & ((df['ae_attendance_covid_status']==0) & (df['positive_covid_test_before_ae_attendance'] ==0) & (df['covid_primary_care_before_ae_attendance'] ==0)))]\n",
    "\n",
    "\n",
    "model_b_patients_positive = set(list(positive_covid_patients_b['patient_id']))\n",
    "model_b_patients_negative = set(list(negative_covid_patients_b['patient_id']))\n",
    "\n",
    "\n",
    "sus_pos_ecds_pos = len(list(set(sus_patients_positive) & set(model_b_patients_positive)))\n",
    "sus_pos_ecds_neg = len(list(set(sus_patients_positive) & set(model_b_patients_negative)))\n",
    "sus_neg_ecds_pos = len(list(set(sus_patients_negative) & set(model_b_patients_positive)))\n",
    "sus_neg_ecds_neg = len(list(set(sus_patients_negative) & set(model_b_patients_negative)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "missing-weekend",
   "metadata": {},
   "outputs": [],
   "source": [
    "sensitivity_b = (sus_pos_ecds_pos/(sus_pos_ecds_pos + sus_pos_ecds_neg))*100\n",
    "specificity_b = (sus_neg_ecds_neg/(sus_neg_ecds_pos + sus_neg_ecds_neg))*100\n",
    "PPV_b = (sus_pos_ecds_pos/(sus_pos_ecds_pos + sus_neg_ecds_pos))*100\n",
    "NPV_b = (sus_neg_ecds_neg/(sus_neg_ecds_neg + sus_pos_ecds_neg))*100\n",
    "MCC_b = ((sus_pos_ecds_pos * sus_neg_ecds_neg)-(sus_neg_ecds_pos * sus_pos_ecds_neg))/math.sqrt((sus_pos_ecds_pos + sus_neg_ecds_pos)*(sus_pos_ecds_neg+sus_neg_ecds_neg)*(sus_pos_ecds_pos + sus_pos_ecds_neg)*(sus_neg_ecds_pos+sus_neg_ecds_neg))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "japanese-bottle",
   "metadata": {},
   "outputs": [],
   "source": [
    "#model_c\n",
    "\n",
    "positive_covid_patients_c = df[(df['ae_attendance_hosp_discharge']==1& (df['discharge_destination'].isin(hospital_discharge_destinations))) & ((df['ae_attendance_covid_status']==1) | (df['ae_attendance_respiratory_status']==1) | (df['positive_covid_test_before_ae_attendance'] ==1) | (df['covid_primary_care_before_ae_attendance'] ==1))]\n",
    "negative_covid_patients_c = df[df['ae_attendance_hosp_discharge']==0 | ((df['ae_attendance_hosp_discharge']==1) & ((df['ae_attendance_covid_status']==0) & (df['ae_attendance_respiratory_status']==1) & (df['positive_covid_test_before_ae_attendance'] ==0) & (df['covid_primary_care_before_ae_attendance'] ==0)))]\n",
    "\n",
    "\n",
    "model_c_patients_positive = set(list(positive_covid_patients_c['patient_id']))\n",
    "model_c_patients_negative = set(list(negative_covid_patients_c['patient_id']))\n",
    "\n",
    "\n",
    "sus_pos_ecds_pos = len(list(set(sus_patients_positive) & set(model_c_patients_positive)))\n",
    "sus_pos_ecds_neg = len(list(set(sus_patients_positive) & set(model_c_patients_negative)))\n",
    "sus_neg_ecds_pos = len(list(set(sus_patients_negative) & set(model_c_patients_positive)))\n",
    "sus_neg_ecds_neg = len(list(set(sus_patients_negative) & set(model_c_patients_negative)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "failing-jewelry",
   "metadata": {},
   "outputs": [],
   "source": [
    "sensitivity_c = (sus_pos_ecds_pos/(sus_pos_ecds_pos + sus_pos_ecds_neg))*100\n",
    "specificity_c = (sus_neg_ecds_neg/(sus_neg_ecds_pos + sus_neg_ecds_neg))*100\n",
    "PPV_c = (sus_pos_ecds_pos/(sus_pos_ecds_pos + sus_neg_ecds_pos))*100\n",
    "NPV_c = (sus_neg_ecds_neg/(sus_neg_ecds_neg + sus_pos_ecds_neg))*100\n",
    "MCC_c = ((sus_pos_ecds_pos * sus_neg_ecds_neg)-(sus_neg_ecds_pos * sus_pos_ecds_neg))/math.sqrt((sus_pos_ecds_pos + sus_neg_ecds_pos)*(sus_pos_ecds_neg+sus_neg_ecds_neg)*(sus_pos_ecds_pos + sus_pos_ecds_neg)*(sus_neg_ecds_pos+sus_neg_ecds_neg))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "commercial-overall",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table><tr><th>Model</th><th>Sensitivity</th><th>Specificity</th><th>PPV</th><th>NPV</th><th>MCC</th></tr><td>A</td><td>39.61</td><td>61.13</td><td>28.84</td><td>71.80</td><td>0.01</td><tr><td>B</td><td>39.72</td><td>61.27</td><td>28.97</td><td>71.88</td><td>0.01</td></tr><tr><td>C</td><td>46.17</td><td>54.10</td><td>28.38</td><td>71.84</td><td>0.00</td></tr></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "display(HTML(f'<table><tr><th>Model</th><th>Sensitivity</th><th>Specificity</th><th>PPV</th><th>NPV</th><th>MCC</th></tr><td>A</td><td>{sensitivity_a:.2f}</td><td>{specificity_a:.2f}</td><td>{PPV_a:.2f}</td><td>{NPV_a:.2f}</td><td>{MCC_a:.2f}</td><tr><td>B</td><td>{sensitivity_b:.2f}</td><td>{specificity_b:.2f}</td><td>{PPV_b:.2f}</td><td>{NPV_b:.2f}</td><td>{MCC_b:.2f}</td></tr><tr><td>C</td><td>{sensitivity_c:.2f}</td><td>{specificity_c:.2f}</td><td>{PPV_c:.2f}</td><td>{NPV_c:.2f}</td><td>{MCC_c:.2f}</td></tr></table>'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "iraqi-polish",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from venn import venn\n",
    "\n",
    "# data = {\"Model A\": model_a_patients_positive, \"Model B\": model_b_patients_positive, \"Model C\": model_c_patients_positive, \"Actual\":sus_patients_positive}\n",
    "# venn(data, fmt=\"{percentage:.1f}%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "patient-gateway",
   "metadata": {},
   "source": [
    "Sensitivity analysis in bets model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "advance-console",
   "metadata": {},
   "outputs": [],
   "source": [
    "sensitivity_dict = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "significant-swing",
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop cov ae\n",
    "positive_covid_patients = df[(df['ae_attendance_hosp_discharge']==1) & ((df['positive_covid_test_before_ae_attendance'] ==1) | (df['covid_primary_care_before_ae_attendance'] ==1))]\n",
    "negative_covid_patients = df[(df['ae_attendance_hosp_discharge']==0) | ((df['ae_attendance_hosp_discharge']==1) &  (df['positive_covid_test_before_ae_attendance'] ==0) & (df['covid_primary_care_before_ae_attendance'] ==0))]\n",
    "model_patients_positive = set(list(positive_covid_patients['patient_id']))\n",
    "model_patients_negative = set(list(negative_covid_patients['patient_id']))\n",
    "sus_pos_ecds_pos = len(list(set(sus_patients_positive) & set(model_patients_positive)))\n",
    "sus_pos_ecds_neg = len(list(set(sus_patients_positive) & set(model_patients_negative)))\n",
    "sus_neg_ecds_pos = len(list(set(sus_patients_negative) & set(model_patients_positive)))\n",
    "sus_neg_ecds_neg = len(list(set(sus_patients_negative) & set(model_patients_negative)))\n",
    "MCC = ((sus_pos_ecds_pos * sus_neg_ecds_neg)-(sus_neg_ecds_pos * sus_pos_ecds_neg))/math.sqrt((sus_pos_ecds_pos + sus_neg_ecds_pos)*(sus_pos_ecds_neg+sus_neg_ecds_neg)*(sus_pos_ecds_pos + sus_pos_ecds_neg)*(sus_neg_ecds_pos+sus_neg_ecds_neg))\n",
    "sensitivity_dict['cov_ae'] = MCC\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "powered-howard",
   "metadata": {},
   "outputs": [],
   "source": [
    "#drop recent pos test\n",
    "positive_covid_patients = df[(df['ae_attendance_hosp_discharge']==1) & ((df['ae_attendance_covid_status']==1)  | (df['covid_primary_care_before_ae_attendance'] ==1))]\n",
    "negative_covid_patients = df[(df['ae_attendance_hosp_discharge']==0) | ((df['ae_attendance_hosp_discharge']==1) & (df['ae_attendance_covid_status']==0)  & (df['covid_primary_care_before_ae_attendance'] ==0))]\n",
    "model_patients_positive = set(list(positive_covid_patients['patient_id']))\n",
    "model_patients_negative = set(list(negative_covid_patients['patient_id']))\n",
    "sus_pos_ecds_pos = len(list(set(sus_patients_positive) & set(model_patients_positive)))\n",
    "sus_pos_ecds_neg = len(list(set(sus_patients_positive) & set(model_patients_negative)))\n",
    "sus_neg_ecds_pos = len(list(set(sus_patients_negative) & set(model_patients_positive)))\n",
    "sus_neg_ecds_neg = len(list(set(sus_patients_negative) & set(model_patients_negative)))\n",
    "MCC = ((sus_pos_ecds_pos * sus_neg_ecds_neg)-(sus_neg_ecds_pos * sus_pos_ecds_neg))/math.sqrt((sus_pos_ecds_pos + sus_neg_ecds_pos)*(sus_pos_ecds_neg+sus_neg_ecds_neg)*(sus_pos_ecds_pos + sus_pos_ecds_neg)*(sus_neg_ecds_pos+sus_neg_ecds_neg))\n",
    "sensitivity_dict['pos_test'] = MCC\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "happy-atlantic",
   "metadata": {},
   "outputs": [],
   "source": [
    "#drop cov pc\n",
    "positive_covid_patients = df[((df['ae_attendance_hosp_discharge']==1) & ((df['ae_attendance_covid_status']==1) | (df['positive_covid_test_before_ae_attendance'] ==1) ))]\n",
    "negative_covid_patients = df[(df['ae_attendance_hosp_discharge']==0) | ((df['ae_attendance_hosp_discharge']==1) & (df['ae_attendance_covid_status']==0) & (df['positive_covid_test_before_ae_attendance'] ==0))]\n",
    "model_patients_positive = set(list(positive_covid_patients['patient_id']))\n",
    "model_patients_negative = set(list(negative_covid_patients['patient_id']))\n",
    "sus_pos_ecds_pos = len(list(set(sus_patients_positive) & set(model_patients_positive)))\n",
    "sus_pos_ecds_neg = len(list(set(sus_patients_positive) & set(model_patients_negative)))\n",
    "sus_neg_ecds_pos = len(list(set(sus_patients_negative) & set(model_patients_positive)))\n",
    "sus_neg_ecds_neg = len(list(set(sus_patients_negative) & set(model_patients_negative)))\n",
    "MCC = ((sus_pos_ecds_pos * sus_neg_ecds_neg)-(sus_neg_ecds_pos * sus_pos_ecds_neg))/math.sqrt((sus_pos_ecds_pos + sus_neg_ecds_pos)*(sus_pos_ecds_neg+sus_neg_ecds_neg)*(sus_pos_ecds_pos + sus_pos_ecds_neg)*(sus_neg_ecds_pos+sus_neg_ecds_neg))\n",
    "sensitivity_dict['cov_pc'] = MCC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "signal-lending",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'cov_ae': 0.006597924332885599,\n",
       " 'pos_test': 0.008630345986045293,\n",
       " 'cov_pc': 0.010326436683224489}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sensitivity_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "configured-spiritual",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table><tr><th>Variable dropped</th><th>MCC</th></tr><tr><td>-</td><td>0.01</td></tr><tr><td>Covid Diagnosis Code</td><td>0.01</td></tr><tr><td>Recent Positive Covid Test</td><td>0.01</td></tr><tr><td>Recent Covid in Primary Care</td><td>0.01</td></tr></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(HTML(f'<table><tr><th>Variable dropped</th><th>MCC</th></tr><tr><td>-</td><td>{MCC_b:.2f}</td></tr><tr><td>Covid Diagnosis Code</td><td>{sensitivity_dict[\"cov_ae\"]:.2f}</td></tr><tr><td>Recent Positive Covid Test</td><td>{sensitivity_dict[\"pos_test\"]:.2f}</td></tr><tr><td>Recent Covid in Primary Care</td><td>{sensitivity_dict[\"cov_pc\"]:.2f}</td></tr></table>'))#\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "patient-adjustment",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SUS-positive</th>\n",
       "      <th>SUS-negative</th>\n",
       "      <th>Total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ECDS-positive</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ECDS-negative</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Total</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               SUS-positive  SUS-negative  Total\n",
       "ECDS-positive             0             0      0\n",
       "ECDS-negative             0             0      0\n",
       "Total                     0             0      0"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output = pd.DataFrame([[sus_pos_ecds_pos, sus_neg_ecds_pos, (sus_pos_ecds_pos + sus_neg_ecds_pos)], [sus_pos_ecds_neg, sus_neg_ecds_neg, (sus_pos_ecds_neg + sus_neg_ecds_neg)], [(sus_pos_ecds_pos+sus_pos_ecds_neg), (sus_neg_ecds_pos+sus_neg_ecds_neg), (sus_pos_ecds_pos + sus_pos_ecds_neg + sus_neg_ecds_pos + sus_neg_ecds_neg)]], columns=[\"SUS-positive\", \"SUS-negative\", \"Total\"], index=[\"ECDS-positive\", \"ECDS-negative\", \"Total\"])\n",
    "output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "essential-burton",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(model_a_patients_positive)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "constitutional-handy",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(list(set(sus_patients_positive) & set(model_a_patients_positive)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "according-belief",
   "metadata": {},
   "outputs": [
    {
     "ename": "ZeroDivisionError",
     "evalue": "division by zero",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m-------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mZeroDivisionError\u001b[0m                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-31-4d8432d045da>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m    118\u001b[0m \u001b[0msus_neg_ecds_neg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msus_patients_negative\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m&\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_a_patients_negative\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 120\u001b[0;31m \u001b[0msensitivity_a\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0msus_pos_ecds_pos\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msus_pos_ecds_pos\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msus_pos_ecds_neg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    121\u001b[0m \u001b[0mspecificity_a\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0msus_neg_ecds_neg\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msus_neg_ecds_pos\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msus_neg_ecds_neg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    122\u001b[0m \u001b[0mPPV_a\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0msus_pos_ecds_pos\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msus_pos_ecds_pos\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msus_neg_ecds_pos\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mZeroDivisionError\u001b[0m: division by zero"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "import math\n",
    "import json\n",
    "\n",
    "df = pd.read_csv('../output/input_descriptives.csv')\n",
    "\n",
    "\n",
    "#emergency hospitalisation\n",
    "\n",
    "num_patients = len(df['patient_id'].unique())\n",
    "num_patients_hosp = len(df[df['hospital_admission'].notna()]['patient_id'].unique())\n",
    "num_patients_hosp_emergency = len(df[df['emergency_hospital_admission'].notna()]['patient_id'].unique())\n",
    "num_patients_hosp_prim_covid = len(df[df['primary_covid_hospital_admission'].notna()]['patient_id'].unique())\n",
    "num_patients_hosp_emergency_prim_covid = len(df[df['emergency_primary_covid_hospital_admission'].notna()]['patient_id'].unique())\n",
    "num_patients_hosp_covid = len(df[df['covid_hospital_admission'].notna()]['patient_id'].unique())\n",
    "num_patients_hosp_emergency_covid = len(df[df['emergency_covid_hospital_admission'].notna()]['patient_id'].unique())\n",
    "\n",
    "emergency_hospitalisation_dict = {\n",
    "    \"hospital_admission\": num_patients_hosp,\n",
    "    \"emergency_hospital_admission\": num_patients_hosp_emergency,\n",
    "    \"admisssion_primary_covid\": num_patients_hosp_prim_covid,\n",
    "    \"admission_secondary_covid\": num_patients_hosp_covid,\n",
    "    \"emergency_admission_primary_covid\": num_patients_hosp_emergency_prim_covid,\n",
    "    \"emergency_admission_secondary_covid\": num_patients_hosp_emergency_covid,\n",
    "}\n",
    "\n",
    "# with open('output/emergency_hospitalisation.json', 'w') as fp:\n",
    "#     json.dump(emergency_hospitalisation_dict, fp)\n",
    "\n",
    "\n",
    "# a&e attendance in emergency primary covid hospital admissions\n",
    "num_patients_attended_ae = len(df[(df['emergency_primary_covid_hospital_admission'].notna() & df['ae_attendance_any_discharge']==1)]['patient_id'].unique())\n",
    "num_patients_attended_ae_with_discharge = len(df[(df['emergency_primary_covid_hospital_admission'].notna() & df['discharge_destination'].notna())]['patient_id'].unique())\n",
    "num_patients_attended_ae_cov = len(df[(df['emergency_primary_covid_hospital_admission'].notna() & df['ae_attendance_covid_status']==1)]['patient_id'].unique())\n",
    "num_patients_attended_ae_resp = len(df[(df['emergency_primary_covid_hospital_admission'].notna() & df['ae_attendance_respiratory_status']==1)]['patient_id'].unique())\n",
    "num_patients_attended_ae_cov_pc = len(df[(df['emergency_primary_covid_hospital_admission'].notna() & df['covid_primary_care_before_ae_attendance']==1)]['patient_id'].unique())\n",
    "num_patients_attended_ae_pos_test = len(df[(df['emergency_primary_covid_hospital_admission'].notna() & df['positive_covid_test_before_ae_attendance']==1)]['patient_id'].unique())\n",
    "num_patients_attended_ae_pos_test_month = len(df[(df['emergency_primary_covid_hospital_admission'].notna() & df['positive_covid_test_month_before_ae_attendance']==1)]['patient_id'].unique())\n",
    "\n",
    "ae_dict = {\n",
    "    \"attended_ae\": num_patients_attended_ae,\n",
    "    \"attended_ae_with_discharge\": num_patients_attended_ae_with_discharge,\n",
    "    \"attended_ae_cov\": num_patients_attended_ae_cov,\n",
    "    \"attended_ae_resp\": num_patients_attended_ae_resp,\n",
    "    \"attended_ae_cov_pc\": num_patients_attended_ae_cov_pc,\n",
    "    \"attended_ae_pos_test\": num_patients_attended_ae_pos_test,\n",
    "    \"attended_ae_pos_test_month\": num_patients_attended_ae_pos_test_month,\n",
    "}\n",
    "\n",
    "# with open('output/ae.json', 'w') as fp:\n",
    "#     json.dump(ae_dict, fp)\n",
    "\n",
    "# discharge destination in those with primary cov in emergency hosp admissions (who went to ae)\n",
    "\n",
    "prim_cov_ae_discharge = df[(df['emergency_primary_covid_hospital_admission'].notna() & df['ae_attendance_any_discharge']==1)]['discharge_destination']\n",
    "missing = prim_cov_ae_discharge.isna().sum()\n",
    "destination_dict = Counter(prim_cov_ae_discharge[prim_cov_ae_discharge.notnull()])\n",
    "destination_dict['missing'] = missing\n",
    "\n",
    "discharge_dict = {1066341000000100:\"Ambulatory Emergency Care\", 19712007: \"Patient Transfer\", 183919006: \"Hospice\", 1066361000000104: \"High dependency unit\", 305398007: \"Mortuary\", 1066381000000108: \"Special baby care unit\", 1066331000000109: \"Emergency department short stay ward\", 306705005: \"Police Custody\", 306706006:\"Ward\", 306689006: \"Home\", 306694006: \"Residential or Nursing Home\", 306691003: \"Residential or Nursing Home\", 1066351000000102: \"Hospital at home\", 1066401000000108: \"Neonatal ICU\", 1066371000000106: \"Coronary Care Unit\", 50861005: \"Legal Custody\", 1066391000000105: \"ICU\", \"missing\": \"missing\"}\n",
    "percent_dict = {}\n",
    "data = []\n",
    "total = 0\n",
    "\n",
    "other_count=0\n",
    "drop_keys=[]\n",
    "#Drop dictionary pairs if value <10\n",
    "for key, value in destination_dict.items():\n",
    "    if value <10:\n",
    "        other_count +=value\n",
    "        drop_keys.append(key)\n",
    "        \n",
    "for key in drop_keys:\n",
    "    del destination_dict[key]\n",
    "    \n",
    "destination_dict[\"Other\"] = other_count\n",
    "\n",
    "for key, value in destination_dict.items():\n",
    "    total+=value\n",
    "\n",
    "for key, value in destination_dict.items():\n",
    "    if key==\"Other\":\n",
    "        pass\n",
    "    else:\n",
    "        percent = (value/total) * 100\n",
    "        row = [discharge_dict[key], value, percent]\n",
    "        data.append(row)\n",
    "\n",
    "discharge_destination_df = pd.DataFrame(data, columns=[\"Discharge Destination\", \"Number\", \"%\"])\n",
    "# discharge_destination_df.to_csv('output/discharge_destination.csv')\n",
    "\n",
    "\n",
    "\n",
    "#models\n",
    "df = pd.read_csv('../output/input.csv')\n",
    "\n",
    "positive_covid_patients_sus = df[df['emergency_primary_covid_hospital_admission'].notna()]\n",
    "negative_covid_patients_sus = df[~df['emergency_primary_covid_hospital_admission'].notna()]\n",
    "\n",
    "# model_a\n",
    "df[(df['ae_attendance_any_discharge']==1)&()]\n",
    "positive_covid_patients_a = df[(df['ae_attendance_any_discharge']==1) & ((df['ae_attendance_covid_status']==1) | (df['positive_covid_test_before_ae_attendance'] ==1) | (df['covid_primary_care_before_ae_attendance'] ==1))]\n",
    "negative_covid_patients_a = df[(df['ae_attendance_any_discharge']==0) | ((df['ae_attendance_any_discharge']==1) & ((df['ae_attendance_covid_status']==0) & (df['positive_covid_test_before_ae_attendance'] ==0) & (df['covid_primary_care_before_ae_attendance'] ==0)))]\n",
    "\n",
    "\n",
    "sus_patients_positive = set(list(positive_covid_patients_sus['patient_id']))\n",
    "model_a_patients_positive = set(list(positive_covid_patients_a['patient_id']))\n",
    "\n",
    "sus_patients_negative = set(list(negative_covid_patients_sus['patient_id']))\n",
    "model_a_patients_negative = set(list(negative_covid_patients_a['patient_id']))\n",
    "\n",
    "\n",
    "sus_pos_ecds_pos = len(list(set(sus_patients_positive) & set(model_a_patients_positive)))\n",
    "sus_pos_ecds_neg = len(list(set(sus_patients_positive) & set(model_a_patients_negative)))\n",
    "sus_neg_ecds_pos = len(list(set(sus_patients_negative) & set(model_a_patients_positive)))\n",
    "sus_neg_ecds_neg = len(list(set(sus_patients_negative) & set(model_a_patients_negative)))\n",
    "\n",
    "sensitivity_a = (sus_pos_ecds_pos/(sus_pos_ecds_pos + sus_pos_ecds_neg))*100\n",
    "specificity_a = (sus_neg_ecds_neg/(sus_neg_ecds_pos + sus_neg_ecds_neg))*100\n",
    "PPV_a = (sus_pos_ecds_pos/(sus_pos_ecds_pos + sus_neg_ecds_pos))*100\n",
    "NPV_a = (sus_neg_ecds_neg/(sus_neg_ecds_neg + sus_pos_ecds_neg))*100\n",
    "MCC_a = ((sus_pos_ecds_pos * sus_neg_ecds_neg)-(sus_neg_ecds_pos * sus_pos_ecds_neg))/math.sqrt((sus_pos_ecds_pos + sus_neg_ecds_pos)*(sus_pos_ecds_neg+sus_neg_ecds_neg)*(sus_pos_ecds_pos + sus_pos_ecds_neg)*(sus_neg_ecds_pos+sus_neg_ecds_neg))\n",
    "output = pd.DataFrame([[sus_pos_ecds_pos, sus_neg_ecds_pos, (sus_pos_ecds_pos + sus_neg_ecds_pos)], [sus_pos_ecds_neg, sus_neg_ecds_neg, (sus_pos_ecds_neg + sus_neg_ecds_neg)], [(sus_pos_ecds_pos+sus_pos_ecds_neg), (sus_neg_ecds_pos+sus_neg_ecds_neg), (sus_pos_ecds_pos + sus_pos_ecds_neg + sus_neg_ecds_pos + sus_neg_ecds_neg)]], columns=[\"SUS-positive\", \"SUS-negative\", \"Total\"], index=[\"ECDS-positive\", \"ECDS-negative\", \"Total\"])\n",
    "# output.to_csv('output/model_a.csv')\n",
    "\n",
    "# model_b\n",
    "hospital_discharge_destinations = [306706006, 1066331000000109, 1066391000000105]\n",
    "positive_covid_patients_b = df[(df['ae_attendance_hosp_discharge']==1) & ((df['ae_attendance_covid_status']==1) | (df['positive_covid_test_before_ae_attendance'] ==1) | (df['covid_primary_care_before_ae_attendance'] ==1))]\n",
    "negative_covid_patients_b = df[df['ae_attendance_hosp_discharge']==0 | ((df['ae_attendance_hosp_discharge']==1) & ((df['ae_attendance_covid_status']==0) & (df['positive_covid_test_before_ae_attendance'] ==0) & (df['covid_primary_care_before_ae_attendance'] ==0)))]\n",
    "\n",
    "\n",
    "model_b_patients_positive = set(list(positive_covid_patients_b['patient_id']))\n",
    "model_b_patients_negative = set(list(negative_covid_patients_b['patient_id']))\n",
    "\n",
    "\n",
    "sus_pos_ecds_pos = len(list(set(sus_patients_positive) & set(model_b_patients_positive)))\n",
    "sus_pos_ecds_neg = len(list(set(sus_patients_positive) & set(model_b_patients_negative)))\n",
    "sus_neg_ecds_pos = len(list(set(sus_patients_negative) & set(model_b_patients_positive)))\n",
    "sus_neg_ecds_neg = len(list(set(sus_patients_negative) & set(model_b_patients_negative)))\n",
    "\n",
    "sensitivity_b = (sus_pos_ecds_pos/(sus_pos_ecds_pos + sus_pos_ecds_neg))*100\n",
    "specificity_b = (sus_neg_ecds_neg/(sus_neg_ecds_pos + sus_neg_ecds_neg))*100\n",
    "PPV_b = (sus_pos_ecds_pos/(sus_pos_ecds_pos + sus_neg_ecds_pos))*100\n",
    "NPV_b = (sus_neg_ecds_neg/(sus_neg_ecds_neg + sus_pos_ecds_neg))*100\n",
    "MCC_b = ((sus_pos_ecds_pos * sus_neg_ecds_neg)-(sus_neg_ecds_pos * sus_pos_ecds_neg))/math.sqrt((sus_pos_ecds_pos + sus_neg_ecds_pos)*(sus_pos_ecds_neg+sus_neg_ecds_neg)*(sus_pos_ecds_pos + sus_pos_ecds_neg)*(sus_neg_ecds_pos+sus_neg_ecds_neg))\n",
    "output = pd.DataFrame([[sus_pos_ecds_pos, sus_neg_ecds_pos, (sus_pos_ecds_pos + sus_neg_ecds_pos)], [sus_pos_ecds_neg, sus_neg_ecds_neg, (sus_pos_ecds_neg + sus_neg_ecds_neg)], [(sus_pos_ecds_pos+sus_pos_ecds_neg), (sus_neg_ecds_pos+sus_neg_ecds_neg), (sus_pos_ecds_pos + sus_pos_ecds_neg + sus_neg_ecds_pos + sus_neg_ecds_neg)]], columns=[\"SUS-positive\", \"SUS-negative\", \"Total\"], index=[\"ECDS-positive\", \"ECDS-negative\", \"Total\"])\n",
    "# output.to_csv('output/model_b.csv')\n",
    "\n",
    "#model_c\n",
    "\n",
    "positive_covid_patients_c = df[(df['ae_attendance_hosp_discharge']==1& (df['discharge_destination'].isin(hospital_discharge_destinations))) & ((df['ae_attendance_covid_status']==1) | (df['ae_attendance_respiratory_status']==1) | (df['positive_covid_test_before_ae_attendance'] ==1) | (df['covid_primary_care_before_ae_attendance'] ==1))]\n",
    "negative_covid_patients_c = df[df['ae_attendance_hosp_discharge']==0 | ((df['ae_attendance_hosp_discharge']==1) & ((df['ae_attendance_covid_status']==0) & (df['ae_attendance_respiratory_status']==1) & (df['positive_covid_test_before_ae_attendance'] ==0) & (df['covid_primary_care_before_ae_attendance'] ==0)))]\n",
    "\n",
    "\n",
    "model_c_patients_positive = set(list(positive_covid_patients_c['patient_id']))\n",
    "model_c_patients_negative = set(list(negative_covid_patients_c['patient_id']))\n",
    "\n",
    "\n",
    "sus_pos_ecds_pos = len(list(set(sus_patients_positive) & set(model_c_patients_positive)))\n",
    "sus_pos_ecds_neg = len(list(set(sus_patients_positive) & set(model_c_patients_negative)))\n",
    "sus_neg_ecds_pos = len(list(set(sus_patients_negative) & set(model_c_patients_positive)))\n",
    "sus_neg_ecds_neg = len(list(set(sus_patients_negative) & set(model_c_patients_negative)))\n",
    "\n",
    "sensitivity_c = (sus_pos_ecds_pos/(sus_pos_ecds_pos + sus_pos_ecds_neg))*100\n",
    "specificity_c = (sus_neg_ecds_neg/(sus_neg_ecds_pos + sus_neg_ecds_neg))*100\n",
    "PPV_c = (sus_pos_ecds_pos/(sus_pos_ecds_pos + sus_neg_ecds_pos))*100\n",
    "NPV_c = (sus_neg_ecds_neg/(sus_neg_ecds_neg + sus_pos_ecds_neg))*100\n",
    "MCC_c = ((sus_pos_ecds_pos * sus_neg_ecds_neg)-(sus_neg_ecds_pos * sus_pos_ecds_neg))/math.sqrt((sus_pos_ecds_pos + sus_neg_ecds_pos)*(sus_pos_ecds_neg+sus_neg_ecds_neg)*(sus_pos_ecds_pos + sus_pos_ecds_neg)*(sus_neg_ecds_pos+sus_neg_ecds_neg))\n",
    "\n",
    "output = pd.DataFrame([[sus_pos_ecds_pos, sus_neg_ecds_pos, (sus_pos_ecds_pos + sus_neg_ecds_pos)], [sus_pos_ecds_neg, sus_neg_ecds_neg, (sus_pos_ecds_neg + sus_neg_ecds_neg)], [(sus_pos_ecds_pos+sus_pos_ecds_neg), (sus_neg_ecds_pos+sus_neg_ecds_neg), (sus_pos_ecds_pos + sus_pos_ecds_neg + sus_neg_ecds_pos + sus_neg_ecds_neg)]], columns=[\"SUS-positive\", \"SUS-negative\", \"Total\"], index=[\"ECDS-positive\", \"ECDS-negative\", \"Total\"])\n",
    "# output.to_csv('output/model_c.csv')\n",
    "# dictionary of model results\n",
    "\n",
    "performance_dict = {\n",
    "    \"A\": {\n",
    "        \"sensitivity\": sensitivity_a,\n",
    "        \"specificity\": specificity_a,\n",
    "        \"ppv\": PPV_a,\n",
    "        \"npv\": NPV_a,\n",
    "        \"mcc\": MCC_a\n",
    "    },\n",
    "    \"B\": {\n",
    "        \"sensitivity\": sensitivity_b,\n",
    "        \"specificity\": specificity_b,\n",
    "        \"ppv\": PPV_b,\n",
    "        \"npv\": NPV_b,\n",
    "        \"mcc\": MCC_b\n",
    "    },\n",
    "    \"C\": {\n",
    "        \"sensitivity\": sensitivity_c,\n",
    "        \"specificity\": specificity_c,\n",
    "        \"ppv\": PPV_c,\n",
    "        \"npv\": NPV_c,\n",
    "        \"mcc\": MCC_c\n",
    "    }\n",
    "}\n",
    "\n",
    "# with open('output/model_performance.json', 'w') as fp:\n",
    "#     json.dump(performance_dict, fp)\n",
    "\n",
    "#sensitivity\n",
    "sensitivity_dict = {}\n",
    "\n",
    "# drop cov ae\n",
    "positive_covid_patients = df[(df['ae_attendance_hosp_discharge']==1) & ((df['positive_covid_test_before_ae_attendance'] ==1) | (df['covid_primary_care_before_ae_attendance'] ==1))]\n",
    "negative_covid_patients = df[(df['ae_attendance_hosp_discharge']==0) | ((df['ae_attendance_hosp_discharge']==1) &  (df['positive_covid_test_before_ae_attendance'] ==0) & (df['covid_primary_care_before_ae_attendance'] ==0))]\n",
    "model_patients_positive = set(list(positive_covid_patients['patient_id']))\n",
    "model_patients_negative = set(list(negative_covid_patients['patient_id']))\n",
    "sus_pos_ecds_pos = len(list(set(sus_patients_positive) & set(model_patients_positive)))\n",
    "sus_pos_ecds_neg = len(list(set(sus_patients_positive) & set(model_patients_negative)))\n",
    "sus_neg_ecds_pos = len(list(set(sus_patients_negative) & set(model_patients_positive)))\n",
    "sus_neg_ecds_neg = len(list(set(sus_patients_negative) & set(model_patients_negative)))\n",
    "MCC = ((sus_pos_ecds_pos * sus_neg_ecds_neg)-(sus_neg_ecds_pos * sus_pos_ecds_neg))/math.sqrt((sus_pos_ecds_pos + sus_neg_ecds_pos)*(sus_pos_ecds_neg+sus_neg_ecds_neg)*(sus_pos_ecds_pos + sus_pos_ecds_neg)*(sus_neg_ecds_pos+sus_neg_ecds_neg))\n",
    "sensitivity_dict['cov_ae'] = MCC\n",
    "\n",
    "#drop recent pos test\n",
    "positive_covid_patients = df[(df['ae_attendance_hosp_discharge']==1) & ((df['ae_attendance_covid_status']==1)  | (df['covid_primary_care_before_ae_attendance'] ==1))]\n",
    "negative_covid_patients = df[(df['ae_attendance_hosp_discharge']==0) | ((df['ae_attendance_hosp_discharge']==1) & (df['ae_attendance_covid_status']==0)  & (df['covid_primary_care_before_ae_attendance'] ==0))]\n",
    "model_patients_positive = set(list(positive_covid_patients['patient_id']))\n",
    "model_patients_negative = set(list(negative_covid_patients['patient_id']))\n",
    "sus_pos_ecds_pos = len(list(set(sus_patients_positive) & set(model_patients_positive)))\n",
    "sus_pos_ecds_neg = len(list(set(sus_patients_positive) & set(model_patients_negative)))\n",
    "sus_neg_ecds_pos = len(list(set(sus_patients_negative) & set(model_patients_positive)))\n",
    "sus_neg_ecds_neg = len(list(set(sus_patients_negative) & set(model_patients_negative)))\n",
    "MCC = ((sus_pos_ecds_pos * sus_neg_ecds_neg)-(sus_neg_ecds_pos * sus_pos_ecds_neg))/math.sqrt((sus_pos_ecds_pos + sus_neg_ecds_pos)*(sus_pos_ecds_neg+sus_neg_ecds_neg)*(sus_pos_ecds_pos + sus_pos_ecds_neg)*(sus_neg_ecds_pos+sus_neg_ecds_neg))\n",
    "sensitivity_dict['pos_test'] = MCC\n",
    "\n",
    "#drop cov pc\n",
    "positive_covid_patients = df[((df['ae_attendance_hosp_discharge']==1) & ((df['ae_attendance_covid_status']==1) | (df['positive_covid_test_before_ae_attendance'] ==1) ))]\n",
    "negative_covid_patients = df[(df['ae_attendance_hosp_discharge']==0) | ((df['ae_attendance_hosp_discharge']==1) & (df['ae_attendance_covid_status']==0) & (df['positive_covid_test_before_ae_attendance'] ==0))]\n",
    "model_patients_positive = set(list(positive_covid_patients['patient_id']))\n",
    "model_patients_negative = set(list(negative_covid_patients['patient_id']))\n",
    "sus_pos_ecds_pos = len(list(set(sus_patients_positive) & set(model_patients_positive)))\n",
    "sus_pos_ecds_neg = len(list(set(sus_patients_positive) & set(model_patients_negative)))\n",
    "sus_neg_ecds_pos = len(list(set(sus_patients_negative) & set(model_patients_positive)))\n",
    "sus_neg_ecds_neg = len(list(set(sus_patients_negative) & set(model_patients_negative)))\n",
    "MCC = ((sus_pos_ecds_pos * sus_neg_ecds_neg)-(sus_neg_ecds_pos * sus_pos_ecds_neg))/math.sqrt((sus_pos_ecds_pos + sus_neg_ecds_pos)*(sus_pos_ecds_neg+sus_neg_ecds_neg)*(sus_pos_ecds_pos + sus_pos_ecds_neg)*(sus_neg_ecds_pos+sus_neg_ecds_neg))\n",
    "sensitivity_dict['cov_pc'] = MCC\n",
    "\n",
    "\n",
    "# with open('output/sensitivity.json', 'w') as fp:\n",
    "#     json.dump(sensitivity_dict, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "third-domestic",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}