Results Performance.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import csv\n",
    "import os \n",
    "import math\n",
    "import gzip\n",
    "from sklearn import metrics\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analysis of similarity results \n",
    "\n",
    "Analysis of the similarity scores file generated with SML Toolkit, saved in Results directory. \n",
    "Here we will compare the primary diagnoses of patients and assess the ability of similarity scores for each measure computed in predicting a shared primary diagnosis. \n",
    "\n",
    "Scores used to evaluate the measures are: area under receiver operating characteristic curve (AUC) for the predictive power of the measure (given with a 95% confidence interval); mean reciprocal rank (MRR) indicating the reciprocal rank of the first matching primary diagnosis given a descending list of similarity scores; and top 10 accuracy, which gives an overall measure of how frequently there is a shared diagnosis in the 10 most similar patients."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create directory for ROC metrics files \n",
    "if not os.path.exists('ROCmetrics'):\n",
    "    os.makedirs('ROCmetrics')\n",
    "    \n",
    "\n",
    "def results_ROC_metrics(filename):\n",
    "    \n",
    "    ''' apply on the similarity scores file generated '''\n",
    "    \n",
    "    ##################### combine similarity results and diagnoses #####################\n",
    "    \n",
    "    # open diagnoses_ICD      \n",
    "    with gzip.open('MIMIC_data/DIAGNOSES_ICD.csv.gz') as g:\n",
    "        D_df = pd.read_csv(g)\n",
    "    \n",
    "    # strip to primary diagosis only \n",
    "    D_df_tidy = D_df.loc[(D_df['SEQ_NUM']==1.0),]    \n",
    "    D_df_tidy = D_df_tidy.drop(['ROW_ID', 'SUBJECT_ID', 'SEQ_NUM'], axis=1)\n",
    "    \n",
    "    \n",
    "    # open similarity scores file \n",
    "    sim = pd.read_csv(filename, sep='\\t')\n",
    "    \n",
    "    sim = sim.rename(mapper={'e1':'HADM_ID', 'e2':'HADM_ID2'}, axis=1, inplace=False)\n",
    "    \n",
    "    \n",
    "    D_df_tidy = D_df_tidy.set_index('HADM_ID')    \n",
    "\n",
    "    # merge similarity scores and primary diagnoses based on first ID\n",
    "    # to assign the first patient primary diagnosis by HADM_ID\n",
    "    merge1 = sim.merge(D_df_tidy, how='inner', on='HADM_ID')\n",
    "    merge1 = merge1.rename(mapper={'ICD9_CODE':'ICD9_CODE1'}, axis=1)\n",
    "    \n",
    "    D_df_2 = D_df_tidy.copy()\n",
    "    D_df_2.index.names = ['HADM_ID2']\n",
    "    \n",
    "    # merge again based on second ID to assign the second patient diagnosis \n",
    "    data = merge1.merge(D_df_2, how='inner', on='HADM_ID2')\n",
    "    \n",
    "    data = data.rename(mapper={'ICD9_CODE':'ICD9_CODE2'}, axis=1)    \n",
    "    \n",
    "    # create a binary column for shared primary diagnosis    \n",
    "    data['shared'] = 0    \n",
    "    data.loc[(data['ICD9_CODE1'] == data['ICD9_CODE2']), 'shared'] = 1 # set to 1 if primary ICD9 diagnoses match    \n",
    "    \n",
    "        \n",
    "    #'data' is now our similarity scores with primary+shared diagnosis\n",
    "    \n",
    "    \n",
    "    ##################### get MRR and top 10 accuracy for each patient #####################\n",
    "    \n",
    "    res = data.copy()\n",
    "    res = res.drop(['HADM_ID2', 'ICD9_CODE1', 'ICD9_CODE2'], axis=1)\n",
    "    res = res.set_index('HADM_ID')\n",
    "\n",
    "    mrr_res = pd.DataFrame(index=res.columns[:-1]) # set up dataframe for MRR with measures as index\n",
    "    \n",
    "    top10_res = pd.DataFrame(index=res.columns[:-1]) # set up dataframe for top 10 accuracy scores \n",
    "    \n",
    "    for i in list(dict.fromkeys(res.index.tolist())): # loop through patient IDs\n",
    "    \n",
    "        res_i = res.loc[(res.index == i), :] # get all columns of similarity scores for this patient \n",
    "\n",
    "        for (j, col) in enumerate(res_i.columns[:-1]): # loop through measures for each patient\n",
    "        \n",
    "            data_measure = res_i.loc[:, [col, 'shared']]\n",
    "\n",
    "            data_measure_sort = data_measure.sort_values(by=col, ascending=False) # sort by similarity scores high to low\n",
    "            \n",
    "            stri = str(i) # patient ID\n",
    "            \n",
    "            ### top 10 accuracy ###          \n",
    "            \n",
    "            top_ten = data_measure_sort.iloc[0:10,:] # top 10 highest scores for this measure col, for patient 1\n",
    "            \n",
    "            top_ten_shared = top_ten.loc[:,'shared'].tolist() # list of shared diagnosis for top 10\n",
    "            \n",
    "            if all(d == 0 for d in top_ten_shared):\n",
    "                \n",
    "                top10_res.loc[col, stri] = 0 # no match in top 10 for this measure (col) for this patient (stri)\n",
    "            \n",
    "            else:\n",
    "                \n",
    "                top10_res.loc[col, stri] = 1 # there is a match in top 10\n",
    "               \n",
    "            ### reciprocal rank ###\n",
    "\n",
    "            shared_diagnosis = data_measure_sort.loc[:,'shared'].tolist() # list of 1s (shared) and 0s (not shared)\n",
    "\n",
    "            if all(d == 0 for d in shared_diagnosis):\n",
    "\n",
    "                mrr_res.loc[col, stri] = np.nan # if primary diagnosis doesn't match any other patient set as NaN\n",
    "\n",
    "            else:\n",
    "\n",
    "                mrr_res.loc[col, stri] = 1 / (shared_diagnosis.index(1) + 1) # compute RR if there is a shared diagnosis\n",
    "    \n",
    "    #### overall top 10 accuracy and mean RR ####\n",
    "    \n",
    "    top10_res['Top10'] = 0 # create column for score and fill with 0\n",
    "    \n",
    "    for (i, measure) in enumerate(top10_res.index): # loop through measures index\n",
    "        \n",
    "        top10_res.loc[measure, 'Top10'] = (sum(top10_res.iloc[i,:-1]) / len(top10_res.iloc[i,:-1])) #* 100 ?\n",
    "    \n",
    "    \n",
    "    for i in mrr_res.index: # loop through measure names\n",
    "        \n",
    "        mrr_res.loc[i, 'MRR_NA'] = np.nanmean(mrr_res.loc[i,:]) # mean RR \n",
    "        \n",
    "    mrr_res1 = mrr_res.copy()\n",
    "    \n",
    "    mrr_res1 = mrr_res1.fillna(0) # set NaN as 0 to compute MRR in this way as well \n",
    "    \n",
    "    for i in mrr_res1.index:\n",
    "        \n",
    "        mrr_res1.loc[i, 'MRR_0'] = mrr_res1.loc[i,:].mean() # mean RR\n",
    "        \n",
    "    \n",
    "    ##################### get ROC metrics and AUC confidence interval #####################\n",
    "    \n",
    "    data = data.drop(['HADM_ID', 'HADM_ID2'], axis=1) # unnecessary columns\n",
    "    \n",
    "    y_true = data.iloc[:,-1] # shared diagnosis column\n",
    "    \n",
    "    measures = data.iloc[:,:-3] # similarity measure scores columns \n",
    "    \n",
    "        \n",
    "    metrics_df = pd.DataFrame(columns=['measure', 'fpr', 'tpr', 'AUC', 'CI95_lower', 'CI95_upper', 'MRR_NA', 'MRR_0', 'Top10_Acc'])   \n",
    "    \n",
    "    \n",
    "    for (i, col) in enumerate(measures.columns):\n",
    "        \n",
    "        metrics_df.loc[i, 'measure'] = col # add measure name to results \n",
    "        \n",
    "        y_pred = measures.loc[:, col] # similarity scores act as predicted values\n",
    "        \n",
    "        # fit ROC and get AUC from this measure \n",
    "        fpr, tpr, thresholds = metrics.roc_curve(y_true, y_pred, pos_label=1)\n",
    "        \n",
    "        metrics_df.loc[i, 'fpr'] = fpr.tolist() # store in df for later plotting\n",
    "        metrics_df.loc[i, 'tpr'] = tpr.tolist()\n",
    "        \n",
    "        auc = metrics.auc(fpr, tpr)\n",
    "        \n",
    "        metrics_df.loc[i, 'AUC'] = auc\n",
    "        \n",
    "        # get 95% confidence interval for this measure \n",
    "        n1 = y_true.to_list().count(0) # number shared diagnosis = 0\n",
    "        n2 = y_true.to_list().count(1) # number shared diagnosis = 1\n",
    "\n",
    "        q1 = auc/(2 - auc)\n",
    "        q2 = 2*(auc**2)/(1 + auc)\n",
    "\n",
    "        se_auc = math.sqrt(((auc*(1-auc))+((n1-1)*(q1-auc**2))+((n2-1)*(q2-auc**2)))/(n1*n2)) # standard error of auc\n",
    "\n",
    "        z = 1.95996 # z score for alpha/2\n",
    "\n",
    "        CI_upper = auc + (z*se_auc)\n",
    "        CI_lower = auc - (z*se_auc)\n",
    "        \n",
    "        metrics_df.loc[i, 'CI95_lower'] = CI_lower\n",
    "        metrics_df.loc[i, 'CI95_upper'] = CI_upper\n",
    "        \n",
    "        metrics_df.loc[i, 'MRR_NA'] = mrr_res.loc[col, 'MRR_NA'] # MRR computed with NaNs\n",
    "        \n",
    "        metrics_df.loc[i, 'MRR_0'] = mrr_res1.loc[col, 'MRR_0'] # MRR computed with 0s\n",
    "        \n",
    "        metrics_df.loc[i, 'Top10_Acc'] = top10_res.loc[col, 'Top10'] # top 10 accuracy for this measure\n",
    "        \n",
    "        \n",
    "    \n",
    "    sim_file = filename.replace('Results/scores_', '')\n",
    "    metrics_df.to_csv('ROCmetrics/metrics_{}'.format(sim_file), sep='\\t')\n",
    "    ## e.g. metrics_max_indirect_groupwise_combinations.tsv\n",
    "    \n",
    "    return metrics_df\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run on all results and store in a new directory "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get list of results files \n",
    "results_files = ['Results/{}'.format(filename) for filename in os.listdir('Results') if filename.startswith(\"scores_\")]\n",
    "    \n",
    "\n",
    "# get metrics for each (slow)\n",
    "\n",
    "for file in results_files:\n",
    "    \n",
    "    results_ROC_metrics(file)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read in one file to have a look\n",
    "metrics_files = os.listdir('ROCmetrics')\n",
    "direct_metrics = pd.read_csv('ROCmetrics/'+metrics_files[0], sep='\\t', index_col='Unnamed: 0')\n",
    "\n",
    "direct_metrics.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}