evaluate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from collections import defaultdict\n",
    "import csv\n",
    "import json\n",
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from tqdm import tqdm\n",
    "\n",
    "from constants import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_pickle('dataframes/result.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label_set</th>\n",
       "      <th>labels</th>\n",
       "      <th>text</th>\n",
       "      <th>predictions</th>\n",
       "      <th>predictions_sigmoid</th>\n",
       "      <th>predictions_binary</th>\n",
       "      <th>predictions_raw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>89109,144591</td>\n",
       "      <td>[244.9, 250.0, 272.4, 38.91, 38.93, 427.31, 42...</td>\n",
       "      <td>[0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>sex:   f   medicine  allergies: levofloxacin  ...</td>\n",
       "      <td>[38.93, 401.9, 518.81, 96.6, 96.72]</td>\n",
       "      <td>[0.23890312016010284, 0.16363947093486786, 0.4...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[-1.1587026119232178, -1.6313940286636353, -0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>89112,183380</td>\n",
       "      <td>[305.1, 37.23, 401.9, 414.01, 427.31, 88.56]</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, ...</td>\n",
       "      <td>sex:   f   medicine  allergies: patient record...</td>\n",
       "      <td>[37.22, 401.9, 414.01, 88.56]</td>\n",
       "      <td>[0.006338672246783972, 0.06435754150152206, 0....</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, ...</td>\n",
       "      <td>[-5.054727077484131, -2.676779270172119, -2.00...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>89119,191630</td>\n",
       "      <td>[244.9, 272.4, 38.93, 39.95, 428.0, 599.0]</td>\n",
       "      <td>[0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>sex:   m   medicine  allergies: patient record...</td>\n",
       "      <td>[39.95, 428.0]</td>\n",
       "      <td>[0.07126564532518387, 0.11346027255058289, 0.2...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[-2.567408323287964, -2.055873155593872, -1.32...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>89124,123322</td>\n",
       "      <td>[272.4]</td>\n",
       "      <td>[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>sex:   f   medicine  allergies: tegaderm / tax...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[0.05356749892234802, 0.16385111212730408, 0.0...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[-2.8717570304870605, -1.629848599433899, -2.4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>89124,189821</td>\n",
       "      <td>[272.4, 38.93, 486, 599.0, 96.04, 96.6, 96.72]</td>\n",
       "      <td>[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>sex:   f   orthopaedics  allergies: patient re...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[0.009319636039435863, 0.08270774781703949, 0....</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[-4.666268348693848, -2.4061129093170166, -2.0...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             id                                          label_set  \\\n",
       "0  89109,144591  [244.9, 250.0, 272.4, 38.91, 38.93, 427.31, 42...   \n",
       "1  89112,183380       [305.1, 37.23, 401.9, 414.01, 427.31, 88.56]   \n",
       "2  89119,191630         [244.9, 272.4, 38.93, 39.95, 428.0, 599.0]   \n",
       "3  89124,123322                                            [272.4]   \n",
       "4  89124,189821     [272.4, 38.93, 486, 599.0, 96.04, 96.6, 96.72]   \n",
       "\n",
       "                                              labels  \\\n",
       "0  [0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "1  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, ...   \n",
       "2  [0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "3  [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "4  [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "\n",
       "                                                text  \\\n",
       "0  sex:   f   medicine  allergies: levofloxacin  ...   \n",
       "1  sex:   f   medicine  allergies: patient record...   \n",
       "2  sex:   m   medicine  allergies: patient record...   \n",
       "3  sex:   f   medicine  allergies: tegaderm / tax...   \n",
       "4  sex:   f   orthopaedics  allergies: patient re...   \n",
       "\n",
       "                           predictions  \\\n",
       "0  [38.93, 401.9, 518.81, 96.6, 96.72]   \n",
       "1        [37.22, 401.9, 414.01, 88.56]   \n",
       "2                       [39.95, 428.0]   \n",
       "3                                   []   \n",
       "4                                   []   \n",
       "\n",
       "                                 predictions_sigmoid  \\\n",
       "0  [0.23890312016010284, 0.16363947093486786, 0.4...   \n",
       "1  [0.006338672246783972, 0.06435754150152206, 0....   \n",
       "2  [0.07126564532518387, 0.11346027255058289, 0.2...   \n",
       "3  [0.05356749892234802, 0.16385111212730408, 0.0...   \n",
       "4  [0.009319636039435863, 0.08270774781703949, 0....   \n",
       "\n",
       "                                  predictions_binary  \\\n",
       "0  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "1  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, ...   \n",
       "2  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "3  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "4  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "\n",
       "                                     predictions_raw  \n",
       "0  [-1.1587026119232178, -1.6313940286636353, -0....  \n",
       "1  [-5.054727077484131, -2.676779270172119, -2.00...  \n",
       "2  [-2.567408323287964, -2.055873155593872, -1.32...  \n",
       "3  [-2.8717570304870605, -1.629848599433899, -2.4...  \n",
       "4  [-4.666268348693848, -2.4061129093170166, -2.0...  "
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat = np.array(list((df['predictions_binary'])))\n",
    "yhat_raw = np.array(list((df['predictions_raw'])))\n",
    "y = np.array(list(df['labels']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def all_metrics(yhat, y, k=8, yhat_raw=None, calc_auc=True):\n",
    "    names = ['acc', 'prec', 'rec', 'f1']\n",
    "    \n",
    "    # macro\n",
    "    macro = all_macro(yhat, y)\n",
    "    \n",
    "    # micro\n",
    "    ymic = y.ravel()\n",
    "    yhatmic = yhat.ravel()\n",
    "    micro = all_micro(yhatmic, ymic)\n",
    "    \n",
    "    metrics = {names[i] + '_macro': macro[i] for i in range(len(macro))}\n",
    "    \n",
    "    metrics.update({names[i] + '_micro': micro[i] for i in range(len(micro))})\n",
    "    \n",
    "    # AUC and @k\n",
    "    if yhat_raw is not None and calc_auc:\n",
    "        if type(k) != list:\n",
    "            k = [k]\n",
    "        for k_i in k:\n",
    "            rec_at_k = recall_at_k(yhat_raw, y, k_i)\n",
    "            metrics['rec_at_%d' % k_i] = rec_at_k\n",
    "            prec_at_k = precision_at_k(yhat_raw, y, k_i)\n",
    "            metrics['prec_at_%d' % k_i] = prec_at_k\n",
    "            metrics['f1_at_%d' % k_i] = 2*(prec_at_k*rec_at_k)/(prec_at_k+rec_at_k)\n",
    "            \n",
    "        roc_auc = auc_metrics(yhat_raw, y, ymic)\n",
    "        metrics.update(roc_auc)\n",
    "        \n",
    "    return metrics\n",
    "\n",
    "def all_macro(yhat, y):\n",
    "    return macro_accuracy(yhat, y), macro_precision(yhat, y), macro_recall(yhat, y), macro_f1(yhat, y)\n",
    "\n",
    "def all_micro(yhatmic, ymic):\n",
    "    return micro_accuracy(yhatmic, ymic), micro_precision(yhatmic, ymic), micro_recall(yhatmic, ymic), micro_f1(yhatmic, ymic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Axis = 0 represents label level whereas axis = 1 represents instance level\n",
    "\n",
    "def union_size(yhat, y, axis):\n",
    "    return np.logical_or(yhat, y).sum(axis=axis).astype(float)\n",
    "\n",
    "def intersect_size(yhat, y, axis):\n",
    "    return np.logical_and(yhat, y).sum(axis=axis).astype(float)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Macro Metrics\n",
    "Calculate metrics for each label and average across the labels. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def macro_accuracy(yhat, y):\n",
    "    num = intersect_size(yhat, y, 0) # This calculates true positives (1, 1) at a label level\n",
    "    den = union_size(yhat, y, 0) + 1e-10 # This calculates all (1, 1), (1, 0), (0, 1) examples. +1e-10 for numerical stability.\n",
    "    return np.mean(num/den)\n",
    "\n",
    "def macro_precision(yhat, y):\n",
    "    num = intersect_size(yhat, y, 0) # TP\n",
    "    den = yhat.sum(axis=0) + 1e-10 # All the ones in yhat i.e. TP + FP\n",
    "    return np.mean(num/den)\n",
    "\n",
    "def macro_recall(yhat, y):\n",
    "    num = intersect_size(yhat, y, 0)\n",
    "    den = y.sum(axis=0) + 1e-10 # All the ones in y i.e. TP + FN\n",
    "    return np.mean(num/den)\n",
    "\n",
    "def macro_f1(yhat, y):\n",
    "    prec = macro_precision(yhat, y)\n",
    "    rec = macro_recall(yhat, y)\n",
    "    if prec + rec == 0:\n",
    "        f1 = 0\n",
    "    else:\n",
    "        f1 = 2*(prec*rec)/(prec+rec)\n",
    "    return f1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Instance Averaged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "def inst_precision(yhat, y):\n",
    "    num = intersect_size(yhat, y, 1) / yhat.sum(axis=1)\n",
    "    #correct for divide-by-zeros\n",
    "    num[np.isnan(num)] = 0.\n",
    "    return np.mean(num)\n",
    "\n",
    "def inst_recall(yhat, y):\n",
    "    num = intersect_size(yhat, y, 1) / y.sum(axis=1)\n",
    "    #correct for divide-by-zeros\n",
    "    num[np.isnan(num)] = 0.\n",
    "    return np.mean(num)\n",
    "\n",
    "def inst_f1(yhat, y):\n",
    "    prec = inst_precision(yhat, y)\n",
    "    rec = inst_recall(yhat, y)\n",
    "    f1 = 2*(prec*rec)/(prec+rec)\n",
    "    return f1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## @k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recall_at_k(yhat_raw, y, k):\n",
    "    #num true labels in top k predictions / num true labels\n",
    "    sortd = np.argsort(yhat_raw)[:,::-1]\n",
    "    topk = sortd[:,:k]\n",
    "\n",
    "    #get recall at k for each example\n",
    "    vals = []\n",
    "    for i, tk in enumerate(topk):\n",
    "        num_true_in_top_k = y[i,tk].sum()\n",
    "        denom = y[i,:].sum()\n",
    "        vals.append(num_true_in_top_k / float(denom))\n",
    "\n",
    "    vals = np.array(vals)\n",
    "    vals[np.isnan(vals)] = 0.\n",
    "\n",
    "    return np.mean(vals)\n",
    "\n",
    "def precision_at_k(yhat_raw, y, k):\n",
    "    #num true labels in top k predictions / k\n",
    "    sortd = np.argsort(yhat_raw)[:,::-1]\n",
    "    topk = sortd[:,:k]\n",
    "\n",
    "    #get precision at k for each example\n",
    "    vals = []\n",
    "    for i, tk in enumerate(topk):\n",
    "        if len(tk) > 0:\n",
    "            num_true_in_top_k = y[i,tk].sum()\n",
    "            denom = len(tk)\n",
    "            vals.append(num_true_in_top_k / float(denom))\n",
    "\n",
    "    return np.mean(vals)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Micro metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "def micro_accuracy(yhatmic, ymic):\n",
    "    return intersect_size(yhatmic, ymic, 0) / union_size(yhatmic, ymic, 0)\n",
    "\n",
    "def micro_precision(yhatmic, ymic):\n",
    "    return intersect_size(yhatmic, ymic, 0) / yhatmic.sum(axis=0)\n",
    "\n",
    "def micro_recall(yhatmic, ymic):\n",
    "    return intersect_size(yhatmic, ymic, 0) / ymic.sum(axis=0)\n",
    "\n",
    "def micro_f1(yhatmic, ymic):\n",
    "    prec = micro_precision(yhatmic, ymic)\n",
    "    rec = micro_recall(yhatmic, ymic)\n",
    "    if prec + rec == 0:\n",
    "        f1 = 0.\n",
    "    else:\n",
    "        f1 = 2*(prec*rec)/(prec+rec)\n",
    "    return f1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AUC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "def auc_metrics(yhat_raw, y, ymic):\n",
    "    if yhat_raw.shape[0] <= 1:\n",
    "        return\n",
    "    fpr = {}\n",
    "    tpr = {}\n",
    "    roc_auc = {}\n",
    "    #get AUC for each label individually\n",
    "    relevant_labels = []\n",
    "    auc_labels = {}\n",
    "    for i in range(y.shape[1]):\n",
    "        #only if there are true positives for this label\n",
    "        if y[:,i].sum() > 0:\n",
    "            fpr[i], tpr[i], _ = roc_curve(y[:,i], yhat_raw[:,i])\n",
    "            if len(fpr[i]) > 1 and len(tpr[i]) > 1:\n",
    "                auc_score = auc(fpr[i], tpr[i])\n",
    "                if not np.isnan(auc_score): \n",
    "                    auc_labels[\"auc_%d\" % i] = auc_score\n",
    "                    relevant_labels.append(i)\n",
    "\n",
    "    #macro-AUC: just average the auc scores\n",
    "    aucs = []\n",
    "    for i in relevant_labels:\n",
    "        aucs.append(auc_labels['auc_%d' % i])\n",
    "    roc_auc['auc_macro'] = np.mean(aucs)\n",
    "\n",
    "    #micro-AUC: just look at each individual prediction\n",
    "    yhatmic = yhat_raw.ravel()\n",
    "    fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(ymic, yhatmic) \n",
    "    roc_auc[\"auc_micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"])\n",
    "\n",
    "    return roc_auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics = all_metrics(yhat, y, k=8, yhat_raw=yhat_raw, calc_auc=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'acc_macro': 0.19345074135362034,\n",
       " 'acc_micro': 0.2543137084886846,\n",
       " 'auc_macro': 0.791074331774784,\n",
       " 'auc_micro': 0.8397917379414975,\n",
       " 'f1_at_8': 0.4721469682885988,\n",
       " 'f1_macro': 0.30314291789090314,\n",
       " 'f1_micro': 0.405502557721554,\n",
       " 'prec_at_8': 0.40202898550724636,\n",
       " 'prec_macro': 0.49631257038914556,\n",
       " 'prec_micro': 0.7263496780584447,\n",
       " 'rec_at_8': 0.5718905523241907,\n",
       " 'rec_macro': 0.21821250175767504,\n",
       " 'rec_micro': 0.2812619869581895}"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}