Redundnacy_Exploration.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [],
   "source": [
    "from icd9cms import search\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import defaultdict\n",
    "import json\n",
    "from itertools import chain\n",
    "import ipywidgets as widgets\n",
    "from IPython.display import display\n",
    "import re\n",
    "from collections import Counter\n",
    "import difflib\n",
    "from differ import html_diffs, align_seqs, sentencize, \\\n",
    "    diff_ratio, show_diffs, compute_sequential_diff_metrics, \\\n",
    "    split_notes, compute_avgs\n",
    "from datasets import list_metrics, load_metric\n",
    "import seaborn as sns\n",
    "from glob import glob\n",
    "from IPython.display import clear_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.path.exists('../data/mimic_3_all_notes_with_20_occurrences_of_primary_condition.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "codes = pd.read_csv('../data/primary_codes.csv')\n",
    "sorted_primary_codes_df =codes['icd9_code'].value_counts().rename_axis('code').reset_index()\n",
    "sorted_primary_codes_df['long_title'] = sorted_primary_codes_df['code'].apply(lambda c: search(c).long_desc or search(c).short_desc)\n",
    "sorted_primary_codes_df.columns = ['code', 'count', 'long_title']\n",
    "sorted_primary_codes_df = sorted_primary_codes_df[~sorted_primary_codes_df.code.str.startswith('V3')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>code</th>\n",
       "      <th>count</th>\n",
       "      <th>long_title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>41401</td>\n",
       "      <td>3498</td>\n",
       "      <td>Coronary atherosclerosis of native coronary ar...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    code  count                                         long_title\n",
       "1  41401   3498  Coronary atherosclerosis of native coronary ar..."
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted_primary_codes_df[sorted_primary_codes_df.code.str.startswith('41401')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>code</th>\n",
       "      <th>count</th>\n",
       "      <th>long_title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>41401</td>\n",
       "      <td>3498</td>\n",
       "      <td>Coronary atherosclerosis of native coronary ar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>41071</td>\n",
       "      <td>1751</td>\n",
       "      <td>Subendocardial infarction, initial episode of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>41041</td>\n",
       "      <td>482</td>\n",
       "      <td>Acute myocardial infarction of other inferior ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>41011</td>\n",
       "      <td>482</td>\n",
       "      <td>Acute myocardial infarction of other anterior ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>41519</td>\n",
       "      <td>343</td>\n",
       "      <td>Other pulmonary embolism and infarction</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     code  count                                         long_title\n",
       "1   41401   3498  Coronary atherosclerosis of native coronary ar...\n",
       "4   41071   1751  Subendocardial infarction, initial episode of ...\n",
       "15  41041    482  Acute myocardial infarction of other inferior ...\n",
       "16  41011    482  Acute myocardial infarction of other anterior ...\n",
       "20  41519    343            Other pulmonary embolism and infarction"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted_primary_codes_df[sorted_primary_codes_df.code.str.startswith('41')].head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Cerebral embolism with cerebral infarction',\n",
       " 'Cerebral artery occlusion, unspecified with cerebral infarction',\n",
       " 'Cerebral thrombosis with cerebral infarction',\n",
       " 'Cerebral artery occlusion, unspecified without mention of cerebral infarction',\n",
       " 'Cerebral embolism without mention of cerebral infarction']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted_primary_codes_df[sorted_primary_codes_df.code.str.startswith('434')].long_title.tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 41401 Notes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "notes = pd.read_csv('../data/41401_notes.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "discharge_notes = notes[(notes.category == 'Discharge summary') & (notes.description != 'Addendum')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "care_notes_df, discharge_notes_df = split_notes(notes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "care_notes_df['cat_descs'] = care_notes_df.apply(lambda r: [(f'{c.strip()}:{d}') for c,d in zip(r.categories, r.descriptions)], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-01-19 12:49:06,517: Removing /Users/tom/.cache/huggingface/metrics/bert_score/default/default_experiment-1-0.arrow\n",
      "2021-01-19 12:49:06,524: Removing /Users/tom/.cache/huggingface/metrics/rouge/default/default_experiment-1-0.arrow\n"
     ]
    }
   ],
   "source": [
    "results_df = compute_diff(stroke_notes_subset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 386,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = pd.concat([rougeL, results_df.loc[:, [c for c in results_df.columns if 'rougeL' in c]]], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 399,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = results_df.loc[:, ['category', 'description', 'avg_diff_ratio', 'rougeL_recall_med', 'rougeL_recall_iqr', 'rougeL_precision_med', 'rougeL_precision_iqr', \n",
    "                            'bert-score_recall_med', 'bert-score_recall_iqr', 'bert-score_precision_med', 'bert-score_precision_iqr']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 400,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>description</th>\n",
       "      <th>avg_diff_ratio</th>\n",
       "      <th>rougeL_recall_med</th>\n",
       "      <th>rougeL_recall_iqr</th>\n",
       "      <th>rougeL_precision_med</th>\n",
       "      <th>rougeL_precision_iqr</th>\n",
       "      <th>bert-score_recall_med</th>\n",
       "      <th>bert-score_recall_iqr</th>\n",
       "      <th>bert-score_precision_med</th>\n",
       "      <th>bert-score_precision_iqr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Nursing</td>\n",
       "      <td>Nursing Transfer Note</td>\n",
       "      <td>0.872569</td>\n",
       "      <td>0.997854</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.621658</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.982755</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.744808</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Nutrition</td>\n",
       "      <td>Clinical Nutrition Note</td>\n",
       "      <td>0.154612</td>\n",
       "      <td>0.921967</td>\n",
       "      <td>0.323136</td>\n",
       "      <td>0.523822</td>\n",
       "      <td>0.138380</td>\n",
       "      <td>0.916548</td>\n",
       "      <td>0.203272</td>\n",
       "      <td>0.601892</td>\n",
       "      <td>0.248073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Physician</td>\n",
       "      <td>Intensivist Note</td>\n",
       "      <td>0.234087</td>\n",
       "      <td>0.651907</td>\n",
       "      <td>0.127212</td>\n",
       "      <td>0.692593</td>\n",
       "      <td>0.118770</td>\n",
       "      <td>0.857477</td>\n",
       "      <td>0.060351</td>\n",
       "      <td>0.853575</td>\n",
       "      <td>0.046447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Nursing</td>\n",
       "      <td>Nursing Progress Note</td>\n",
       "      <td>0.440298</td>\n",
       "      <td>0.632599</td>\n",
       "      <td>0.675619</td>\n",
       "      <td>0.666618</td>\n",
       "      <td>0.682953</td>\n",
       "      <td>0.701072</td>\n",
       "      <td>0.424521</td>\n",
       "      <td>0.730515</td>\n",
       "      <td>0.501814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Radiology</td>\n",
       "      <td>CHEST (PORTABLE AP)</td>\n",
       "      <td>0.088830</td>\n",
       "      <td>0.417902</td>\n",
       "      <td>0.021350</td>\n",
       "      <td>0.540940</td>\n",
       "      <td>0.270825</td>\n",
       "      <td>0.791767</td>\n",
       "      <td>0.009032</td>\n",
       "      <td>0.827279</td>\n",
       "      <td>0.074302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Radiology</td>\n",
       "      <td>CT HEAD W/O CONTRAST</td>\n",
       "      <td>0.088893</td>\n",
       "      <td>0.411212</td>\n",
       "      <td>0.231364</td>\n",
       "      <td>0.399956</td>\n",
       "      <td>0.201956</td>\n",
       "      <td>0.755250</td>\n",
       "      <td>0.107351</td>\n",
       "      <td>0.772630</td>\n",
       "      <td>0.083226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Social Work</td>\n",
       "      <td>Social Work Progress Note</td>\n",
       "      <td>0.073434</td>\n",
       "      <td>0.383871</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.162568</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.702947</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.552549</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ECG</td>\n",
       "      <td>Report</td>\n",
       "      <td>0.186325</td>\n",
       "      <td>0.331579</td>\n",
       "      <td>0.167044</td>\n",
       "      <td>0.231640</td>\n",
       "      <td>0.159380</td>\n",
       "      <td>0.569639</td>\n",
       "      <td>0.144755</td>\n",
       "      <td>0.428310</td>\n",
       "      <td>0.057963</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      category                description  avg_diff_ratio  rougeL_recall_med  \\\n",
       "2      Nursing      Nursing Transfer Note        0.872569           0.997854   \n",
       "3    Nutrition    Clinical Nutrition Note        0.154612           0.921967   \n",
       "4   Physician            Intensivist Note        0.234087           0.651907   \n",
       "1      Nursing      Nursing Progress Note        0.440298           0.632599   \n",
       "5    Radiology        CHEST (PORTABLE AP)        0.088830           0.417902   \n",
       "6    Radiology       CT HEAD W/O CONTRAST        0.088893           0.411212   \n",
       "7  Social Work  Social Work Progress Note        0.073434           0.383871   \n",
       "0          ECG                     Report        0.186325           0.331579   \n",
       "\n",
       "   rougeL_recall_iqr  rougeL_precision_med  rougeL_precision_iqr  \\\n",
       "2           0.000000              0.621658              0.000000   \n",
       "3           0.323136              0.523822              0.138380   \n",
       "4           0.127212              0.692593              0.118770   \n",
       "1           0.675619              0.666618              0.682953   \n",
       "5           0.021350              0.540940              0.270825   \n",
       "6           0.231364              0.399956              0.201956   \n",
       "7           0.000000              0.162568              0.000000   \n",
       "0           0.167044              0.231640              0.159380   \n",
       "\n",
       "   bert-score_recall_med  bert-score_recall_iqr  bert-score_precision_med  \\\n",
       "2               0.982755               0.000000                  0.744808   \n",
       "3               0.916548               0.203272                  0.601892   \n",
       "4               0.857477               0.060351                  0.853575   \n",
       "1               0.701072               0.424521                  0.730515   \n",
       "5               0.791767               0.009032                  0.827279   \n",
       "6               0.755250               0.107351                  0.772630   \n",
       "7               0.702947               0.000000                  0.552549   \n",
       "0               0.569639               0.144755                  0.428310   \n",
       "\n",
       "   bert-score_precision_iqr  \n",
       "2                  0.000000  \n",
       "3                  0.248073  \n",
       "4                  0.046447  \n",
       "1                  0.501814  \n",
       "5                  0.074302  \n",
       "6                  0.083226  \n",
       "7                  0.000000  \n",
       "0                  0.057963  "
      ]
     },
     "execution_count": 400,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores.sort_values(['rougeL_recall_med', 'rougeL_precision_med'], ascending=[False, True])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 402,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>description</th>\n",
       "      <th>avg_diff_ratio</th>\n",
       "      <th>rougeL_recall_med</th>\n",
       "      <th>rougeL_recall_iqr</th>\n",
       "      <th>rougeL_precision_med</th>\n",
       "      <th>rougeL_precision_iqr</th>\n",
       "      <th>bert-score_recall_med</th>\n",
       "      <th>bert-score_recall_iqr</th>\n",
       "      <th>bert-score_precision_med</th>\n",
       "      <th>bert-score_precision_iqr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Nursing</td>\n",
       "      <td>Nursing Transfer Note</td>\n",
       "      <td>0.872569</td>\n",
       "      <td>0.997854</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.621658</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.982755</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.744808</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Nutrition</td>\n",
       "      <td>Clinical Nutrition Note</td>\n",
       "      <td>0.154612</td>\n",
       "      <td>0.921967</td>\n",
       "      <td>0.323136</td>\n",
       "      <td>0.523822</td>\n",
       "      <td>0.138380</td>\n",
       "      <td>0.916548</td>\n",
       "      <td>0.203272</td>\n",
       "      <td>0.601892</td>\n",
       "      <td>0.248073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Physician</td>\n",
       "      <td>Intensivist Note</td>\n",
       "      <td>0.234087</td>\n",
       "      <td>0.651907</td>\n",
       "      <td>0.127212</td>\n",
       "      <td>0.692593</td>\n",
       "      <td>0.118770</td>\n",
       "      <td>0.857477</td>\n",
       "      <td>0.060351</td>\n",
       "      <td>0.853575</td>\n",
       "      <td>0.046447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Radiology</td>\n",
       "      <td>CHEST (PORTABLE AP)</td>\n",
       "      <td>0.088830</td>\n",
       "      <td>0.417902</td>\n",
       "      <td>0.021350</td>\n",
       "      <td>0.540940</td>\n",
       "      <td>0.270825</td>\n",
       "      <td>0.791767</td>\n",
       "      <td>0.009032</td>\n",
       "      <td>0.827279</td>\n",
       "      <td>0.074302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Radiology</td>\n",
       "      <td>CT HEAD W/O CONTRAST</td>\n",
       "      <td>0.088893</td>\n",
       "      <td>0.411212</td>\n",
       "      <td>0.231364</td>\n",
       "      <td>0.399956</td>\n",
       "      <td>0.201956</td>\n",
       "      <td>0.755250</td>\n",
       "      <td>0.107351</td>\n",
       "      <td>0.772630</td>\n",
       "      <td>0.083226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Social Work</td>\n",
       "      <td>Social Work Progress Note</td>\n",
       "      <td>0.073434</td>\n",
       "      <td>0.383871</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.162568</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.702947</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.552549</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Nursing</td>\n",
       "      <td>Nursing Progress Note</td>\n",
       "      <td>0.440298</td>\n",
       "      <td>0.632599</td>\n",
       "      <td>0.675619</td>\n",
       "      <td>0.666618</td>\n",
       "      <td>0.682953</td>\n",
       "      <td>0.701072</td>\n",
       "      <td>0.424521</td>\n",
       "      <td>0.730515</td>\n",
       "      <td>0.501814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ECG</td>\n",
       "      <td>Report</td>\n",
       "      <td>0.186325</td>\n",
       "      <td>0.331579</td>\n",
       "      <td>0.167044</td>\n",
       "      <td>0.231640</td>\n",
       "      <td>0.159380</td>\n",
       "      <td>0.569639</td>\n",
       "      <td>0.144755</td>\n",
       "      <td>0.428310</td>\n",
       "      <td>0.057963</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      category                description  avg_diff_ratio  rougeL_recall_med  \\\n",
       "2      Nursing      Nursing Transfer Note        0.872569           0.997854   \n",
       "3    Nutrition    Clinical Nutrition Note        0.154612           0.921967   \n",
       "4   Physician            Intensivist Note        0.234087           0.651907   \n",
       "5    Radiology        CHEST (PORTABLE AP)        0.088830           0.417902   \n",
       "6    Radiology       CT HEAD W/O CONTRAST        0.088893           0.411212   \n",
       "7  Social Work  Social Work Progress Note        0.073434           0.383871   \n",
       "1      Nursing      Nursing Progress Note        0.440298           0.632599   \n",
       "0          ECG                     Report        0.186325           0.331579   \n",
       "\n",
       "   rougeL_recall_iqr  rougeL_precision_med  rougeL_precision_iqr  \\\n",
       "2           0.000000              0.621658              0.000000   \n",
       "3           0.323136              0.523822              0.138380   \n",
       "4           0.127212              0.692593              0.118770   \n",
       "5           0.021350              0.540940              0.270825   \n",
       "6           0.231364              0.399956              0.201956   \n",
       "7           0.000000              0.162568              0.000000   \n",
       "1           0.675619              0.666618              0.682953   \n",
       "0           0.167044              0.231640              0.159380   \n",
       "\n",
       "   bert-score_recall_med  bert-score_recall_iqr  bert-score_precision_med  \\\n",
       "2               0.982755               0.000000                  0.744808   \n",
       "3               0.916548               0.203272                  0.601892   \n",
       "4               0.857477               0.060351                  0.853575   \n",
       "5               0.791767               0.009032                  0.827279   \n",
       "6               0.755250               0.107351                  0.772630   \n",
       "7               0.702947               0.000000                  0.552549   \n",
       "1               0.701072               0.424521                  0.730515   \n",
       "0               0.569639               0.144755                  0.428310   \n",
       "\n",
       "   bert-score_precision_iqr  \n",
       "2                  0.000000  \n",
       "3                  0.248073  \n",
       "4                  0.046447  \n",
       "5                  0.074302  \n",
       "6                  0.083226  \n",
       "7                  0.000000  \n",
       "1                  0.501814  \n",
       "0                  0.057963  "
      ]
     },
     "execution_count": 402,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores.sort_values(['bert-score_recall_med', 'bert-score_precision_med'], ascending=[False, True])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sort and re-rank, set of documents, extract phenotypes to test how excluding \n",
    "# the set of redundant document effects phenotype extraction??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plots between max / min... as things progress... - heavy bias between how the clinician records the notes.."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 479,
   "metadata": {},
   "outputs": [],
   "source": [
    "avgs = compute_avgs(more_notes_results_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 495,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat_desc</th>\n",
       "      <th>micro_avg</th>\n",
       "      <th>rg_rec_avg</th>\n",
       "      <th>rg_prec_avg</th>\n",
       "      <th>bs_rec_avg</th>\n",
       "      <th>bs_prec_avg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Nursing:Nursing Progress Note</td>\n",
       "      <td>0.227994</td>\n",
       "      <td>0.345551</td>\n",
       "      <td>0.348121</td>\n",
       "      <td>0.496873</td>\n",
       "      <td>0.494258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Physician :Intensivist Note</td>\n",
       "      <td>0.169265</td>\n",
       "      <td>0.594225</td>\n",
       "      <td>0.597550</td>\n",
       "      <td>0.856726</td>\n",
       "      <td>0.858064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Radiology:CT HEAD W/O CONTRAST</td>\n",
       "      <td>0.084276</td>\n",
       "      <td>0.400569</td>\n",
       "      <td>0.471806</td>\n",
       "      <td>0.732597</td>\n",
       "      <td>0.739952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Nursing:Nursing Transfer Note</td>\n",
       "      <td>0.421328</td>\n",
       "      <td>0.725352</td>\n",
       "      <td>0.545839</td>\n",
       "      <td>0.701177</td>\n",
       "      <td>0.572983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nutrition:Clinical Nutrition Note</td>\n",
       "      <td>0.278534</td>\n",
       "      <td>0.723751</td>\n",
       "      <td>0.591569</td>\n",
       "      <td>0.789677</td>\n",
       "      <td>0.708747</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            cat_desc  micro_avg  rg_rec_avg  rg_prec_avg  \\\n",
       "0      Nursing:Nursing Progress Note   0.227994    0.345551     0.348121   \n",
       "1        Physician :Intensivist Note   0.169265    0.594225     0.597550   \n",
       "2     Radiology:CT HEAD W/O CONTRAST   0.084276    0.400569     0.471806   \n",
       "3      Nursing:Nursing Transfer Note   0.421328    0.725352     0.545839   \n",
       "4  Nutrition:Clinical Nutrition Note   0.278534    0.723751     0.591569   \n",
       "\n",
       "   bs_rec_avg  bs_prec_avg  \n",
       "0    0.496873     0.494258  \n",
       "1    0.856726     0.858064  \n",
       "2    0.732597     0.739952  \n",
       "3    0.701177     0.572983  \n",
       "4    0.789677     0.708747  "
      ]
     },
     "execution_count": 495,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avgs.loc[:, ['cat_desc', 'micro_avg', 'rg_rec_avg', 'rg_prec_avg', 'bs_rec_avg', 'bs_prec_avg']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare Discharge Summary to last note types via Summarisation Metrics\n",
    "- recall will likely be low, but any non-zero ness is potentially interesting\n",
    "- do neural methods work at all here??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# only take last note...? Or take all notes of each category\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 502,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-01-15 00:19:26,721: Removing /Users/tom/.cache/huggingface/metrics/rouge/default/default_experiment-1-0.arrow\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[Score(precision=0.3333333333333333, recall=1.0, fmeasure=0.5),\n",
       " Score(precision=0.3333333333333333, recall=1.0, fmeasure=0.5)]"
      ]
     },
     "execution_count": 502,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rouge_metric.compute(rouge_types=['rougeL'], use_agregator=False)['rougeL']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_diff_metrics(ds: str, prev_notes: dict):\n",
    "    # calc diff between \n",
    "    rouge_metric = load_metric('rouge')\n",
    "    bert_metric = load_metric('bertscore')\n",
    "    output = defaultdict(list)\n",
    "    for k, note_text in prev_notes.items():\n",
    "        ratio, std_dev, max_ratio, min_ratio, redundant_toks_len, total_toks = diff_ratio(ds, note_text)\n",
    "        output['ratio'].append(ratio)\n",
    "        output['cat_desc'].append(':'.join(k))\n",
    "        rouge_metric.add(prediction=ds, reference=note_text)\n",
    "        bert_metric.add(prediction=ds, reference=note_text)\n",
    "#     bert_scores = bert_metric.compute(lang='en', rescale_with_baseline=True, model_type='xlnet-base-cased')\n",
    "#     output['bs_prec'].extend(bert_scores.precision)\n",
    "#     output['bs_rec'].extend(bert_scores.recall)\n",
    "    rougeL = rouge_metric.compute(rouge_types=['rougeL'], use_agregator=False)['rougeL']\n",
    "    output['rg_prec'].extend([s.precision for s in rougeL])\n",
    "    output['rg_rec'].extend([s.recall for s in rougeL])\n",
    "    return pd.DataFrame(output)\n",
    "    \n",
    "def calc_discharge_summary_redundancy(notes: pd.DataFrame):\n",
    "    summaries = notes[notes.category == 'Discharge summary'].loc[:, ['hadm_id', 'text']]\n",
    "    ds_diffs = []\n",
    "    for r in summaries.itertuples():\n",
    "        hadm_notes = notes[(notes.hadm_id == r.hadm_id) & (notes.category != 'Discharge summary')]\n",
    "        last_notes_per_type = {}\n",
    "        for k, df_g in hadm_notes.groupby(['category', 'description']):\n",
    "            last_notes_per_type[':'.join(k)] = df_g.text.iloc[-1]\n",
    "        ds_diffs.append(calc_diff_metrics(r.text, last_notes_per_type))\n",
    "    return pd.concat(ds_diffs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 518,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-01-15 00:24:18,249: Removing /Users/tom/.cache/huggingface/metrics/rouge/default/default_experiment-1-0.arrow\n",
      "2021-01-15 00:24:21,168: Removing /Users/tom/.cache/huggingface/metrics/rouge/default/default_experiment-1-0.arrow\n"
     ]
    }
   ],
   "source": [
    "calc_discharge_summary_redundancy(stroke_notes_subset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/thomass/anaconda3/envs/transformers/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3072: DtypeWarning: Columns (5,6) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "all_data = pd.read_csv('../data/mimic_3_all_notes_with_20_occurrences_of_primary_condition.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "41463"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_data.hadm_id.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = all_data.loc[:, ['row_id', 'subject_id', 'hadm_id', 'chartdate', 'charttime', 'storetime', 'category', 'description', 'cgid', 'iserror', 'text', 'icd9_code']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data.to_csv('../data/mimic_3_all_notes_with_20_occurrences_of_primary_condition.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "block_size = 500\n",
    "df_splits = []\n",
    "adm_ids = all_data.hadm_id.unique()\n",
    "for block_num in range(int(np.ceil(len(adm_ids) / block_size))):\n",
    "    block_start = block_num * block_size\n",
    "    block_end = (block_num+1) * block_size \n",
    "    ids = adm_ids[block_num * block_size:block_end if block_end < len(adm_ids) else len(adm_ids)]\n",
    "    df_splits.append(all_data[all_data.hadm_id.isin(ids)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from multiprocessing import Pool\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.multiprocessing.set_start_method('spawn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with Pool(3) as p:\n",
    "    ms_res = p.map(compute_sequential_diff_metrics, [df_splits[0], df_splits[1], df_splits[2]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token level / global note redundancy with concurrent notes:\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'results_df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-38-e6218271b972>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Token level / global note redundancy with concurrent notes:'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Overall Average Internote type redundnacy:{results_df.avg_diff_ratio.sum() / results_df.avg_diff_ratio.shape[0]}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'results_df' is not defined"
     ]
    }
   ],
   "source": [
    "print('Token level / global note redundancy with concurrent notes:')\n",
    "print(f'Overall Average Internote type redundnacy:{results_df.avg_diff_ratio.sum() / results_df.avg_diff_ratio.shape[0]}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 43411 Note Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "stroke_notes = pd.read_csv('../data/43411_notes.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "337"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(stroke_notes.hadm_id.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8213, 16)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stroke_notes.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "stroke_subset = stroke_notes[stroke_notes.hadm_id.isin(stroke_notes.hadm_id.unique()[0:2])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(159, 16)"
      ]
     },
     "execution_count": 252,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stroke_subset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [],
   "source": [
    "stroke_subset.to_csv('../data/stroke_subset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "avgs = compute_avgs(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['cat_desc', 'redundant_toks', 'total_toks', 'avg_txt_len', 'macro_avg',\n",
       "       'micro_avg', 'num_instances', 'rg_rec_avg', 'rg_rec_med', 'rg_rec_iqr',\n",
       "       'rg_prec_avg', 'rg_prec_med', 'rg_prec_iqr', 'bs_rec_avg', 'bs_rec_med',\n",
       "       'bs_rec_iqr', 'bs_prec_avg', 'bs_prec_med', 'bs_prec_iqr'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avgs.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "avgs_condensed = avgs.loc[:, ['cat_desc', 'num_instances', 'macro_avg', 'rg_rec_med', 'rg_prec_med', 'bs_rec_med', 'bs_prec_med']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat_desc</th>\n",
       "      <th>macro_avg</th>\n",
       "      <th>num_instances</th>\n",
       "      <th>rg_rec_med</th>\n",
       "      <th>rg_prec_med</th>\n",
       "      <th>bs_rec_med</th>\n",
       "      <th>bs_prec_med</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Physician :Physician Resident Admission Note</td>\n",
       "      <td>0.853431</td>\n",
       "      <td>13</td>\n",
       "      <td>0.998481</td>\n",
       "      <td>0.979633</td>\n",
       "      <td>0.992970</td>\n",
       "      <td>0.992560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Nursing:Nursing Transfer Note</td>\n",
       "      <td>0.590977</td>\n",
       "      <td>159</td>\n",
       "      <td>0.867188</td>\n",
       "      <td>0.757576</td>\n",
       "      <td>0.785322</td>\n",
       "      <td>0.744731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Physician :Physician Resident Progress Note</td>\n",
       "      <td>0.517666</td>\n",
       "      <td>34</td>\n",
       "      <td>0.734496</td>\n",
       "      <td>0.688121</td>\n",
       "      <td>0.896835</td>\n",
       "      <td>0.876756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Physician :Physician Surgical Progress Note</td>\n",
       "      <td>0.738827</td>\n",
       "      <td>7</td>\n",
       "      <td>0.717286</td>\n",
       "      <td>0.637016</td>\n",
       "      <td>0.883960</td>\n",
       "      <td>0.858172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Respiratory :Respiratory Care Shift Note</td>\n",
       "      <td>0.407171</td>\n",
       "      <td>79</td>\n",
       "      <td>0.698630</td>\n",
       "      <td>0.679104</td>\n",
       "      <td>0.751476</td>\n",
       "      <td>0.742840</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        cat_desc  macro_avg  num_instances  \\\n",
       "15  Physician :Physician Resident Admission Note   0.853431             13   \n",
       "6                  Nursing:Nursing Transfer Note   0.590977            159   \n",
       "10   Physician :Physician Resident Progress Note   0.517666             34   \n",
       "19   Physician :Physician Surgical Progress Note   0.738827              7   \n",
       "7       Respiratory :Respiratory Care Shift Note   0.407171             79   \n",
       "\n",
       "    rg_rec_med  rg_prec_med  bs_rec_med  bs_prec_med  \n",
       "15    0.998481     0.979633    0.992970     0.992560  \n",
       "6     0.867188     0.757576    0.785322     0.744731  \n",
       "10    0.734496     0.688121    0.896835     0.876756  \n",
       "19    0.717286     0.637016    0.883960     0.858172  \n",
       "7     0.698630     0.679104    0.751476     0.742840  "
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avgs_condensed.sort_values('rg_rec_med', ascending=False).head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analysis of ALL MIMIC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_mimic_res = pd.concat([pd.read_pickle(f) for f in glob('rouge-and-bert-score-res-dfs/*.pickle')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38685"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_mimic_res.hadm_id.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_mimic_avgs = compute_avgs(all_mimic_res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def avg_over_toks(df):\n",
    "    def avg_col(col):\n",
    "        return (df.total_toks * df[col]).sum() / df.total_toks.sum()\n",
    "    return avg_col('micro_avg'), avg_col('rg_rec_avg'), avg_col('rg_prec_avg'), \\\n",
    "        avg_col('bs_rec_avg'), avg_col('bs_prec_avg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.2636800382046393,\n",
       " 0.42792174543186917,\n",
       " 0.4233153323191832,\n",
       " 0.584446257648712,\n",
       " 0.5816599581880683)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg_over_toks(all_mimic_avgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "avgs_condensed = all_mimic_avgs.loc[:, ['cat_desc', 'num_instances', 'macro_avg', 'rg_rec_med', 'rg_prec_med', 'bs_rec_med', 'bs_prec_med']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "cmap = sns.color_palette('flare', as_cmap=True)\n",
    "cmap = sns.light_palette('cornflowerblue', as_cmap=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['cat_desc', 'num_instances', 'macro_avg', 'rg_rec_med', 'rg_prec_med',\n",
       "       'bs_rec_med', 'bs_prec_med'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avgs_condensed.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "mac_avg = avgs_condensed.loc[:, ['cat_desc', 'macro_avg']].sort_values('macro_avg', ascending=False).reset_index(drop=True)\n",
    "rg = avgs_condensed.loc[:, ['cat_desc', 'rg_rec_med', 'rg_prec_med']].sort_values('rg_rec_med', ascending=False).reset_index(drop=True)\n",
    "bs = avgs_condensed.loc[:, ['cat_desc', 'bs_rec_med', 'bs_prec_med']].sort_values('bs_rec_med', ascending=False).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = avgs_and_format(all_mimic_res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[:, [('Type', ''), ('ROUGE', 'Recall')]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def avgs_and_format(df, head=20):\n",
    "    df = compute_avgs(df)\n",
    "    df = df.loc[:, ['cat_desc', 'num_instances', 'macro_avg', 'rg_rec_med', 'rg_prec_med', 'bs_rec_med', 'bs_prec_med']]\n",
    "    df.columns = [',Type', ',# Notes', ',DiffLib', 'ROUGE,Recall', 'ROUGE,Precision', 'BERTScore,Recall', 'BERTScore,Precision']\n",
    "    df.columns = pd.MultiIndex.from_tuples(df.columns.str.split(',', expand=True).values)\n",
    "    heading_properties = [('font-size', '12px'), ('text-align', 'center')]\n",
    "\n",
    "    cell_properties = [('font-size', '13px')]\n",
    "\n",
    "    dfstyle = [dict(selector=\"th\", props=heading_properties),\\\n",
    "     dict(selector=\"td\", props=cell_properties)]\n",
    "\n",
    "#     df.style.set_table_attributes('style=font-size: 40px')\n",
    "    df = df.loc[:, [('', 'Type'), ('', '# Notes'), ('', 'DiffLib'), ('ROUGE', 'Recall'), ('ROUGE', 'Precision'), ('BERTScore', 'Recall'), ('BERTScore', 'Precision')]]\\\n",
    "        .sort_values(('ROUGE', 'Recall'), ascending=False).reset_index(drop=True)\\\n",
    "        .head(20)\n",
    "    return df, df.style.background_gradient(cmap, subset=[\n",
    "        ('', 'DiffLib'), ('ROUGE', 'Recall'), ('ROUGE', 'Precision'), ('BERTScore', 'Recall'), ('BERTScore', 'Precision')], axis=0)\\\n",
    "        .set_table_styles(dfstyle)\\\n",
    "        .set_precision(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "mcm_df, styled_df = avgs_and_format(all_mimic_res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "def weighted_avg(df):\n",
    "    return ((df['# Notes'] / df['# Notes'].sum()) * df['DiffLib']).sum(),\\\n",
    "    ((df['# Notes'] / df['# Notes'].sum()) * df['RG R']).sum(),\\\n",
    "    ((df['# Notes'] / df['# Notes'].sum()) * df['RG P']).sum(),\\\n",
    "    ((df['# Notes'] / df['# Notes'].sum()) * df['BS R']).sum(),\\\n",
    "    ((df['# Notes'] / df['# Notes'].sum()) * df['BS P']).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.21505814788701702,\n",
       " 0.3030732784730055,\n",
       " 0.29988579090196477,\n",
       " 0.488720143482771,\n",
       " 0.4869750266105248)"
      ]
     },
     "execution_count": 280,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weighted_avg(mcm_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## KCH Stroke Notes Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from differ import compute_sequential_diff_metrics\n",
    "kch_df = pd.read_pickle('stroke_notes.pickle')\n",
    "kch_df['text'] = kch_df.body_analysed\n",
    "sort_cols = ['clientvisit_guid', 'updatetime', 'document_description']\n",
    "group_cols = ['clientvisit_guid', 'document_description']\n",
    "kch_df_diffs = compute_sequential_diff_metrics(kch_df, sort_cols, group_cols) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_note_pairs(df, shuffled_visits, types_to_include, admission_col,  sort_cols, group_cols):\n",
    "    note_pairs = defaultdict(list)\n",
    "    for visit in shuffled_visits:\n",
    "        if len(note_pairs) >= 10:\n",
    "            break\n",
    "        rows = df[df[admission_col] == visit].sort_values(sort_cols)\n",
    "        for k, g in rows.groupby(group_cols):\n",
    "            if g.shape[0] > 1 and k in types_to_include:\n",
    "                for i in range(len(g.text.tolist())-1):\n",
    "                    note_pairs[k].append((visit, g.text.iloc[i], g.text.iloc[i+1]))\n",
    "    return note_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "shuffled_visits = pd.Series(kch_df.clientvisit_guid.unique()).sample(frac=1.0)\n",
    "types_to_include = kch_avgs_df[('', 'Type')].tolist()\n",
    "types_to_include = [t[0:-1] for t in types_to_include]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [],
   "source": [
    "samples = get_note_pairs(kch_df, shuffled_visits, types_to_include, 'clientvisit_guid', ['updatetime', 'document_description'], 'document_description')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [],
   "source": [
    "rouge_metric = load_metric('./rouge.py')\n",
    "bert_metric = load_metric('./bertscore.py')\n",
    "def output_note_pair(pair):\n",
    "    print(pair[0])\n",
    "    rouge_metric.add(prediction=pair[2], reference=pair[1])\n",
    "    bert_metric.add(prediction=pair[2], reference=pair[1])\n",
    "    rougeL = rouge_metric.compute(rouge_types=['rougeL'], use_agregator=False)['rougeL']\n",
    "    bert_score = bert_metric.compute(lang='en', rescale_with_baseline=True, model_type='./xlnet-base-cased')\n",
    "    print(f'Rouge Rec:{round(rougeL[0].recall, 3)}  Prec:{round(rougeL[0].precision, 3)}')\n",
    "    print(f'BERTScore Rec:{round(float(bert_score[\"recall\"].cpu().numpy()), 3)} BERTSCore Prec:{round(float(bert_score[\"precision\"].cpu().numpy()), 3)}')\n",
    "    show_diffs(pair[1], pair[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {},
   "outputs": [],
   "source": [
    "def review_pairs(samples):\n",
    "    for note_type, pairs in samples.items():\n",
    "        print('NOTE TYPE:' + str(note_type))\n",
    "        input()\n",
    "        for pair in pairs:\n",
    "            clear_output(wait=True)\n",
    "            output_note_pair(pair)\n",
    "            skip_type = input()\n",
    "            if skip_type == 'skip':\n",
    "                break\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MIMIC FULL Manual Review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [],
   "source": [
    "shuffled_visits = pd.Series(all_data.hadm_id.unique()).sample(frac=1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [],
   "source": [
    "types_to_include = [tuple(t.split(':')) for t in mcm_df[('', 'Type')]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [],
   "source": [
    "note_pairs = get_note_pairs(all_data, shuffled_visits, types_to_include, 'hadm_id', \n",
    "                            ['chartdate', 'charttime', 'category', 'description'],\n",
    "                            ['category', 'description'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('ECG', 'Report'), 8),\n",
       " (('Echo', 'Report'), 3),\n",
       " (('Nursing/other', 'Report'), 26),\n",
       " (('Radiology', 'CHEST (PORTABLE AP)'), 15),\n",
       " (('General', 'ICU Event Note'), 1),\n",
       " (('Nursing', 'Nursing Progress Note'), 10),\n",
       " (('Nursing', 'Nursing Transfer Note'), 1),\n",
       " (('Physician ', 'Physician Attending Progress Note'), 1),\n",
       " (('Physician ', 'Physician Resident Admission Note'), 1),\n",
       " (('Physician ', 'Physician Resident Progress Note'), 1)]"
      ]
     },
     "execution_count": 277,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(k, len(v)) for k,v in note_pairs.items()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "review_pairs(note_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(sampled_notes.sort_values(sort_cols).groupby(group_cols))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "kch_avgs_df, styled_df = avgs_and_format(kch_df_diffs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.24334423874924335,\n",
       " 0.4169590003434991,\n",
       " 0.4281630516124084,\n",
       " 0.6039267108678945,\n",
       " 0.6059166560277742)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg_over_toks(compute_avgs(kch_df_diffs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weighted_avg(kch_avgs_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kch_avgs.loc[:, ['Type', '# Notes', 'DiffLib Avg', 'RG Rec', 'RG Prec', 'BS Rec', 'BS Prec']]\\\n",
    "    .sort_values('RG Rec', ascending=False).reset_index(drop=True)\\\n",
    "    .style.background_gradient(cmap, subset=['DiffLib Avg', 'RG Rec', 'RG Prec', 'BS Rec', 'BS Prec'], axis=0)\\\n",
    "    .set_precision(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "group_num = 1\n",
    "rand_group = list(kch_df.groupby(['clientvisit_guid', 'document_description', 'updatetime']))[0][group_num]\n",
    "print(rand_group['document_description'])\n",
    "print(rand_group['text'].tolist()[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_metric = load_metric('./bertscore.py')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_metric.add(prediction='foo some text', reference='foo some other text ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-03-01 08:20:18,052: Removing /home/thomass/.cache/huggingface/metrics/bert_score/default/default_experiment-ad0cea4d-c04d-4ec5-90ba-eb6d691977d4-1-0.arrow\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'precision': tensor([0.8856]),\n",
       " 'recall': tensor([0.8320]),\n",
       " 'f1': tensor([0.8588]),\n",
       " 'hashcode': './xlnet-base-cased_L5_no-idf_version=0.3.7(hug_trans=4.3.0.dev0)-rescaled'}"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bert_metric.compute(lang='en', rescale_with_baseline=True, model_type='./xlnet-base-cased')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Randomly sample notes of interesting types\n",
    "- rouge_recall of physicain: intesivist note and nursing transfer\n",
    "- rouge_recall of "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "\n",
    "category = 'Physician '\n",
    "description = 'Intensivist Note'\n",
    "metric = 'rg_recall_med'\n",
    "sample_size = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_sample = df[(df.category == category) & (df.description == description)].sort_values(sort_col, ascending=False).sample(sample_size).loc[:, [metric, 'hadm_id']]\n",
    "for k, g in stroke_notes[stroke_notes.hadm_id.isin(random_sample.hadm_id) & (stroke_notes.description == description) & (stroke_notes.category == category)]\\\n",
    "        .sort_values(['hadm_id', 'category', 'description', 'charttime'])\\\n",
    "        .groupby(['hadm_id', 'category', 'description']):        \n",
    "    for i in range(len(g.text) - 1):\n",
    "        print(f'Group Key:{k}')\n",
    "        print(f'Recall Median:{random_sample[random_sample.hadm_id == k[0]][metric].tolist()[0]}')\n",
    "(g.text.iloc[i], g.text.iloc[i + 1])\n",
    "        input('\\n\\nEnter for next...')\n",
    "        clear_output()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat_desc</th>\n",
       "      <th>micro_avg</th>\n",
       "      <th>rg_recall_avg</th>\n",
       "      <th>rg_precision_avg</th>\n",
       "      <th>bs_recall_avg</th>\n",
       "      <th>bs_precision_avg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Nursing/other:Report</td>\n",
       "      <td>0.071173</td>\n",
       "      <td>0.135753</td>\n",
       "      <td>0.137183</td>\n",
       "      <td>0.256340</td>\n",
       "      <td>0.255637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Radiology:CHEST (PORTABLE AP)</td>\n",
       "      <td>0.173144</td>\n",
       "      <td>0.380864</td>\n",
       "      <td>0.382532</td>\n",
       "      <td>0.748738</td>\n",
       "      <td>0.746997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Nursing:Nursing Progress Note</td>\n",
       "      <td>0.266359</td>\n",
       "      <td>0.409362</td>\n",
       "      <td>0.399518</td>\n",
       "      <td>0.550573</td>\n",
       "      <td>0.543845</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ECG:Report</td>\n",
       "      <td>0.317580</td>\n",
       "      <td>0.405494</td>\n",
       "      <td>0.408634</td>\n",
       "      <td>0.465594</td>\n",
       "      <td>0.471461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Radiology:CT HEAD W/O CONTRAST</td>\n",
       "      <td>0.139997</td>\n",
       "      <td>0.388552</td>\n",
       "      <td>0.430355</td>\n",
       "      <td>0.735919</td>\n",
       "      <td>0.744523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Physician :Intensivist Note</td>\n",
       "      <td>0.438114</td>\n",
       "      <td>0.717188</td>\n",
       "      <td>0.716120</td>\n",
       "      <td>0.886030</td>\n",
       "      <td>0.886067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Nursing:Nursing Transfer Note</td>\n",
       "      <td>0.521301</td>\n",
       "      <td>0.668460</td>\n",
       "      <td>0.668038</td>\n",
       "      <td>0.710947</td>\n",
       "      <td>0.712035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Respiratory :Respiratory Care Shift Note</td>\n",
       "      <td>0.428573</td>\n",
       "      <td>0.653369</td>\n",
       "      <td>0.639566</td>\n",
       "      <td>0.702126</td>\n",
       "      <td>0.696569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Echo:Report</td>\n",
       "      <td>0.199431</td>\n",
       "      <td>0.446879</td>\n",
       "      <td>0.460420</td>\n",
       "      <td>0.701508</td>\n",
       "      <td>0.717413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Radiology:MR HEAD W/O CONTRAST</td>\n",
       "      <td>0.179018</td>\n",
       "      <td>0.339165</td>\n",
       "      <td>0.740209</td>\n",
       "      <td>0.698005</td>\n",
       "      <td>0.808940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Physician :Physician Resident Progress Note</td>\n",
       "      <td>0.413553</td>\n",
       "      <td>0.699704</td>\n",
       "      <td>0.680356</td>\n",
       "      <td>0.865365</td>\n",
       "      <td>0.856367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Radiology:CHEST PORT. LINE PLACEMENT</td>\n",
       "      <td>0.206106</td>\n",
       "      <td>0.432117</td>\n",
       "      <td>0.470552</td>\n",
       "      <td>0.774468</td>\n",
       "      <td>0.780827</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Physician :Physician Attending Progress Note</td>\n",
       "      <td>0.393551</td>\n",
       "      <td>0.739688</td>\n",
       "      <td>0.725586</td>\n",
       "      <td>0.893823</td>\n",
       "      <td>0.885280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Nutrition:Clinical Nutrition Note</td>\n",
       "      <td>0.143914</td>\n",
       "      <td>0.629580</td>\n",
       "      <td>0.592148</td>\n",
       "      <td>0.794563</td>\n",
       "      <td>0.772899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Radiology:PORTABLE ABDOMEN</td>\n",
       "      <td>0.266914</td>\n",
       "      <td>0.559821</td>\n",
       "      <td>0.526943</td>\n",
       "      <td>0.825759</td>\n",
       "      <td>0.814024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Physician :Physician Resident Admission Note</td>\n",
       "      <td>0.777752</td>\n",
       "      <td>0.898663</td>\n",
       "      <td>0.904502</td>\n",
       "      <td>0.954672</td>\n",
       "      <td>0.959385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Radiology:CHEST (PA &amp; LAT)</td>\n",
       "      <td>0.153568</td>\n",
       "      <td>0.400688</td>\n",
       "      <td>0.381554</td>\n",
       "      <td>0.746528</td>\n",
       "      <td>0.756983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Radiology:BY DIFFERENT PHYSICIAN</td>\n",
       "      <td>0.126584</td>\n",
       "      <td>0.386852</td>\n",
       "      <td>0.409420</td>\n",
       "      <td>0.725512</td>\n",
       "      <td>0.743351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Radiology:MR HEAD W &amp; W/O CONTRAST</td>\n",
       "      <td>0.145339</td>\n",
       "      <td>0.378185</td>\n",
       "      <td>0.712067</td>\n",
       "      <td>0.711225</td>\n",
       "      <td>0.776496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Physician :Physician Surgical Progress Note</td>\n",
       "      <td>0.612482</td>\n",
       "      <td>0.769602</td>\n",
       "      <td>0.768946</td>\n",
       "      <td>0.900219</td>\n",
       "      <td>0.899437</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        cat_desc  micro_avg  rg_recall_avg  \\\n",
       "0                           Nursing/other:Report   0.071173       0.135753   \n",
       "1                  Radiology:CHEST (PORTABLE AP)   0.173144       0.380864   \n",
       "2                  Nursing:Nursing Progress Note   0.266359       0.409362   \n",
       "3                                     ECG:Report   0.317580       0.405494   \n",
       "4                 Radiology:CT HEAD W/O CONTRAST   0.139997       0.388552   \n",
       "5                    Physician :Intensivist Note   0.438114       0.717188   \n",
       "6                  Nursing:Nursing Transfer Note   0.521301       0.668460   \n",
       "7       Respiratory :Respiratory Care Shift Note   0.428573       0.653369   \n",
       "8                                    Echo:Report   0.199431       0.446879   \n",
       "9                 Radiology:MR HEAD W/O CONTRAST   0.179018       0.339165   \n",
       "10   Physician :Physician Resident Progress Note   0.413553       0.699704   \n",
       "11          Radiology:CHEST PORT. LINE PLACEMENT   0.206106       0.432117   \n",
       "12  Physician :Physician Attending Progress Note   0.393551       0.739688   \n",
       "13             Nutrition:Clinical Nutrition Note   0.143914       0.629580   \n",
       "14                    Radiology:PORTABLE ABDOMEN   0.266914       0.559821   \n",
       "15  Physician :Physician Resident Admission Note   0.777752       0.898663   \n",
       "16                    Radiology:CHEST (PA & LAT)   0.153568       0.400688   \n",
       "17              Radiology:BY DIFFERENT PHYSICIAN   0.126584       0.386852   \n",
       "18            Radiology:MR HEAD W & W/O CONTRAST   0.145339       0.378185   \n",
       "19   Physician :Physician Surgical Progress Note   0.612482       0.769602   \n",
       "\n",
       "    rg_precision_avg  bs_recall_avg  bs_precision_avg  \n",
       "0           0.137183       0.256340          0.255637  \n",
       "1           0.382532       0.748738          0.746997  \n",
       "2           0.399518       0.550573          0.543845  \n",
       "3           0.408634       0.465594          0.471461  \n",
       "4           0.430355       0.735919          0.744523  \n",
       "5           0.716120       0.886030          0.886067  \n",
       "6           0.668038       0.710947          0.712035  \n",
       "7           0.639566       0.702126          0.696569  \n",
       "8           0.460420       0.701508          0.717413  \n",
       "9           0.740209       0.698005          0.808940  \n",
       "10          0.680356       0.865365          0.856367  \n",
       "11          0.470552       0.774468          0.780827  \n",
       "12          0.725586       0.893823          0.885280  \n",
       "13          0.592148       0.794563          0.772899  \n",
       "14          0.526943       0.825759          0.814024  \n",
       "15          0.904502       0.954672          0.959385  \n",
       "16          0.381554       0.746528          0.756983  \n",
       "17          0.409420       0.725512          0.743351  \n",
       "18          0.712067       0.711225          0.776496  \n",
       "19          0.768946       0.900219          0.899437  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avgs.loc[:, ['cat_desc', 'micro_avg', 'rg_recall_avg', 'rg_precision_avg', 'bs_recall_avg', 'bs_precision_avg']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "stroke_care_notes_df, stroke_discharge_df = split_notes(stroke_notes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hadm_id</th>\n",
       "      <th>category</th>\n",
       "      <th>description</th>\n",
       "      <th>avg_diff_ratio</th>\n",
       "      <th>rougeL_rec</th>\n",
       "      <th>rougeL_prec</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>121824</td>\n",
       "      <td>Physician</td>\n",
       "      <td>Physician Resident Admission Note</td>\n",
       "      <td>0.853598</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.769582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>107913</td>\n",
       "      <td>Physician</td>\n",
       "      <td>Physician Surgical Admission Note</td>\n",
       "      <td>0.996592</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.980357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>860</th>\n",
       "      <td>180112</td>\n",
       "      <td>Nursing</td>\n",
       "      <td>Nursing Transfer Note</td>\n",
       "      <td>0.932375</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.420744</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>450</th>\n",
       "      <td>140761</td>\n",
       "      <td>Physician</td>\n",
       "      <td>Physician Surgical Admission Note</td>\n",
       "      <td>0.999477</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.996753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>586</th>\n",
       "      <td>154473</td>\n",
       "      <td>General</td>\n",
       "      <td>Cardiology consult</td>\n",
       "      <td>0.904114</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.949074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1079</th>\n",
       "      <td>195985</td>\n",
       "      <td>Nursing/other</td>\n",
       "      <td>Report</td>\n",
       "      <td>0.035890</td>\n",
       "      <td>0.073285</td>\n",
       "      <td>0.183257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>889</th>\n",
       "      <td>182153</td>\n",
       "      <td>Nursing/other</td>\n",
       "      <td>Report</td>\n",
       "      <td>0.036163</td>\n",
       "      <td>0.066773</td>\n",
       "      <td>0.135223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>799</th>\n",
       "      <td>171903</td>\n",
       "      <td>Nursing/other</td>\n",
       "      <td>Report</td>\n",
       "      <td>0.028571</td>\n",
       "      <td>0.060440</td>\n",
       "      <td>0.180328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>109832</td>\n",
       "      <td>Nursing</td>\n",
       "      <td>Nursing Transfer Note</td>\n",
       "      <td>0.037115</td>\n",
       "      <td>0.049407</td>\n",
       "      <td>0.806452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>224</th>\n",
       "      <td>120556</td>\n",
       "      <td>Nutrition</td>\n",
       "      <td>Clinical Nutrition Note</td>\n",
       "      <td>0.021978</td>\n",
       "      <td>0.020619</td>\n",
       "      <td>0.342857</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1135 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      hadm_id       category                        description  \\\n",
       "247    121824     Physician   Physician Resident Admission Note   \n",
       "98     107913     Physician   Physician Surgical Admission Note   \n",
       "860    180112        Nursing              Nursing Transfer Note   \n",
       "450    140761     Physician   Physician Surgical Admission Note   \n",
       "586    154473        General                 Cardiology consult   \n",
       "...       ...            ...                                ...   \n",
       "1079   195985  Nursing/other                             Report   \n",
       "889    182153  Nursing/other                             Report   \n",
       "799    171903  Nursing/other                             Report   \n",
       "122    109832        Nursing              Nursing Transfer Note   \n",
       "224    120556      Nutrition            Clinical Nutrition Note   \n",
       "\n",
       "      avg_diff_ratio  rougeL_rec  rougeL_prec  \n",
       "247         0.853598    1.000000     0.769582  \n",
       "98          0.996592    1.000000     0.980357  \n",
       "860         0.932375    1.000000     0.420744  \n",
       "450         0.999477    1.000000     0.996753  \n",
       "586         0.904114    1.000000     0.949074  \n",
       "...              ...         ...          ...  \n",
       "1079        0.035890    0.073285     0.183257  \n",
       "889         0.036163    0.066773     0.135223  \n",
       "799         0.028571    0.060440     0.180328  \n",
       "122         0.037115    0.049407     0.806452  \n",
       "224         0.021978    0.020619     0.342857  \n",
       "\n",
       "[1135 rows x 6 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comp_metrics.sort_values('rougeL_rec', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = stroke_notes[(stroke_notes.hadm_id == 121824) & (stroke_notes.category == 'Physician ') & (stroke_notes.description == 'Physician Resident Admission Note')].text.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 2.148E+18\n"
     ]
    }
   ],
   "source": [
    "print(\"%10.3aE\" % 2147889734063751200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 8.701E+16\n"
     ]
    }
   ],
   "source": [
    "print(\"%10.3E\" % 87008250040418300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"%10.3E\" %)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "group_avgs = compute_avgs(results_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "group_avgs['redundant_ratio'] = group_avgs['redundant_toks'] / group_avgs.total_toks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat_desc</th>\n",
       "      <th>redundant_toks</th>\n",
       "      <th>total_toks</th>\n",
       "      <th>avg_txt_len</th>\n",
       "      <th>macro_avg</th>\n",
       "      <th>micro_avg</th>\n",
       "      <th>num_instances</th>\n",
       "      <th>redundant_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Physician :Physician Resident Admission Note</td>\n",
       "      <td>28842.0</td>\n",
       "      <td>33109</td>\n",
       "      <td>12525.819048</td>\n",
       "      <td>0.856518</td>\n",
       "      <td>0.809448</td>\n",
       "      <td>19</td>\n",
       "      <td>0.871123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Physician :Physician Surgical Progress Note</td>\n",
       "      <td>3181.0</td>\n",
       "      <td>4347</td>\n",
       "      <td>5587.000000</td>\n",
       "      <td>0.809635</td>\n",
       "      <td>0.764214</td>\n",
       "      <td>7</td>\n",
       "      <td>0.731769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Physician :Physician Attending Progress Note</td>\n",
       "      <td>14380.0</td>\n",
       "      <td>21148</td>\n",
       "      <td>5941.186706</td>\n",
       "      <td>0.532457</td>\n",
       "      <td>0.536417</td>\n",
       "      <td>31</td>\n",
       "      <td>0.679970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Physician :Intensivist Note</td>\n",
       "      <td>100737.0</td>\n",
       "      <td>159909</td>\n",
       "      <td>5958.690231</td>\n",
       "      <td>0.691003</td>\n",
       "      <td>0.639584</td>\n",
       "      <td>211</td>\n",
       "      <td>0.629965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Respiratory :Respiratory Care Shift Note</td>\n",
       "      <td>7263.0</td>\n",
       "      <td>12124</td>\n",
       "      <td>1230.545915</td>\n",
       "      <td>0.404803</td>\n",
       "      <td>0.443782</td>\n",
       "      <td>79</td>\n",
       "      <td>0.599060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Nursing:Nursing Transfer Note</td>\n",
       "      <td>30799.0</td>\n",
       "      <td>53260</td>\n",
       "      <td>2230.841242</td>\n",
       "      <td>0.598621</td>\n",
       "      <td>0.550831</td>\n",
       "      <td>159</td>\n",
       "      <td>0.578276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Physician :Physician Resident Progress Note</td>\n",
       "      <td>40675.0</td>\n",
       "      <td>73517</td>\n",
       "      <td>6890.886364</td>\n",
       "      <td>0.551072</td>\n",
       "      <td>0.467104</td>\n",
       "      <td>94</td>\n",
       "      <td>0.553273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Nursing:Nursing Progress Note</td>\n",
       "      <td>64447.0</td>\n",
       "      <td>136297</td>\n",
       "      <td>1291.039215</td>\n",
       "      <td>0.442277</td>\n",
       "      <td>0.419754</td>\n",
       "      <td>620</td>\n",
       "      <td>0.472842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Radiology:PORTABLE ABDOMEN</td>\n",
       "      <td>923.0</td>\n",
       "      <td>2992</td>\n",
       "      <td>1084.420513</td>\n",
       "      <td>0.274080</td>\n",
       "      <td>0.259291</td>\n",
       "      <td>23</td>\n",
       "      <td>0.308489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Radiology:CHEST PORT. LINE PLACEMENT</td>\n",
       "      <td>1237.0</td>\n",
       "      <td>4903</td>\n",
       "      <td>1240.143860</td>\n",
       "      <td>0.203611</td>\n",
       "      <td>0.219498</td>\n",
       "      <td>34</td>\n",
       "      <td>0.252295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Nutrition:Clinical Nutrition Note</td>\n",
       "      <td>1907.0</td>\n",
       "      <td>7893</td>\n",
       "      <td>2573.938462</td>\n",
       "      <td>0.135038</td>\n",
       "      <td>0.150969</td>\n",
       "      <td>23</td>\n",
       "      <td>0.241606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Radiology:CHEST (PA &amp; LAT)</td>\n",
       "      <td>356.0</td>\n",
       "      <td>1512</td>\n",
       "      <td>1208.121212</td>\n",
       "      <td>0.151395</td>\n",
       "      <td>0.147700</td>\n",
       "      <td>12</td>\n",
       "      <td>0.235450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Radiology:CHEST (PORTABLE AP)</td>\n",
       "      <td>26063.0</td>\n",
       "      <td>113265</td>\n",
       "      <td>1171.532585</td>\n",
       "      <td>0.185413</td>\n",
       "      <td>0.188866</td>\n",
       "      <td>837</td>\n",
       "      <td>0.230106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>ECG:Report</td>\n",
       "      <td>3664.0</td>\n",
       "      <td>16233</td>\n",
       "      <td>223.494057</td>\n",
       "      <td>0.321421</td>\n",
       "      <td>0.319381</td>\n",
       "      <td>512</td>\n",
       "      <td>0.225713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Echo:Report</td>\n",
       "      <td>5402.0</td>\n",
       "      <td>26459</td>\n",
       "      <td>2524.776515</td>\n",
       "      <td>0.202008</td>\n",
       "      <td>0.197195</td>\n",
       "      <td>77</td>\n",
       "      <td>0.204165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Radiology:MR HEAD W/O CONTRAST</td>\n",
       "      <td>3433.0</td>\n",
       "      <td>18228</td>\n",
       "      <td>2300.468085</td>\n",
       "      <td>0.177119</td>\n",
       "      <td>0.178997</td>\n",
       "      <td>48</td>\n",
       "      <td>0.188337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Radiology:CT HEAD W/O CONTRAST</td>\n",
       "      <td>14838.0</td>\n",
       "      <td>90085</td>\n",
       "      <td>1918.349546</td>\n",
       "      <td>0.148847</td>\n",
       "      <td>0.149810</td>\n",
       "      <td>373</td>\n",
       "      <td>0.164711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Radiology:MR HEAD W &amp; W/O CONTRAST</td>\n",
       "      <td>494.0</td>\n",
       "      <td>3188</td>\n",
       "      <td>2696.861111</td>\n",
       "      <td>0.161859</td>\n",
       "      <td>0.158945</td>\n",
       "      <td>7</td>\n",
       "      <td>0.154956</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Radiology:BY DIFFERENT PHYSICIAN</td>\n",
       "      <td>218.0</td>\n",
       "      <td>1542</td>\n",
       "      <td>1365.694444</td>\n",
       "      <td>0.138657</td>\n",
       "      <td>0.131175</td>\n",
       "      <td>9</td>\n",
       "      <td>0.141375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Nursing/other:Report</td>\n",
       "      <td>6334.0</td>\n",
       "      <td>298384</td>\n",
       "      <td>1011.361172</td>\n",
       "      <td>0.061062</td>\n",
       "      <td>0.068108</td>\n",
       "      <td>1889</td>\n",
       "      <td>0.021228</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        cat_desc  redundant_toks  total_toks  \\\n",
       "0   Physician :Physician Resident Admission Note         28842.0       33109   \n",
       "1    Physician :Physician Surgical Progress Note          3181.0        4347   \n",
       "2   Physician :Physician Attending Progress Note         14380.0       21148   \n",
       "3                    Physician :Intensivist Note        100737.0      159909   \n",
       "4       Respiratory :Respiratory Care Shift Note          7263.0       12124   \n",
       "5                  Nursing:Nursing Transfer Note         30799.0       53260   \n",
       "6    Physician :Physician Resident Progress Note         40675.0       73517   \n",
       "7                  Nursing:Nursing Progress Note         64447.0      136297   \n",
       "8                     Radiology:PORTABLE ABDOMEN           923.0        2992   \n",
       "9           Radiology:CHEST PORT. LINE PLACEMENT          1237.0        4903   \n",
       "10             Nutrition:Clinical Nutrition Note          1907.0        7893   \n",
       "11                    Radiology:CHEST (PA & LAT)           356.0        1512   \n",
       "12                 Radiology:CHEST (PORTABLE AP)         26063.0      113265   \n",
       "13                                    ECG:Report          3664.0       16233   \n",
       "14                                   Echo:Report          5402.0       26459   \n",
       "15                Radiology:MR HEAD W/O CONTRAST          3433.0       18228   \n",
       "16                Radiology:CT HEAD W/O CONTRAST         14838.0       90085   \n",
       "17            Radiology:MR HEAD W & W/O CONTRAST           494.0        3188   \n",
       "18              Radiology:BY DIFFERENT PHYSICIAN           218.0        1542   \n",
       "19                          Nursing/other:Report          6334.0      298384   \n",
       "\n",
       "     avg_txt_len  macro_avg  micro_avg  num_instances  redundant_ratio  \n",
       "0   12525.819048   0.856518   0.809448             19         0.871123  \n",
       "1    5587.000000   0.809635   0.764214              7         0.731769  \n",
       "2    5941.186706   0.532457   0.536417             31         0.679970  \n",
       "3    5958.690231   0.691003   0.639584            211         0.629965  \n",
       "4    1230.545915   0.404803   0.443782             79         0.599060  \n",
       "5    2230.841242   0.598621   0.550831            159         0.578276  \n",
       "6    6890.886364   0.551072   0.467104             94         0.553273  \n",
       "7    1291.039215   0.442277   0.419754            620         0.472842  \n",
       "8    1084.420513   0.274080   0.259291             23         0.308489  \n",
       "9    1240.143860   0.203611   0.219498             34         0.252295  \n",
       "10   2573.938462   0.135038   0.150969             23         0.241606  \n",
       "11   1208.121212   0.151395   0.147700             12         0.235450  \n",
       "12   1171.532585   0.185413   0.188866            837         0.230106  \n",
       "13    223.494057   0.321421   0.319381            512         0.225713  \n",
       "14   2524.776515   0.202008   0.197195             77         0.204165  \n",
       "15   2300.468085   0.177119   0.178997             48         0.188337  \n",
       "16   1918.349546   0.148847   0.149810            373         0.164711  \n",
       "17   2696.861111   0.161859   0.158945              7         0.154956  \n",
       "18   1365.694444   0.138657   0.131175              9         0.141375  \n",
       "19   1011.361172   0.061062   0.068108           1889         0.021228  "
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group_avgs.sort_values('redundant_ratio', ascending=False).reset_index(drop=True).head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-01-07 10:30:29,573: Removing /Users/tom/.cache/huggingface/metrics/rouge/default/default_experiment-1-0.arrow\n"
     ]
    }
   ],
   "source": [
    "pt = stroke_notes[stroke_notes.hadm_id == 195402]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "pt_nurs_notes = pt[pt.description == 'Nursing Progress Note']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "def single_group_ratios(pt):\n",
    "    internote_sort_cols = ['hadm_id', 'chartdate', 'charttime', 'category', 'description']\n",
    "    internote_group_keys = ['hadm_id', 'category', 'description']\n",
    "    groups_gen = pt.sort_values(internote_sort_cols).groupby(internote_group_keys)\n",
    "    results = defaultdict(list)\n",
    "    metric = load_metric('rouge')\n",
    "    score = None\n",
    "    ratios = []\n",
    "    for k, g in [list(groups_gen)[-1]]:\n",
    "        if g.shape[0] > 1:\n",
    "            for i in range(len(g.text.tolist()) - 1):\n",
    "                metric.add(prediction=g.text.iloc[i], reference=g.text.iloc[i+1])\n",
    "                ratio, std_dev, max_ratio, min_ratio, redundant_toks_len, total_toks = \\\n",
    "                    diff_ratio(g.text.iloc[i], g.text.iloc[i+1])\n",
    "                ratios.append(ratio)\n",
    "            score = metric.compute(rouge_types=['rougeL'])\n",
    "    return ratios, score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-01-07 11:32:03,220: Removing /Users/tom/.cache/huggingface/metrics/rouge/default/default_experiment-1-0.arrow\n"
     ]
    }
   ],
   "source": [
    "ratios, rouge_scores = single_group_ratios(pt_nurs_notes)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:cattrainer]",
   "language": "python",
   "name": "conda-env-cattrainer-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}