notebooks/01_Statistics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "13d752d3-f3d0-49e3-b75a-e9453b095e0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../util/')\n",
    "sys.path.append('../experiments/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c39a2c05-c1e3-4329-8b1f-fd147cbf7285",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "import spacy\n",
    "import re\n",
    "from collections import defaultdict\n",
    "from huggingface_utils import load_custom_dataset\n",
    "from convert_annotations import entity_values\n",
    "from stats_util import sort_fn, sort_order\n",
    "import datasets\n",
    "datasets.logging.set_verbosity_error()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6b22384-204f-4d70-bf11-b2205b368830",
   "metadata": {},
   "source": [
    "# Corpus Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7d5d3a97-66ed-4976-a2fd-521c8ea33aa3",
   "metadata": {},
   "outputs": [],
   "source": [
    "translations = pd.read_csv('../data/guideline_translations.csv').set_index('german_name')\n",
    "\n",
    "def translator_fn(name):\n",
    "    if name == 'Sum':\n",
    "        return name\n",
    "    return translations.loc[name].english_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1a801902-f33b-4781-8f9f-cce2ef756730",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>num_docs</th>\n",
       "      <th>num_recommendations</th>\n",
       "      <th>num_sentences</th>\n",
       "      <th>num_tokens</th>\n",
       "      <th>num_types</th>\n",
       "      <th>num_litref</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Pancreatic cancer</td>\n",
       "      <td>292</td>\n",
       "      <td>158</td>\n",
       "      <td>854</td>\n",
       "      <td>18901</td>\n",
       "      <td>3602</td>\n",
       "      <td>1154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Penis cancer</td>\n",
       "      <td>167</td>\n",
       "      <td>94</td>\n",
       "      <td>960</td>\n",
       "      <td>20915</td>\n",
       "      <td>4542</td>\n",
       "      <td>561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Psycho-oncology</td>\n",
       "      <td>121</td>\n",
       "      <td>47</td>\n",
       "      <td>778</td>\n",
       "      <td>21909</td>\n",
       "      <td>4113</td>\n",
       "      <td>835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Oral cavity cancer</td>\n",
       "      <td>132</td>\n",
       "      <td>96</td>\n",
       "      <td>763</td>\n",
       "      <td>22256</td>\n",
       "      <td>3947</td>\n",
       "      <td>1172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Malignant ovarian tumors</td>\n",
       "      <td>195</td>\n",
       "      <td>97</td>\n",
       "      <td>1103</td>\n",
       "      <td>27432</td>\n",
       "      <td>5139</td>\n",
       "      <td>1035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Anal cancer</td>\n",
       "      <td>216</td>\n",
       "      <td>93</td>\n",
       "      <td>1248</td>\n",
       "      <td>34429</td>\n",
       "      <td>5246</td>\n",
       "      <td>724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Chronic lymphocytic leukemia</td>\n",
       "      <td>285</td>\n",
       "      <td>138</td>\n",
       "      <td>1417</td>\n",
       "      <td>36811</td>\n",
       "      <td>5680</td>\n",
       "      <td>726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Laryngeal cancer</td>\n",
       "      <td>189</td>\n",
       "      <td>118</td>\n",
       "      <td>1526</td>\n",
       "      <td>37374</td>\n",
       "      <td>6812</td>\n",
       "      <td>681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Follicular lymphoma</td>\n",
       "      <td>296</td>\n",
       "      <td>149</td>\n",
       "      <td>1537</td>\n",
       "      <td>38206</td>\n",
       "      <td>6344</td>\n",
       "      <td>761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Oesophageal cancer</td>\n",
       "      <td>172</td>\n",
       "      <td>91</td>\n",
       "      <td>1530</td>\n",
       "      <td>38574</td>\n",
       "      <td>6615</td>\n",
       "      <td>1026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Hodgkin lymphoma</td>\n",
       "      <td>253</td>\n",
       "      <td>168</td>\n",
       "      <td>1710</td>\n",
       "      <td>38899</td>\n",
       "      <td>5886</td>\n",
       "      <td>976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Hepatocellular and biliary cancer</td>\n",
       "      <td>263</td>\n",
       "      <td>146</td>\n",
       "      <td>1599</td>\n",
       "      <td>41125</td>\n",
       "      <td>6552</td>\n",
       "      <td>990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Testicular tumors</td>\n",
       "      <td>315</td>\n",
       "      <td>163</td>\n",
       "      <td>1923</td>\n",
       "      <td>47024</td>\n",
       "      <td>6746</td>\n",
       "      <td>1412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Prevention of cervix cancer</td>\n",
       "      <td>302</td>\n",
       "      <td>103</td>\n",
       "      <td>2058</td>\n",
       "      <td>52351</td>\n",
       "      <td>7928</td>\n",
       "      <td>1388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Renal cell carcinoma</td>\n",
       "      <td>293</td>\n",
       "      <td>131</td>\n",
       "      <td>2284</td>\n",
       "      <td>52576</td>\n",
       "      <td>8255</td>\n",
       "      <td>1507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Endometrial cancer</td>\n",
       "      <td>317</td>\n",
       "      <td>173</td>\n",
       "      <td>2005</td>\n",
       "      <td>53773</td>\n",
       "      <td>8005</td>\n",
       "      <td>1340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Stomach cancer</td>\n",
       "      <td>246</td>\n",
       "      <td>142</td>\n",
       "      <td>2282</td>\n",
       "      <td>54281</td>\n",
       "      <td>8028</td>\n",
       "      <td>1671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Adult soft tissue sarcomas</td>\n",
       "      <td>407</td>\n",
       "      <td>228</td>\n",
       "      <td>2407</td>\n",
       "      <td>56358</td>\n",
       "      <td>8785</td>\n",
       "      <td>1169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Actinic keratosis</td>\n",
       "      <td>193</td>\n",
       "      <td>74</td>\n",
       "      <td>2599</td>\n",
       "      <td>57375</td>\n",
       "      <td>6853</td>\n",
       "      <td>1278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Malignant melanoma</td>\n",
       "      <td>297</td>\n",
       "      <td>167</td>\n",
       "      <td>2746</td>\n",
       "      <td>65207</td>\n",
       "      <td>9241</td>\n",
       "      <td>1718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Cervical cancer</td>\n",
       "      <td>415</td>\n",
       "      <td>127</td>\n",
       "      <td>2829</td>\n",
       "      <td>68576</td>\n",
       "      <td>9593</td>\n",
       "      <td>1549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Colorectal cancer</td>\n",
       "      <td>546</td>\n",
       "      <td>278</td>\n",
       "      <td>3021</td>\n",
       "      <td>73389</td>\n",
       "      <td>9507</td>\n",
       "      <td>2446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Prostate cancer</td>\n",
       "      <td>351</td>\n",
       "      <td>238</td>\n",
       "      <td>3403</td>\n",
       "      <td>81899</td>\n",
       "      <td>10154</td>\n",
       "      <td>2357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Supportive therapy</td>\n",
       "      <td>819</td>\n",
       "      <td>337</td>\n",
       "      <td>4288</td>\n",
       "      <td>96734</td>\n",
       "      <td>12369</td>\n",
       "      <td>2401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Lung cancer</td>\n",
       "      <td>665</td>\n",
       "      <td>312</td>\n",
       "      <td>4302</td>\n",
       "      <td>100930</td>\n",
       "      <td>12591</td>\n",
       "      <td>2345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Breast cancer</td>\n",
       "      <td>685</td>\n",
       "      <td>362</td>\n",
       "      <td>4232</td>\n",
       "      <td>101254</td>\n",
       "      <td>12592</td>\n",
       "      <td>2831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Bladder cancer</td>\n",
       "      <td>364</td>\n",
       "      <td>230</td>\n",
       "      <td>4280</td>\n",
       "      <td>102192</td>\n",
       "      <td>11965</td>\n",
       "      <td>2631</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Prevention of skin cancer</td>\n",
       "      <td>370</td>\n",
       "      <td>141</td>\n",
       "      <td>4298</td>\n",
       "      <td>106488</td>\n",
       "      <td>13817</td>\n",
       "      <td>1578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Palliative medicine</td>\n",
       "      <td>700</td>\n",
       "      <td>442</td>\n",
       "      <td>6029</td>\n",
       "      <td>145657</td>\n",
       "      <td>15806</td>\n",
       "      <td>3117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Complementary medicine</td>\n",
       "      <td>327</td>\n",
       "      <td>149</td>\n",
       "      <td>8079</td>\n",
       "      <td>184205</td>\n",
       "      <td>15622</td>\n",
       "      <td>3286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Sum</td>\n",
       "      <td>10193</td>\n",
       "      <td>5192</td>\n",
       "      <td>78090</td>\n",
       "      <td>1877100</td>\n",
       "      <td>89256</td>\n",
       "      <td>46665</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 name  num_docs  num_recommendations  \\\n",
       "0                   Pancreatic cancer       292                  158   \n",
       "1                        Penis cancer       167                   94   \n",
       "2                     Psycho-oncology       121                   47   \n",
       "3                  Oral cavity cancer       132                   96   \n",
       "4            Malignant ovarian tumors       195                   97   \n",
       "5                         Anal cancer       216                   93   \n",
       "6        Chronic lymphocytic leukemia       285                  138   \n",
       "7                    Laryngeal cancer       189                  118   \n",
       "8                 Follicular lymphoma       296                  149   \n",
       "9                  Oesophageal cancer       172                   91   \n",
       "10                   Hodgkin lymphoma       253                  168   \n",
       "11  Hepatocellular and biliary cancer       263                  146   \n",
       "12                  Testicular tumors       315                  163   \n",
       "13        Prevention of cervix cancer       302                  103   \n",
       "14               Renal cell carcinoma       293                  131   \n",
       "15                 Endometrial cancer       317                  173   \n",
       "16                     Stomach cancer       246                  142   \n",
       "17         Adult soft tissue sarcomas       407                  228   \n",
       "18                  Actinic keratosis       193                   74   \n",
       "19                 Malignant melanoma       297                  167   \n",
       "20                    Cervical cancer       415                  127   \n",
       "21                  Colorectal cancer       546                  278   \n",
       "22                    Prostate cancer       351                  238   \n",
       "23                 Supportive therapy       819                  337   \n",
       "24                        Lung cancer       665                  312   \n",
       "25                      Breast cancer       685                  362   \n",
       "26                     Bladder cancer       364                  230   \n",
       "27          Prevention of skin cancer       370                  141   \n",
       "28                Palliative medicine       700                  442   \n",
       "29             Complementary medicine       327                  149   \n",
       "30                                Sum     10193                 5192   \n",
       "\n",
       "    num_sentences  num_tokens  num_types  num_litref  \n",
       "0             854       18901       3602        1154  \n",
       "1             960       20915       4542         561  \n",
       "2             778       21909       4113         835  \n",
       "3             763       22256       3947        1172  \n",
       "4            1103       27432       5139        1035  \n",
       "5            1248       34429       5246         724  \n",
       "6            1417       36811       5680         726  \n",
       "7            1526       37374       6812         681  \n",
       "8            1537       38206       6344         761  \n",
       "9            1530       38574       6615        1026  \n",
       "10           1710       38899       5886         976  \n",
       "11           1599       41125       6552         990  \n",
       "12           1923       47024       6746        1412  \n",
       "13           2058       52351       7928        1388  \n",
       "14           2284       52576       8255        1507  \n",
       "15           2005       53773       8005        1340  \n",
       "16           2282       54281       8028        1671  \n",
       "17           2407       56358       8785        1169  \n",
       "18           2599       57375       6853        1278  \n",
       "19           2746       65207       9241        1718  \n",
       "20           2829       68576       9593        1549  \n",
       "21           3021       73389       9507        2446  \n",
       "22           3403       81899      10154        2357  \n",
       "23           4288       96734      12369        2401  \n",
       "24           4302      100930      12591        2345  \n",
       "25           4232      101254      12592        2831  \n",
       "26           4280      102192      11965        2631  \n",
       "27           4298      106488      13817        1578  \n",
       "28           6029      145657      15806        3117  \n",
       "29           8079      184205      15622        3286  \n",
       "30          78090     1877100      89256       46665  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus_stats = pd.read_csv('../data/cpg-stats.csv', sep=';')\n",
    "corpus_stats.name = corpus_stats.name.map(translator_fn)\n",
    "corpus_stats.sort_values('num_tokens', inplace=True)\n",
    "corpus_stats.reset_index(inplace=True)\n",
    "corpus_stats[['name', 'num_docs', 'num_recommendations', 'num_sentences', 'num_tokens', 'num_types', 'num_litref' ]]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d24f7fb7-b08e-4d51-b62c-62571de89175",
   "metadata": {},
   "source": [
    "# Annotation Statistics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "613b36f8-804f-4003-97bf-f60e7bcde121",
   "metadata": {},
   "source": [
    "We use the IOB-encoded tags from the converted HuggingFace dataset to obtain statistics of entity annotation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "020a95a6-2368-41e4-a136-9f51712b75d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "ggponc_basepath = Path('..') / 'data' / 'annotations' / 'huggingface'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ed258b94-6ad4-49b6-b07e-ca663a4b6a67",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5228f6ee4f864658abc9fefa39ea1710",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5770e98a668d4af8a8bdc73e29e7c2ef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e26e0e01e7d41e7ae6459de4935038e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66ba75550f5846b4a9bd1a6d24526daf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "result_list = []\n",
    "\n",
    "for level in ['value', 'detail']:\n",
    "    granularity = 'fine' if level == 'detail' else 'coarse'\n",
    "    tagset = [t.replace(' ', '_') for t in entity_values[level]]\n",
    "    for spans in ['short', 'long']:\n",
    "        folder = ggponc_basepath / granularity / spans\n",
    "        train_file = folder / f'train_{granularity}_{spans}.json'\n",
    "        dev_file = folder / f'dev_{granularity}_{spans}.json'\n",
    "        test_file = folder / f'test_{granularity}_{spans}.json'\n",
    "\n",
    "        dataset, tags = load_custom_dataset(\n",
    "            train=train_file, dev=dev_file, test=test_file, tag_strings=tagset)\n",
    "\n",
    "        for split in ['train', 'dev', 'test']:\n",
    "            counts = defaultdict(lambda: 0)\n",
    "            token_counts = defaultdict(lambda: 0)\n",
    "            \n",
    "            n_sentences = len(set(list(zip(dataset[split]['fname'], dataset[split]['sentence_id']))))\n",
    "            n_files = len(set(dataset[split]['fname']))\n",
    "            \n",
    "            cur_token_length = 0\n",
    "            \n",
    "            for tags in dataset[split][\"tags\"]:\n",
    "                for tag in tags:\n",
    "                    if tag.startswith('B-'):\n",
    "                        tag_name = tag.replace('B-', '').replace('_', ' ')\n",
    "                        counts[tag_name] += 1\n",
    "                        counts['Total'] += 1\n",
    "                        if cur_token_length > 0:\n",
    "                            token_counts[tag_name] += cur_token_length\n",
    "                            token_counts['Total'] += cur_token_length\n",
    "                        cur_token_length = 1\n",
    "                    elif tag.startswith('I'):\n",
    "                        assert cur_token_length > 0\n",
    "                        cur_token_length += 1\n",
    "                    elif tag.startswith('O'):\n",
    "                        counts['O'] += 1\n",
    "                        if cur_token_length > 0:\n",
    "                            token_counts[tag_name] += cur_token_length\n",
    "                            token_counts['Total'] += cur_token_length\n",
    "                            cur_token_length = 0\n",
    "            for k, v in counts.items():\n",
    "                result_list.append(\n",
    "                    {'tag' : k, \n",
    "                     'count' : v,\n",
    "                     'token_count' : token_counts[k],\n",
    "                     'split' : split.capitalize(), \n",
    "                     'level' : granularity.capitalize(), \n",
    "                     'spans' : spans.capitalize(),\n",
    "                     'n_sentences' : n_sentences,\n",
    "                     'n_files' : n_files})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "66fc3222-2500-4afc-86b8-a3f940e22bf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame(result_list).set_index(['tag', 'level', 'spans', 'split'])\n",
    "results = results.unstack(['spans', 'split']).sort_index(axis=1, key=sort_fn).sort_index(axis=0, key=sort_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53c4019e-276e-42b4-9525-1dfeccf066c9",
   "metadata": {},
   "source": [
    "### Calculate token lengths per mention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8da6506f-cbd8-4e7f-9de5-a674075767d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "token_lengths = results['token_count'].groupby('spans', axis=1).sum() / results['count'].groupby('spans', axis=1).sum()\n",
    "token_lengths = pd.concat({'Tokens / Mention': pd.concat({'count': token_lengths}, axis=1)}, names=['split'], axis=1)\n",
    "token_lengths.columns = token_lengths.columns.reorder_levels(results.columns.names)\n",
    "\n",
    "results = pd.concat([token_lengths, results], axis=1).sort_index(axis=1, key=sort_fn).sort_index(axis=0, key=sort_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e9e5ec4-15a7-4ba4-b263-923b07d1e752",
   "metadata": {},
   "source": [
    "### Show results for short spans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "bc201782-eb95-4b87-90c4-c736deadcde2",
   "metadata": {},
   "outputs": [],
   "source": [
    "entity_counts = results['count'].droplevel('level')\n",
    "pd.options.display.float_format = '{:,.1f}'.format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a417afd3-a67a-49db-8b78-625156e8c952",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tokens / Mention</th>\n",
       "      <th>Total</th>\n",
       "      <th>Train</th>\n",
       "      <th>Dev</th>\n",
       "      <th>Test</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tag</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Finding</th>\n",
       "      <td>1.1</td>\n",
       "      <td>132756</td>\n",
       "      <td>92053</td>\n",
       "      <td>19743</td>\n",
       "      <td>20960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Diagnosis or Pathology</th>\n",
       "      <td>1.1</td>\n",
       "      <td>81380</td>\n",
       "      <td>56216</td>\n",
       "      <td>12200</td>\n",
       "      <td>12964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Other Finding</th>\n",
       "      <td>1.1</td>\n",
       "      <td>51376</td>\n",
       "      <td>35837</td>\n",
       "      <td>7543</td>\n",
       "      <td>7996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Substance</th>\n",
       "      <td>1.2</td>\n",
       "      <td>24871</td>\n",
       "      <td>17288</td>\n",
       "      <td>4017</td>\n",
       "      <td>3566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Clinical Drug</th>\n",
       "      <td>1.2</td>\n",
       "      <td>19478</td>\n",
       "      <td>13647</td>\n",
       "      <td>3105</td>\n",
       "      <td>2726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Nutrient or Body Substance</th>\n",
       "      <td>1.3</td>\n",
       "      <td>4348</td>\n",
       "      <td>2995</td>\n",
       "      <td>694</td>\n",
       "      <td>659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>External Substance</th>\n",
       "      <td>1.2</td>\n",
       "      <td>1045</td>\n",
       "      <td>646</td>\n",
       "      <td>218</td>\n",
       "      <td>181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Procedure</th>\n",
       "      <td>1.1</td>\n",
       "      <td>88863</td>\n",
       "      <td>62017</td>\n",
       "      <td>13063</td>\n",
       "      <td>13783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Therapeutic</th>\n",
       "      <td>1.0</td>\n",
       "      <td>61034</td>\n",
       "      <td>42675</td>\n",
       "      <td>8557</td>\n",
       "      <td>9802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Diagnostic</th>\n",
       "      <td>1.1</td>\n",
       "      <td>27829</td>\n",
       "      <td>19342</td>\n",
       "      <td>4506</td>\n",
       "      <td>3981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Total</th>\n",
       "      <td>1.1</td>\n",
       "      <td>246490</td>\n",
       "      <td>171358</td>\n",
       "      <td>36823</td>\n",
       "      <td>38309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Total</th>\n",
       "      <td>1.1</td>\n",
       "      <td>246490</td>\n",
       "      <td>171358</td>\n",
       "      <td>36823</td>\n",
       "      <td>38309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>O</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1687002</td>\n",
       "      <td>1163834</td>\n",
       "      <td>256568</td>\n",
       "      <td>266600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>O</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1687002</td>\n",
       "      <td>1163834</td>\n",
       "      <td>256568</td>\n",
       "      <td>266600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            Tokens / Mention    Total    Train     Dev    Test\n",
       "tag                                                                           \n",
       "Finding                                  1.1   132756    92053   19743   20960\n",
       "Diagnosis or Pathology                   1.1    81380    56216   12200   12964\n",
       "Other Finding                            1.1    51376    35837    7543    7996\n",
       "Substance                                1.2    24871    17288    4017    3566\n",
       "Clinical Drug                            1.2    19478    13647    3105    2726\n",
       "Nutrient or Body Substance               1.3     4348     2995     694     659\n",
       "External Substance                       1.2     1045      646     218     181\n",
       "Procedure                                1.1    88863    62017   13063   13783\n",
       "Therapeutic                              1.0    61034    42675    8557    9802\n",
       "Diagnostic                               1.1    27829    19342    4506    3981\n",
       "Total                                    1.1   246490   171358   36823   38309\n",
       "Total                                    1.1   246490   171358   36823   38309\n",
       "O                                        0.0  1687002  1163834  256568  266600\n",
       "O                                        0.0  1687002  1163834  256568  266600"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.concat([entity_counts['Short'], entity_counts['Short'].loc[:, ['Train', 'Dev', 'Test']].sum(axis=1).rename('Total')], axis=1)[['Tokens / Mention', 'Total', 'Train', 'Dev', 'Test']]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d3b757f-48ff-4570-a39a-45be31c96238",
   "metadata": {},
   "source": [
    "### Show results for long spans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cc5064b7-6dd9-44a5-aa65-6dc58107bc06",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tokens / Mention</th>\n",
       "      <th>Total</th>\n",
       "      <th>Train</th>\n",
       "      <th>Dev</th>\n",
       "      <th>Test</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tag</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Finding</th>\n",
       "      <td>2.5</td>\n",
       "      <td>105035</td>\n",
       "      <td>72860</td>\n",
       "      <td>15470</td>\n",
       "      <td>16705</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Diagnosis or Pathology</th>\n",
       "      <td>2.3</td>\n",
       "      <td>60898</td>\n",
       "      <td>42098</td>\n",
       "      <td>9042</td>\n",
       "      <td>9758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Other Finding</th>\n",
       "      <td>2.7</td>\n",
       "      <td>44521</td>\n",
       "      <td>31036</td>\n",
       "      <td>6469</td>\n",
       "      <td>7016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Substance</th>\n",
       "      <td>1.9</td>\n",
       "      <td>18169</td>\n",
       "      <td>12566</td>\n",
       "      <td>2976</td>\n",
       "      <td>2627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Clinical Drug</th>\n",
       "      <td>2.0</td>\n",
       "      <td>14092</td>\n",
       "      <td>9851</td>\n",
       "      <td>2257</td>\n",
       "      <td>1984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Nutrient or Body Substance</th>\n",
       "      <td>1.7</td>\n",
       "      <td>3277</td>\n",
       "      <td>2219</td>\n",
       "      <td>547</td>\n",
       "      <td>511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>External Substance</th>\n",
       "      <td>1.8</td>\n",
       "      <td>809</td>\n",
       "      <td>500</td>\n",
       "      <td>172</td>\n",
       "      <td>137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Procedure</th>\n",
       "      <td>2.5</td>\n",
       "      <td>77686</td>\n",
       "      <td>54121</td>\n",
       "      <td>11455</td>\n",
       "      <td>12110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Therapeutic</th>\n",
       "      <td>2.4</td>\n",
       "      <td>52887</td>\n",
       "      <td>36906</td>\n",
       "      <td>7415</td>\n",
       "      <td>8566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Diagnostic</th>\n",
       "      <td>2.5</td>\n",
       "      <td>25354</td>\n",
       "      <td>17618</td>\n",
       "      <td>4122</td>\n",
       "      <td>3614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Total</th>\n",
       "      <td>2.4</td>\n",
       "      <td>200890</td>\n",
       "      <td>139547</td>\n",
       "      <td>29901</td>\n",
       "      <td>31442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Total</th>\n",
       "      <td>2.4</td>\n",
       "      <td>201838</td>\n",
       "      <td>140228</td>\n",
       "      <td>30024</td>\n",
       "      <td>31586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>O</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1467393</td>\n",
       "      <td>1011438</td>\n",
       "      <td>223666</td>\n",
       "      <td>232289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>O</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1468288</td>\n",
       "      <td>1012055</td>\n",
       "      <td>223767</td>\n",
       "      <td>232466</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            Tokens / Mention    Total    Train     Dev    Test\n",
       "tag                                                                           \n",
       "Finding                                  2.5   105035    72860   15470   16705\n",
       "Diagnosis or Pathology                   2.3    60898    42098    9042    9758\n",
       "Other Finding                            2.7    44521    31036    6469    7016\n",
       "Substance                                1.9    18169    12566    2976    2627\n",
       "Clinical Drug                            2.0    14092     9851    2257    1984\n",
       "Nutrient or Body Substance               1.7     3277     2219     547     511\n",
       "External Substance                       1.8      809      500     172     137\n",
       "Procedure                                2.5    77686    54121   11455   12110\n",
       "Therapeutic                              2.4    52887    36906    7415    8566\n",
       "Diagnostic                               2.5    25354    17618    4122    3614\n",
       "Total                                    2.4   200890   139547   29901   31442\n",
       "Total                                    2.4   201838   140228   30024   31586\n",
       "O                                        0.0  1467393  1011438  223666  232289\n",
       "O                                        0.0  1468288  1012055  223767  232466"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.concat([entity_counts['Long'], entity_counts['Long'].loc[:, ['Train', 'Dev', 'Test']].sum(axis=1).rename('Total')], axis=1)[['Tokens / Mention', 'Total', 'Train', 'Dev', 'Test']]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dca8e957-74d7-41d5-9066-78b7cbfec37f",
   "metadata": {},
   "source": [
    "#### Number of files per split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1ee22fa5-97ea-4ff6-a3de-bf4fa1ff4fb8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "split\n",
       "Train    7135\n",
       "Dev      1529\n",
       "Test     1529\n",
       "Name: (Finding, Coarse), dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results['n_files'].iloc[0].loc['Short']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17ee4558-cfae-4b46-8fde-39f73615ec91",
   "metadata": {},
   "source": [
    "#### Sanity check: make sure the sums align for short spans (they cannot for long spans due to partial subsumption)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a1b18f87-0f59-4669-bf1f-39ea9787fb94",
   "metadata": {},
   "outputs": [],
   "source": [
    "for split in ['Train', 'Dev', 'Test']:\n",
    "    c_short = results[('count', 'Short', split)]\n",
    "    assert c_short.loc['O']['Coarse'] == c_short.loc['O']['Fine']\n",
    "    assert c_short.loc['Finding']['Coarse'] == c_short.loc['Diagnosis or Pathology']['Fine'] + c_short.loc['Other Finding']['Fine']\n",
    "    assert c_short.loc['Substance']['Coarse'] == c_short.loc['Clinical Drug']['Fine'] + c_short.loc['Nutrient or Body Substance']['Fine'] + c_short.loc['External Substance']['Fine']\n",
    "    assert c_short.loc['Procedure']['Coarse'] == c_short.loc['Therapeutic']['Fine'] + c_short.loc['Diagnostic']['Fine']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2356b4ff-0e35-4809-a263-1850524a2d90",
   "metadata": {},
   "source": [
    "# Inter-annotator Agreement"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33b65739-b225-472e-9f1b-39dafe43f1d9",
   "metadata": {},
   "source": [
    "__Note:__ Here we just analyze the IAA results.\n",
    "\n",
    "Use `python ../gamma_agreement.py <zip file>` for the actual IAA calculation using inception-analytics and pygamma-agreement .\n",
    "\n",
    "As the calculation of gamma agreement becomes very time-consuming with increasing number of annotators, the results are provided with the release in the `data` folder.\n",
    "\n",
    "Time to compute IAA using inception-analytics and pygamma-agreement:\n",
    "- Phase 1a:    185 s =    3 min = 0.05h\n",
    "- Phase 1b: 282532 s = 4709 min = __78 h__\n",
    "- Phase 2 : 200935 s = 3349 min = 56 h\n",
    "- Phase 3 :  78419 s = 1307 min = 22 h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8e59f562-fce7-41d8-8db5-8cd5bc7b36e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "agreement_results_path = Path('../data/annotations/agreement_sets/results/')\n",
    "agreement = pd.concat([pd.read_csv(f, index_col=0) for f in agreement_results_path.glob('*_gamma.csv')])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "61e2017c-7856-4490-9567-a4deda17912c",
   "metadata": {},
   "source": [
    "### Number of annotated sentences and docs used for IAA calculation (excluding sentences without any annotations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0e91c228-4101-4f4c-82b9-12b36e4ff993",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>n_docs</th>\n",
       "      <th>n_sentences</th>\n",
       "      <th>n_anno</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>file</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>phase_1a.zip</th>\n",
       "      <td>5</td>\n",
       "      <td>129</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>phase_1b.zip</th>\n",
       "      <td>5</td>\n",
       "      <td>132</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>phase_2.zip</th>\n",
       "      <td>6</td>\n",
       "      <td>147</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>phase_3.zip</th>\n",
       "      <td>3</td>\n",
       "      <td>61</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              n_docs  n_sentences  n_anno\n",
       "file                                     \n",
       "phase_1a.zip       5          129       3\n",
       "phase_1b.zip       5          132       7\n",
       "phase_2.zip        6          147       7\n",
       "phase_3.zip        3           61       7"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agreement.groupby('file')[['n_docs', 'n_sentences', 'n_anno']].max()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c117316f-5fac-4e6d-a196-75e15b926e25",
   "metadata": {},
   "source": [
    "### Total number of sentences and docs in the agreement sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9885eb1b-f475-4490-9ec7-6c91db37e58f",
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m spacy download de_core_news_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "5f10bacb-a5b1-46c7-b278-56c8b0777cf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('de_core_news_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3cfd702d-2593-441f-866b-c790d18c35b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>n_docs</th>\n",
       "      <th>n_sentences_total</th>\n",
       "      <th>n_tokens_total</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>file</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>phase_1</th>\n",
       "      <td>5</td>\n",
       "      <td>149</td>\n",
       "      <td>4206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>phase_2</th>\n",
       "      <td>6</td>\n",
       "      <td>158</td>\n",
       "      <td>3725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>phase_3</th>\n",
       "      <td>3</td>\n",
       "      <td>67</td>\n",
       "      <td>1814</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         n_docs  n_sentences_total  n_tokens_total\n",
       "file                                              \n",
       "phase_1       5                149            4206\n",
       "phase_2       6                158            3725\n",
       "phase_3       3                 67            1814"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats = []\n",
    "\n",
    "for folder in Path('../data/annotations/agreement_sets/input/').glob(r'phase_?'):\n",
    "    txt_files = list(folder.glob('*.txt'))\n",
    "    n_sentences = 0\n",
    "    n_tokens = 0\n",
    "    for f in txt_files:\n",
    "        with open(f) as txt_file:\n",
    "            sentences = txt_file.readlines()\n",
    "            n_sentences += len(sentences)\n",
    "            n_tokens += sum([len(nlp(s)) for s in sentences])\n",
    "    \n",
    "    stats.append({\n",
    "        'file' : folder.name,\n",
    "        'n_docs': len(txt_files),\n",
    "        'n_sentences_total': n_sentences,\n",
    "        'n_tokens_total': n_tokens\n",
    "    })\n",
    "stats = pd.DataFrame(stats).set_index('file').sort_index()\n",
    "stats"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd7679c8-f2b4-4f16-aea2-b61178319d4c",
   "metadata": {},
   "source": [
    "### IAA results for all entity classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "15d7cea9-154b-4183-a1a2-1f4a39e2198a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file</th>\n",
       "      <th>phase_1a.zip</th>\n",
       "      <th>phase_1b.zip</th>\n",
       "      <th>phase_2.zip</th>\n",
       "      <th>phase_3.zip</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature</th>\n",
       "      <th>label</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">coarse</th>\n",
       "      <th>all</th>\n",
       "      <td>0.75</td>\n",
       "      <td>0.89</td>\n",
       "      <td>0.93</td>\n",
       "      <td>0.94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Finding</th>\n",
       "      <td>0.82</td>\n",
       "      <td>0.93</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Substance</th>\n",
       "      <td>0.92</td>\n",
       "      <td>0.99</td>\n",
       "      <td>0.98</td>\n",
       "      <td>0.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Procedure</th>\n",
       "      <td>0.82</td>\n",
       "      <td>0.93</td>\n",
       "      <td>0.96</td>\n",
       "      <td>0.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Specification</th>\n",
       "      <td>0.71</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.91</td>\n",
       "      <td>0.89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"8\" valign=\"top\">fine</th>\n",
       "      <th>all</th>\n",
       "      <td>-</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.92</td>\n",
       "      <td>0.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Diagnosis or Pathology</th>\n",
       "      <td>-</td>\n",
       "      <td>0.91</td>\n",
       "      <td>0.94</td>\n",
       "      <td>0.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Other Finding</th>\n",
       "      <td>-</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Clinical Drug</th>\n",
       "      <td>-</td>\n",
       "      <td>0.97</td>\n",
       "      <td>0.98</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Nutrient or Body Substance</th>\n",
       "      <td>-</td>\n",
       "      <td>0.99</td>\n",
       "      <td>0.99</td>\n",
       "      <td>0.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>External Substance</th>\n",
       "      <td>-</td>\n",
       "      <td>0.96</td>\n",
       "      <td>-</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Therapeutic</th>\n",
       "      <td>-</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.96</td>\n",
       "      <td>0.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Diagnostic</th>\n",
       "      <td>-</td>\n",
       "      <td>0.89</td>\n",
       "      <td>0.98</td>\n",
       "      <td>0.93</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "file                               phase_1a.zip  phase_1b.zip phase_2.zip  \\\n",
       "feature label                                                               \n",
       "coarse  all                                0.75          0.89        0.93   \n",
       "        Finding                            0.82          0.93        0.95   \n",
       "        Substance                          0.92          0.99        0.98   \n",
       "        Procedure                          0.82          0.93        0.96   \n",
       "        Specification                      0.71          0.87        0.91   \n",
       "fine    all                                   -          0.88        0.92   \n",
       "        Diagnosis or Pathology                -          0.91        0.94   \n",
       "        Other Finding                         -          0.85        0.87   \n",
       "        Clinical Drug                         -          0.97        0.98   \n",
       "        Nutrient or Body Substance            -          0.99        0.99   \n",
       "        External Substance                    -          0.96           -   \n",
       "        Therapeutic                           -          0.95        0.96   \n",
       "        Diagnostic                            -          0.89        0.98   \n",
       "\n",
       "file                                phase_3.zip  \n",
       "feature label                                    \n",
       "coarse  all                                0.94  \n",
       "        Finding                            0.97  \n",
       "        Substance                          0.99  \n",
       "        Procedure                          0.96  \n",
       "        Specification                      0.89  \n",
       "fine    all                                0.93  \n",
       "        Diagnosis or Pathology             0.96  \n",
       "        Other Finding                      0.91  \n",
       "        Clinical Drug                      1.00  \n",
       "        Nutrient or Body Substance         0.98  \n",
       "        External Substance                 1.00  \n",
       "        Therapeutic                        0.96  \n",
       "        Diagnostic                         0.93  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.options.display.float_format = '{:,.2f}'.format\n",
    "agreement.replace({'detail' : 'fine', 'value' : 'coarse'}).set_index(['file', 'feature', 'label']).unstack('file').gamma.sort_index(ascending=True, key=sort_fn).fillna('-')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ad2af41-b8e8-4492-a5b1-a41e491b171b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:ggponc]",
   "language": "python",
   "name": "conda-env-ggponc-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}