model_training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='Top'></a>\n",
    "\n",
    "# Multisurv model training<a class='tocSkip'></a>\n",
    "\n",
    "Train MultiSurv models with different combinations of input data modalities."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "code_folding": [],
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> PyTorch detected CUDA <<<\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "%load_ext watermark\n",
    "\n",
    "import sys\n",
    "import os\n",
    "\n",
    "import ipywidgets as widgets\n",
    "import pandas as pd\n",
    "import torch\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    print('>>> PyTorch detected CUDA <<<')\n",
    "\n",
    "# Make modules in \"src\" dir visible\n",
    "if os.getcwd() not in sys.path:\n",
    "    sys.path.append(os.path.join(os.getcwd(), 'src'))\n",
    "\n",
    "import utils\n",
    "from model import Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#DataLoader\" data-toc-modified-id=\"DataLoader-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span><code>DataLoader</code></a></span></li><li><span><a href=\"#Model\" data-toc-modified-id=\"Model-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Model</a></span><ul class=\"toc-item\"><li><span><a href=\"#Different-intervals\" data-toc-modified-id=\"Different-intervals-2.1\"><span class=\"toc-item-num\">2.1&nbsp;&nbsp;</span>Different intervals</a></span><ul class=\"toc-item\"><li><span><a href=\"#Equidistant-times\" data-toc-modified-id=\"Equidistant-times-2.1.1\"><span class=\"toc-item-num\">2.1.1&nbsp;&nbsp;</span>Equidistant times</a></span></li><li><span><a href=\"#By-duration-quantiles\" data-toc-modified-id=\"By-duration-quantiles-2.1.2\"><span class=\"toc-item-num\">2.1.2&nbsp;&nbsp;</span>By duration quantiles</a></span></li></ul></li><li><span><a href=\"#Pick-learning-rate\" data-toc-modified-id=\"Pick-learning-rate-2.2\"><span class=\"toc-item-num\">2.2&nbsp;&nbsp;</span>Pick learning rate</a></span></li><li><span><a href=\"#Fit\" data-toc-modified-id=\"Fit-2.3\"><span class=\"toc-item-num\">2.3&nbsp;&nbsp;</span>Fit</a></span><ul class=\"toc-item\"><li><span><a href=\"#Save-model-weights\" data-toc-modified-id=\"Save-model-weights-2.3.1\"><span class=\"toc-item-num\">2.3.1&nbsp;&nbsp;</span>Save model weights</a></span></li></ul></li><li><span><a href=\"#Check-validation-metrics\" data-toc-modified-id=\"Check-validation-metrics-2.4\"><span class=\"toc-item-num\">2.4&nbsp;&nbsp;</span>Check validation metrics</a></span></li></ul></li></ul></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA = utils.INPUT_DATA_DIR\n",
    "MODELS = utils.TRAINED_MODEL_DIR\n",
    "\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `DataLoader`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "287c833f549b4c689e3eb9d2caa4f1c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "SelectMultiple(description='Input data', index=(0, 1), options=('clinical', 'mRNA', 'DNAm', 'miRNA', 'CNV', 'w…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_modalities = widgets.SelectMultiple(\n",
    "    options=['clinical', 'mRNA', 'DNAm', 'miRNA', 'CNV', 'wsi'],\n",
    "    index=[0, 1],\n",
    "    rows=6,\n",
    "    description='Input data',\n",
    "    disabled=False\n",
    ")\n",
    "display(data_modalities)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "#-----------------------------------------------------------------------------#\n",
    "#                             20-CANCER SUBSET                                #\n",
    "#                 (to compare to Cheerla and Gevaert 2019)                    #\n",
    "#-----------------------------------------------------------------------------#\n",
    "\n",
    "cancers = ['BLCA', 'BRCA', 'CESC', 'COAD', 'READ',\n",
    "           'HNSC', 'KICH', 'KIRC', 'KIRP', 'LAML',\n",
    "           'LGG', 'LIHC', 'LUAD', 'LUSC', 'OV',\n",
    "           'PAAD', 'PRAD', 'SKCM', 'STAD', 'THCA', 'UCEC']\n",
    "\n",
    "labels = pd.read_csv('data/labels.tsv', sep='\\t')\n",
    "print(labels.head(3))\n",
    "\n",
    "# List of patients to exclude: patients with cancers that are not in the subset\n",
    "exclude_cancers = list(labels.loc[~labels['project_id'].isin(cancers), 'submitter_id'])\n",
    "len(exclude_cancers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data modalities:\n",
      "   clinical\n",
      "   mRNA\n",
      "\n",
      "Dataset sizes (# patients):\n",
      "   train: 8880\n",
      "   val: 1109\n",
      "   test: 1092\n",
      "\n",
      "Batch size: 128\n"
     ]
    }
   ],
   "source": [
    "dataloaders = utils.get_dataloaders(data_location=DATA,\n",
    "                                    labels_file='data/labels.tsv',\n",
    "                                    modalities=data_modalities.value,\n",
    "                                    wsi_patch_size=299,\n",
    "                                    n_wsi_patches=5,\n",
    "#                                     batch_size=20,\n",
    "#                                     batch_size=64,\n",
    "#                                     batch_size=32,\n",
    "#                                     exclude_patients=exclude_cancers,\n",
    "                                   )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Different intervals\n",
    "\n",
    "If trying out different time interval outputs."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Equidistant times"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "interval_cuts = torch.arange(0., 365 * 5.1, 365 / 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### By duration quantiles"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "labels = [(t, e) for t, e in dataloaders['train'].dataset.label_map.values()]\n",
    "durations = [t for t, _ in labels]\n",
    "events = [e for _, e in labels]\n",
    "\n",
    "interval_cuts = utils.discretize_time_by_duration_quantiles(durations, events, 20)\n",
    "interval_cuts = torch.from_numpy(interval_cuts)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "#-----------------------------------------------------------------------------#\n",
    "#                       PRE-TRAINED UNIMODAL MODELS                           #\n",
    "#-----------------------------------------------------------------------------#\n",
    "\n",
    "unimodal_weigths = {'clinical': 'clinical_lr0.005_epoch49_acc0.78.pth',\n",
    "                    'mRNA': 'mRNA_lr0.005_epoch54_acc0.76.pth',\n",
    "                    'DNAm': 'DNAm_lr0.005_epoch57_acc0.77.pth',\n",
    "                    'miRNA': None,\n",
    "                    'CNV': None,\n",
    "                    'wsi': None,}\n",
    "\n",
    "unimodal_weigths = {k: os.path.join(MODELS, v) if v is not None else None\n",
    "                    for k, v in unimodal_weigths.items()}\n",
    "\n",
    "multisurv = Model(dataloaders=dataloaders,\n",
    "                  unimodal_state_files=unimodal_weigths,\n",
    "                  freeze_up_to='aggregator',\n",
    "                  device=device)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "#-----------------------------------------------------------------------------#\n",
    "#                              AUXILIARY LOSS                                 #\n",
    "#-----------------------------------------------------------------------------#\n",
    "\n",
    "# cosine_embedding_margin = 1e-5\n",
    "# auxiliary_criterion = torch.nn.CosineEmbeddingLoss(margin=cosine_embedding_margin)\n",
    "\n",
    "multisurv = Model(dataloaders=dataloaders,\n",
    "                  auxiliary_criterion=auxiliary_criterion,\n",
    "                  device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Instantiating MultiSurv model...\n"
     ]
    }
   ],
   "source": [
    "multisurv = Model(dataloaders=dataloaders,\n",
    "#                   fusion_method='attention',\n",
    "#                   output_intervals=interval_cuts,\n",
    "                  device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output intervals (in years):\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([ 0.,  1.,  2.,  3.,  4.,  5.,  6.,  7.,  8.,  9., 10., 11., 12., 13.,\n",
       "        14., 15., 16., 17., 18., 19., 20., 21., 22., 23., 24., 25., 26., 27.,\n",
       "        28., 29., 30.])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('Output intervals (in years):')\n",
    "multisurv.output_intervals / 365"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['clinical_submodel', 'mRNA_submodel', 'aggregator', 'fc_block', 'risk_layer']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multisurv.model_blocks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trainable blocks:\n",
      "   clinical_submodel: True\n",
      "   mRNA_submodel: True\n",
      "   fc_block: True\n",
      "   risk_layer: True\n"
     ]
    }
   ],
   "source": [
    "print('Trainable blocks:')\n",
    "layer = None\n",
    "\n",
    "for name, child in multisurv.model.named_children():\n",
    "    for name_2, params in child.named_parameters():\n",
    "        if name is not layer:\n",
    "            print(f'   {name}: {params.requires_grad}')\n",
    "        layer = name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultiSurv(\n",
       "  (clinical_submodel): ClinicalNet(\n",
       "    (embedding_layers): ModuleList(\n",
       "      (0): Embedding(33, 17)\n",
       "      (1): Embedding(2, 1)\n",
       "      (2): Embedding(8, 4)\n",
       "      (3): Embedding(3, 2)\n",
       "      (4): Embedding(3, 2)\n",
       "      (5): Embedding(3, 2)\n",
       "      (6): Embedding(3, 2)\n",
       "      (7): Embedding(3, 2)\n",
       "      (8): Embedding(20, 10)\n",
       "    )\n",
       "    (linear): Linear(in_features=43, out_features=256, bias=True)\n",
       "    (embedding_dropout): Dropout(p=0.5, inplace=False)\n",
       "    (bn_layer): BatchNorm1d(1, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (output_layer): FC(\n",
       "      (fc): Sequential(\n",
       "        (0): Dropout(p=0.5, inplace=False)\n",
       "        (1): Linear(in_features=256, out_features=512, bias=True)\n",
       "        (2): ReLU(inplace=True)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (mRNA_submodel): FC(\n",
       "    (fc): Sequential(\n",
       "      (0): Dropout(p=0.5, inplace=False)\n",
       "      (1): Linear(in_features=1000, out_features=2048, bias=True)\n",
       "      (2): ReLU(inplace=True)\n",
       "      (3): BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (4): Dropout(p=0.5, inplace=False)\n",
       "      (5): Linear(in_features=2048, out_features=1024, bias=True)\n",
       "      (6): ReLU(inplace=True)\n",
       "      (7): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (8): Dropout(p=0.5, inplace=False)\n",
       "      (9): Linear(in_features=1024, out_features=512, bias=True)\n",
       "      (10): ReLU(inplace=True)\n",
       "    )\n",
       "  )\n",
       "  (aggregator): Fusion()\n",
       "  (fc_block): FC(\n",
       "    (fc): Sequential(\n",
       "      (0): Dropout(p=0.5, inplace=False)\n",
       "      (1): Linear(in_features=512, out_features=2048, bias=True)\n",
       "      (2): ReLU(inplace=True)\n",
       "      (3): BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (4): Dropout(p=0.5, inplace=False)\n",
       "      (5): Linear(in_features=2048, out_features=1024, bias=True)\n",
       "      (6): ReLU(inplace=True)\n",
       "      (7): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (8): Dropout(p=0.5, inplace=False)\n",
       "      (9): Linear(in_features=1024, out_features=512, bias=True)\n",
       "      (10): ReLU(inplace=True)\n",
       "      (11): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (12): Dropout(p=0.5, inplace=False)\n",
       "      (13): Linear(in_features=512, out_features=512, bias=True)\n",
       "      (14): ReLU(inplace=True)\n",
       "    )\n",
       "  )\n",
       "  (risk_layer): Sequential(\n",
       "    (0): Linear(in_features=512, out_features=30, bias=True)\n",
       "    (1): Sigmoid()\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multisurv.model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pick learning rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Instantiating MultiSurv model...\n",
      ">>> Compute loss at increasing LR values\n",
      "    Iterate over mini-batches: 69\n",
      "    Completed test.\n",
      "\n",
      "CPU times: user 4.46 s, sys: 193 ms, total: 4.65 s\n",
      "Wall time: 2min 55s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "multisurv.test_lr_range()\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVEAAADVCAYAAADuIMM4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deXwV9bnH8c+XhC0JYQsugAoqtbWKG9elLkUrrWiVulyX0iIqRRS3or0utVWxtuoVF0QF7FVxrztU8aq3iBYVBURRUSrihqIm7JuEwHP/+A32eMxywslkzkme9+s1r8x65nuG8GTW38jMcM45t3laJB3AOefymRdR55zLghdR55zLghdR55zLghdR55zLghdR55zLghdRFxtJT0s6OekczsXJi2gTJOkjSYcmncPM+pvZhKRzAEiaKmlI0jni0FDfTVJfSQsbIlNz4kXUbRZJhUln2KSxsyjw/zsO8CLa7Ej6uaQ3JC2T9LKk3inTLpL0gaSVkuZKOjpl2mBJL0m6QdIS4PJo3DRJ10laKulDSf1TlvlmDymDeXtKejFa9/9JukXSvTV8h76SFkq6UNIXwJ2SOkp6UlJ59PlPSuoezX8VcCAwRtIqSWOi8d+X9JykJZLmSTq+lu02VdJVkl4C1gDbSzpF0rtR5gWSTq8m4/mSvpK0SNIpKdM7S/q7pBWSZkj6k6RpKdMzyrY5303S4dG/70pJn0m6QFIx8DTQNfqcVZK61rQ9XAoz866JdcBHwKHVjN8T+ArYBygATo7mbR1N/0+gK+GP6wnAamDraNpgoAo4GygE2kbj1gO/iT7vDOBzQNEyU4EhKcvXNu8rwHVAK+AAYAVwbw3fr2+U5RqgdZSlM3AsUAS0Ax4GnkhZ5pss0XAx8ClwSvR99gQqgB/WsM6pwCfAD6P5WwJHADsAAn5MKK57pmUcGc17eDS9YzT9wagrAnaOskzLIlvG3w1YBBwY9XdMy7ww6d/ffOsSD+BdDP+oNRfR24Ar08bNA35cw+e8AQyI+gcDn6RNHwzMTxkuAgzYKhr+5j93bfMC20YFpyhl+r11FNFKoE0t22B3YGnKcHqhOQH4Z9oy44DLavi8qcDIOrb7E8C5KRnXAoUp078C9iX8EVkP7JQy7U8pRXRzsmX83Qh/DE4HSqvZrl5E69n54Xzzsh1wfnQov0zSMmAbwt4nkgalHOovA3YBylKW/7Saz/xiU4+ZrYl6S2pYf03zdgWWpIyraV2pys3s600DkookjZP0saQVwItAB0kFNSy/HbBP2rYYSCjqNflWJkn9JU2PDpmXEfY2U7fXYjOrShleE33fLoQ9xNTPS+3fnGz1+W7HRlk/lvSCpP0y/FxXjZy5OOAaxafAVWZ2VfoESdsBtwM/AV4xsw2S3iAcqm4SV5Nfi4BOkopSCuk2dSyTnuV8YCdgHzP7QtLuwGz+nT99/k+BF8ysXz1yfvMZkloDjwKDgIlmtl7SE3x7e9WknLDn3R34VzQu9fvWN1u9vpuZzQAGSGoJnAU8FK3fm3TbDL4n2nS1lNQmpSskFMlhkvZRUCzpCEntCOfRjPAfnOgiyC6NEdTMPgZmEi5WtYr2jI6s58e0Ixw+L5PUCbgsbfqXwPYpw08C35P0a0kto+4/JP0gw/W1IpyPLQeqootkP81kQTPbADxG+L5Fkr5PKMabmy3j7xZt34GS2pvZesK55w0pn9NZUvsMt4HDi2hTNplQVDZ1l5vZTMKFnTHAUmA+4VwlZjYXGEW4wPMlsCvwUiPmHQjsBywmnB/8G7CuHsvfSLjAVAFMB/43bfpNwHEKV+5Hm9lKQtE7kXCB6wv+faGqTtHy5xD24pYCvwQm1SPvWUD7aL33AA8Qfd/NyFbf7/Zr4KPotMcw4FfRet+LciyITgP41fkMbLoy6lxOkfQ34D0zS9+jbJIkXUO4IOdPeOUZ3xN1OSE63NxBUgtJhwEDCFe7m6ToPs7e0WmVvYHTgMeTzuXqzy8suVyxFeE8YWdgIXCGmc1ONlKs2hEOnbsSbn0aBUxMNJHbLH4475xzWfDDeeecy4IXUeecy0KTOidaVlZm3bt3p7AwfK2qqqrv9Kf/rI+6lqluevq42oZryuhZ685aU77qstY0riHy1jQtk21ZV+5c+z3Ip6y1Zcwk66xZsyrMrEu1E5N+7rQhu7322svKy8ttk+r603/WR13LVDc9fVxtwzVl9Kx1Z60pX03Z4spb07RMtmVdGXPt9yCfstaWMZOswExL4tl5SYdFzXDNl3RRNdMlaXQ0fY6kPVOmdZD0iKT3FJob8+d7nXM5J7YiGjX8cAvQn9DU10mSdk6brT/QK+qGEloZ2uQm4H/N7PvAbsC7cWV1zrnNFeee6N6Eps8WmFkloe3EAWnzDADujvaYpxNa3dlaUilwEPA/AGZWaWbLYszqnHObJc4i2o1vN++1MBqXyTzbExp2uFPSbEl/jVreds65zTdnDgXvv9+gHxlnEa2uSbD0O/trmmdTa9y3mdkehBbWv3NOFUDSUEkzJc0sLy/PJq9zrqlav562o0ZBnz4UX9awzTHEWUQX8u02ErsTWpTJZJ6FhBa2X43GP0Ioqt9hZuPNrI+Z9enSpfo7EJxzzdhbb8G++1J89dVw3HGsHDOmQT8+ziI6A+il8AKyVoRmudKbCpsEDIqu0u8LLDezRWb2BfCppJ2i+X4CzI0xq3OuqamqCnufe+0FCxey4s474f77sU6dGnQ1sd1sb2ZVks4CniG8U+YOM3tH0rBo+lhCm5eHE9q1XEN4sdYmZwP3RQV4Qdo055yr2caNMGQIxRMmwAknwJgxVMa0qlifWDKzyYRCmTpubEq/AcNrWPYNoE+c+ZxzTZAZjBgBEyaw5ne/o+jaa8P4iopYVufPzjvnmpaRI+Gmm+Dcc1nzu9/Fvjovos65JqPNuHFw+eUweDBcfz0ok/cGZqdJNUDinGumzOC22yi59FI4+mi4/XZo0Tj7iF5EnXP5bd48OOMMeP55Kg8+mFYPPAD1bEEqG34475zLT2vXUnT11dC7N8yeDWPHsuLBB6F1Ri9sbTC+J+qcyz+vvQYDB1I0fz4MHAijRsGWW8Z2Bb42vifqnMsfZnDjjXDAAbB+PcsfeQTuvTcU0IT4nqhzLi9o2TIYMgQmToQBA+DOO1m/YUPSsXxP1DmXB159lQ6HHAKTJ8MNN8Djj0PHjkmnAryIOudy2YYN8Je/hMN3gGnT4LzzGuX+z0x5EXXO5abPPoN+/eCSS+CYY1j2/POw995Jp/oOL6LOudwzcWK4dem11+COO+DBB7H27ZNOVS0vos653GEGV1wBv/gF9OgBr78Op5ySU4fv6fzqvHMuN6xbByefDPfcA4MGwfjxjX7j/ObwIuqcS96SJbQ//nh4+WW48kr4/e9zeu8zlRdR51yyPvwQ+ven8MMP4b774Je/TDpRvXgRdc4lZ9kyOOwwKC9n+SOP0OHII5NOVG9eRJ1zydiwIex1LlgAU6ZQ9YMfJJ1os/jVeedcMi65BJ5+GsaMgQMPTDrNZvMi6pxrfPffD9deC8OGwemnJ50mK15EnXONa+ZMOO00OOig8C6kPOdF1DnXeKZPh6OOgi22gIcfhlatkk6UNS+izrn4mcHo0WHvs00beOqpUEibAC+izrl4rVgBJ5wA554L/fvDrFmwyy5Jp2owfouTcy4+770XGlD+4INwIemCC/LmSaRMeRF1zsXjzTfh0EPDq4unTAmH8k2QH8475xpc4ezZcPDB4fzntGlNtoCCF1HnXEN76SVKjz0WOnSAF1+EXr2SThQrL6LOuQbT8qWX4Gc/w7p0CQW0Z8+kI8XOi6hzrmG89Ralv/wlbLcdyyZOhO7dk07UKPzCknMua1qyBAYMYGO7dhQ89xzWBG6iz5TviTrnslNVRbvf/AY++4yVEyZA165JJ2pUvifqnMvO735HqxdfhDvuoGqvvZJO0+hi3ROVdJikeZLmS7qomumSNDqaPkfSninTPpL0lqQ3JM2MM6dzbjPddRfceCNrhw4NL5RrhmLbE5VUANwC9AMWAjMkTTKzuSmz9Qd6Rd0+wG3Rz00ONrOKuDI65zZf4axZoRm7Qw5h9RVX0DbpQAmJc090b2C+mS0ws0rgQWBA2jwDgLstmA50kLR1jJmccw3hs88oHTQIunWDhx6CwuZ7ZjDOItoN+DRleGE0LtN5DHhW0ixJQ2taiaShkmZKmlleXt4AsZ1ztVq7Fo4+Gq1aBRMnQufOSSdKVJxFtLpWBqwe8+xvZnsSDvmHS6r2uTEzG29mfcysT5cuXTY/rXOubmaUnH8+zJjByttug113TTpR4uIsoguBbVKGuwOfZzqPmW36+RXwOOH0gHMuSaNG0ebhh2HkSCoPPzzpNDkhziI6A+glqaekVsCJwKS0eSYBg6Kr9PsCy81skaRiSe0AJBUDPwXejjGrc64uTz8N//VfrDvySLj00qTT5IzYzgabWZWks4BngALgDjN7R9KwaPpYYDJwODAfWANsukdiS+BxhXYHC4H7zex/48rqnKvDokXwq19B796svPlmWjexNkGzEeslNTObTCiUqePGpvQbMLya5RYAu8WZzTmXIbPwYrm1a+Fvf4Pi4qQT5RR/7NM5V6s2EyaEQ/lrr4Wddko6Ts7xIuqcq9n771N82WXQrx+ceWbSaXKSF1HnXPWqqmDQIKxlS7jzzvCaD/cdzfcxA+dc7a65BqZPZ9W4cZR2S39Oxm3if1qcc9/1xhtw+eVwwglUHnNM0mlymhdR59y3VVbC4MFQVga33pp0mpznh/POuW9pe9NN4XXHEydCp05Q4Q2p1cb3RJ1z//bmmxRdfz0MHAhHHZV0mrzgRdQ5F6xfD4MHYx07wk03JZ0mb/jhvHMuuPpqeOMNVk2YQGkzb96uPnxP1DkHc+bAlVfCSSd560z15EXUueaushJOPhk6doTRo5NOk3cyKqKSdpDUOurvK+kcSR3ijeacaxRXXBHuC7399nBbk6uXTPdEHwU2SNoR+B+gJ3B/bKmcc42i8LXXwrnQU0/1q/GbKdMiutHMqoCjgRvN7LeAv1DOuXy2ahXthg+HbbeFG25IOk3eyvTq/HpJJwEnA0dG41rGE8k51yguuIAWH38ML7wApaVJp8lbme6JngLsB1xlZh9K6gncG18s51ysJk+GceNYO3w4HHhg0mnyWkZ7omY2FzgHQFJHoJ2ZXR1nMOdcTJYuDS3V77oray66iKKk8+S5jIqopKnAUdH8bwDlkl4wsxExZnPOxWHkSPjqq7A32rp10mnyXqaH8+3NbAVwDHCnme0FHBpfLOdcLN57D8aMgSFDYI89kk7TJGRaRAslbQ0cDzwZYx7nXJwuuACKisLTSa5BZFpERxJeffyBmc2QtD3wfnyxnHMN7pln4Kmn4A9/gC22SDpNk5HphaWHgYdThhcAx8YVyjnXwKqqYMQI2GEHOPvspNM0KZk+9tld0uOSvpL0paRHJXWPO5xzroGMGwdz58KoUX4xqYFlejh/JzAJ6Ap0A/4ejXPO5bolS+CPf4RDDvFHO2OQaRHtYmZ3mllV1N0FdIkxl3OuIWzcCGedBcuWhUc7paQTNTmZFtEKSb+SVBB1vwIWxxnMOZeljRspOf98eOAB+NOfoHfvpBM1SZkW0VMJtzd9ASwCjiM8Cuqcy0VmcM45tLn3Xrj0Urj44qQTNVkZFVEz+8TMjjKzLma2hZn9gnDjvXMu15iF+0FvuYU1w4eHJ5RcbLJp2d4f+XQuF116KVx/PZxzDmsuu8zPg8YsmyLq/zLO5ZpHH4U//xmGDoUbb/QC2giyKaLWYCmcc9lbtAhOPx369AnPx3sBbRS1FlFJKyWtqKZbSbhntFaSDpM0T9J8SRdVM12SRkfT50jaM216gaTZkvx5fedqYxYaFVm9Gu65B1p6m+mNpdbHPs2s3eZ+sKQC4BagH7AQmCFpUtQ26Sb9gV5Rtw9wW/Rzk3OBdwFvdtu52owfH5q2Gz0avv/9pNM0K3G+MnlvYL6ZLTCzSuBBYEDaPAOAuy2YDnSIWosieqz0COCvMWZ0Lv/Nnx+eiz/0UBg+POk0zU6cRbQb8GnK8MJoXKbz3Aj8F7CxtpVIGipppqSZ5eXl2SV2Lt9UVcGvfw2tWsGdd0KLOP9Lu+rEucWrO6udfjGq2nkk/Rz4ysxm1bUSMxtvZn3MrE+XLv4kqmtm/vu/Yfp0uPVW6O5tAiUhziK6ENgmZbg78HmG8+wPHCXpI8JpgEMk+YvxnEs1bx5ccQUceyycdFLSaZqtOIvoDKCXpJ6SWgEnElqCSjUJGBRdpd8XWG5mi8zsYjPrbmY9ouWmmNmvYszqXH7ZuDFcjS8qCrczucRk+t75ejOzKklnEVrELwDuMLN3JA2Lpo8FJgOHA/OBNfjz+M5lZuxYmDYtnAfdaquk0zRrsRVRADObTCiUqePGpvQbUOvlRDObCkyNIZ5z+emTT+DCC6FfPzj55KTTNHt+Kc+5fGIGZ5wRfo4f708l5YBY90Sdcw3sgQfCTfU33gg9eiSdxuF7os7ljw0bwk31++4bWqt3OcH3RJ3LE4WzZ8OXX8JNN0FBQdJxXMT3RJ3LEy2nTg3nQA89NOkoLoUXUefyRKvnnw/N3HXunHQUl8KLqHP5YPlyCmfNgp/9LOkkLo0XUefywZQpaMMG+OlPk07i0ngRdS4fPPMMG0tKwpV5l1O8iDqX68zgmWdYf+CB3mJ9DvIi6lyOa7FgAXz0Eev79k06iquGF1HnclyrqVMBqDz44ERzuOp5EXUux7V8/nnYfns29uyZdBRXDS+izuWyykpaTpvmtzblMC+izuWyV16hxerVfmtTDvNn52tTWQmffw5vvw1vv03JrFmwcCH8+Mdwyimw005JJ3RN3bPPYgUF6JBDwu+jyznNtogWzJkD77wTiuSmbtEiWL4cVq+GVatg/fpvLdOyWzfYbju47jq45hr40Y/g1FPhmGOgY8eEvolr0p55hqo+fWhZWgoVFUmncdVotkW01XPPwdVXh/vutt4aunaF730POnSAkpLQFRdDWRnssgvssgtL16+nrKwMvvgC7r0X7rgjvOdmyBDYaitKd9wRevcOe6idOkFpKS3NYJttwno2bqRg8WIoLQ2vul29mlaLFoXX3K5aRZtly6CkhDYrV0JJCa3XrYOddqKgpAR23TUU6k2N8FZVwddfh3ftlJT4q3KbIFVUwOuvU3nhhfjdobmr2RbRr087jeIRI0JjDpkWoE17AlttBRdcAOefD6++Ci++CO++i956KxTXFSu+WaR92kek76+WpvSXpP1sl75MUVHI+vXXoYimatcO2rcP3a670qZ37/D6iN128xu081Sr554DM9b7rU05rdkWUevQIexlZkMKj+FFj+Itr6igrHPnUGyXLYMVK1j+6ae0l0LRa9GCFatWUdqhAxQWQnExS9evp2P37lBczOKVK+ncuTOLFy+mc1kZSz7/nE5r17LinXcoXbEinI81g7ZtoU2b0EmwcmUo3MuXw+LF8M9/UvLgg3DJJWHeffYJ53H79g1Z27TJfgO6eFVWUnT99bDrrlTtvnvSaVwtmm0RjY0EXbqEDli/3XbfKtaVFRXfGt6QMmxRv7VoAWVlbJSgrIzKHXesd8Ff8uabdJo3D15+ObwV8sorwzvKW7WCffahaI89QruU++77TVaXQ8aPp+Cjj8KrQPxUTU7zItpEbezWLRzKH398GLF8eSimU6fCCy/Q9tZbYfToMG3HHeEnP6Hw8MPh5z/3/7RJW7kSRo6kcv/9aXXYYeHowuUsL6LNRfv2cMQRoQMWf/opZR9/HPZUX34Z7rmHDuPGhYtgJ50EAweGC26u8V13HZSXs+a++2jlb/PMeV5Em6u2beGAA0IHsGoVK++7j3aTJsGoUXDttXTs2ROOPRaOPtqbYGsk+vLLsP3/8z+p2mOPpOO4DPhxmwtKSlh37LHw1FPhftmxY9nQo0d4Kdr++0PXrhSNHBlelOZiUzRqFKxbB1ddlXQUlyEvou67unSB009nxUMPQXk53H8//OhHtL3lFujZE377W1p88UXSKZuef/2LNnffDUOHQq9eSadxGfIi6mrXvn04R/rYYyx9+eVwoermm+nYpw8MH07B/PlJJ2wazODCC6F1a/jjH5NO4+rBi6jL2MYddoC77oJ581h33HFw++103G8/6NePVpMnf/cBAJe5m2+GJ55gzQUXwJZbJp3G1YMXUVd/O+zAqhtvhE8+YfXFF8N771F68smw/fa0GTs2PFHlMlb48svh6bcBA1g7fHjScVw9eRF1m2+rrVg7YgR8+CErJkyAHXek5A9/COfzxo//TgMurhqffUbpkCGw/fYwYYLfo5uH/F/MZa+wkMrDD4cpU1j++OPhXtPTT4cf/CC0JbBhQ9IJc9O6dXDccbBmDTz+eDj/7PKOF1HXoNYfcAC89BL8/e+hdalf/zq0gvXgg15M0517Lkyfzqqbb4add046jdtMsRZRSYdJmidpvqSLqpkuSaOj6XMk7RmNbyPpNUlvSnpH0hVx5nQNTAqPj77+Ojz8MBQUhCv8vXvDQw+F5vuauVbPPAPjxsGFF1J55JFJx3FZiK2ISioAbgH6AzsDJ0lK/3PbH+gVdUOB26Lx64BDzGw3YHfgMEn+yEy+adEiHK7OmRP2RM3ghBNg773hhReSTpccM9pef3245/ZPf0o6jctSnHuiewPzzWyBmVUCDwID0uYZANxtwXSgg6Sto+FV0Twto85izOri1KJFKJ5vvQV33x2eeurbNzxO+v77SadrfFOm0PL118N9oYX+5HW+i7OIdgM+TRleGI3LaB5JBZLeAL4CnjOzV2PM6hpDQUE4R/qvf4XHGv/v/2DnnSn+/e9h6dKk0zWeP/+ZDVtuCSefnHQS1wDiLKLVNT+TvjdZ4zxmtsHMdge6A3tL2qXalUhDJc2UNLO8vDyrwK6RtG0bGoyePx9OPZU2f/1raI7vllua/g37r74KU6bw9ZlneuPYTUScRXQhsE3KcHfg8/rOY2bLgKnAYdWtxMzGm1kfM+vTxRsXzi9bbgnjxrFsypTQ9ulZZ4Wfzz6bdLL4/OUv0LEjXw8alHQS10DiLKIzgF6SekpqBZwITEqbZxIwKLpKvy+w3MwWSeoiqQOApLbAocB7MWZ1Cdrwwx/CP/4Bjz0Wnnb62c9g993hhhuaVqtRb78NEyfCOedgJSV1z+/yQmxF1MyqgLOAZ4B3gYfM7B1JwyQNi2abDCwA5gO3A2dG47cGnpc0h1CMnzOzJ+PK6nKAFC40zZ0Lt94aXmMyYgR060bpwIHwxBP5f6h/9dXhDbJnn510EteAYr00aGaTCYUyddzYlH4DvvOwsJnNAbxF2uaodWs444zQzZ0LEyZQcPfdocBuu20YP2RI0inrrcVHH4XbvM47L7xh1t8h32T4E0sud+28M1xzDUtnzw6H+jvuCBdfDN27U3L22eGWqXywZg3Fl18e7k4YMSLpNK6BeRF1ua+wMOyJ/uMf4bziqafSetKk8ATUEUeEG/ctR28jnjYNdtuN1k89BZddBl27Jp3INTAvoi6//PCHcOutLJk9O7wGesaMcOP+fvvR5vbbw5X9jz9O/tHSNWvgt7+Fgw6CqqrQMMsllySbycXCH5dweck6dYJLLw3tcN51F4waRUlKkercpg3stBP85Ce03G+/8Cx/3DZsoHDmTJg+PbRe9cEHcOaZcM01rPc2VpssL6Iuv7VtGy42DRvG4rlz6bx4Mcybx9o336TovfdgzBjaX389tG1L6f77hxal2rSh7caN0LkzrQsL4cc/DqcGWrb89mcvXgyzZsHnn4eLQZ07Q1kZ2rgRliwJ4z//HD77LMz37LN0WLo03Gmwzz6hTdVDDgmf5UW0yfIi6poGCdtyy3C4f9BBrKmooKisDFavZvmkSbR/5RVaPP00vPYarF1LcdRgdLtNy7dpA3vsAXvtBV99BTNnwoIF1a6qc3Ujt94aBgxgxY9+ROkxx4SC65oFL6KuaSsuZn2/fnDSSSyrqKCsrAyAii+/pKykhCXvvkunDz4IxfW11+COO2CLLaBPn/DWzT59oEeP8Gx/RQUsXsyqTz6hZNttw0WiTV27UI4rKyq8gDYzXkRd81RQAMXFbOzRIxTKE07IeNGvKyooiYqxc3513jnnsuBF1DnnsuBF1DnnsuBF1DnnsuBF1DnnsiDL1WeON4OkcmAZsDwa1b6a/k0/y4D6NqWT+nmZTk8fV9twekbPmnnWmvJVlzV1XEPnrWlaJtuyrty59nuQT1mry1ifrNuZWfWtvptZk+qA8bX1p/ycmc1nZzo9fVxtw9Vk9KwZZq3Pv32ceWualsm2zGAb59TvQT5lrSHjZmdN7Zri4fzf6+hPHZfNZ2c6PX1cbcPpGT1rzdPrGlfXv31ceWualsm2rKk/6W1b03A+ZU3tb4is32hSh/P1IWmmmfVJOkcmPGt88imvZ41Htlmb4p5opsYnHaAePGt88imvZ41HVlmb7Z6oc841hOa8J+qcc1nzIuqcc1nwIuqcc1nwIloNSX0l/VPSWEl9k85TF0nFkmZJaoR3YGw+ST+Itukjks5IOk9tJP1C0u2SJkr6adJ56iJpe0n/I+mRpLNUJ/odnRBt04FJ56lNfbdlkyuiku6Q9JWkt9PGHyZpnqT5ki6q42MMWAW0ARbmeFaAC4GH4kn5Taass5rZu2Y2DDgeiO32lwbK+oSZ/QYYDGTe2OhmaKC8C8zstDhzpqtn7mOAR6JtelRj5qxv1npvy2zu1M/FDjgI2BN4O2VcAfABsD3QCngT2BnYFXgyrdsCaBEttyVwX45nPRQ4kfCf/ee5nDVa5ijgZeCXuZ41Wm4UsGeu/86mLPdInFmzyH0xsHs0z/2NlXFzstZ3Wza5lu3N7EVJPdJG7w3MN7MFAJIeBAaY2V+A2g6BlwKt48gJDZNV0sFAMeEXda2kyWbW4O8LbqjtamaTgEmSngLub+icDZVVkoCrgafN7PU4cjZk3iTUJzfhiK478AYJHAHXM+vc+nx2kzucr0E34NOU4YXRuGpJOjVF06IAAAPySURBVEbSOOAeYEzM2dLVK6uZ/d7MziMUpNvjKKC1qO927StpdLRtJ8cdLk29sgJnE/byj5M0LM5gNajvtu0saSywh6SL4w5Xi5pyPwYcK+k2GuhxywZQbdb6bssmtydaA1UzrsanDMzsMcI/ehLqlfWbGczuavgodarvdp0KTI0rTB3qm3U0MDq+OHWqb97FQBLFPl21uc1sNXBKY4epQ01Z67Utm8ue6EJgm5Th7sDnCWWpi2eNRz5lhfzLu0k+5W6QrM2liM4AeknqKakV4ULMpIQz1cSzxiOfskL+5d0kn3I3TNbGvkrWCFfhHgAWAesJf2lOi8YfDvyLcDXu90nn9KyeNV/z5mPuOLN6AyTOOZeF5nI475xzsfAi6pxzWfAi6pxzWfAi6pxzWfAi6pxzWfAi6pxzWfAi6hqdpFWNvL6/Stq5kdd5nqSixlynS4bfJ+oanaRVZlbSgJ9XaGZVDfV5Ga5ThP8/1Tb4IukjoI+ZVTRmLtf4fE/U5QRJXSQ9KmlG1O0fjd9b0suSZkc/d4rGD5b0sKS/A89GLURNVWg1/z1J90WFjmh8n6h/laSrJL0pabqkLaPxO0TDMySNrG5vWVIPSe9KuhV4HdhG0m2SZkp6R9IV0XznAF2B5yU9H437qaRXJL0e5W6wPyIuYUk/juVd8+uAVdWMux84IOrfFng36i8FCqP+Q4FHo/7BhMf3OkXDfYHlhEYkWgCvpHzeVMJeIYSWkI6M+q8FLo36nwROivqH1ZCxB7AR2Ddl3Kb1F0Tr6R0NfwSURf1lwItAcTR8IfDHpP8dvGuYrrk0hedy36HAztHOI0CppHZAe2CCpF6EAtgyZZnnzGxJyvBrZrYQQNIbhKI3LW09lYSCCTAL6Bf17wf8Iuq/H7iuhpwfm9n0lOHjJQ0lNCu5NaFx7Dlpy+wbjX8p+n6tCEXeNQFeRF2uaAHsZ2ZrU0dKuhl43syOjlomn5oyeXXaZ6xL6d9A9b/f6y3aHaxlntp8s05JPYELgP8ws6WS7iK8lyudCAX/pHquy+UBPyfqcsWzwFmbBiTtHvW2Bz6L+gfHuP7pwLFR/4kZLlNKKKrLo3Or/VOmrQTapXz2/pJ2BJBUJOl72Ud2ucCLqEtCkaSFKd0I4Bygj6Q5kuby75bFrwX+IuklwnnHuJwHjJD0GuGwfHldC5jZm8Bs4B3gDuCllMnjgaclPW9m5YQ/AA9ImkMoqt9v2PguKX6Lk3OEvUNgrZmZpBMJF5kGJJ3L5T4/J+pcsBcwJrotahlwasJ5XJ7wPVHnnMuCnxN1zrkseBF1zrkseBF1zrkseBF1zrkseBF1zrkseBF1zrks/D/elrsZSKxSQgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "multisurv.plot_lr_range(trim=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run tag: \"clinical_mRNA_lr0.005_\"\n"
     ]
    }
   ],
   "source": [
    "picked_lr = 5e-3\n",
    "\n",
    "run_tag = utils.compose_run_tag(model=multisurv, lr=picked_lr,\n",
    "                                dataloaders=dataloaders,\n",
    "                                log_dir='.training_logs/',\n",
    "                                suffix='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Instantiating MultiSurv model...\n",
      "\n",
      "------------------------------------------\n",
      "             Training        Validation\n",
      "           ------------     ------------\n",
      " Epoch     Loss     Ctd     Loss     Ctd\n",
      "------------------------------------------\n",
      " 1/75     0.0185   0.631   0.0175   0.747\n",
      " 5/75     0.0150   0.752   0.0168   0.765\n",
      " 10/75    0.0145   0.776   0.0163   0.784\n",
      " 15/75    0.0143   0.779   0.0159   0.788\n",
      " 20/75    0.0140   0.790   0.0159   0.792\n",
      " 25/75    0.0139   0.793   0.0159   0.792\n",
      "Epoch    27: reducing learning rate of group 0 to 2.5000e-03.\n",
      " 30/75    0.0136   0.804   0.0157   0.795\n",
      "\n",
      ">>> Keyboard interrupt! <<<\n",
      "(trained for 0h 32m 19s)\n",
      "\n",
      "Best validation concordance values:\n",
      "     0.7935 (epoch23)\n",
      "     0.7942 (epoch24)\n",
      "     0.795 (epoch30)\n"
     ]
    }
   ],
   "source": [
    "fit_args = {\n",
    "    'lr': picked_lr,\n",
    "    'num_epochs': 75,\n",
    "    'info_freq': 5,\n",
    "#     'info_freq': None,\n",
    "#     'lr_factor': 0.25,\n",
    "#     'scheduler_patience': 5,\n",
    "    'lr_factor': 0.5,\n",
    "    'scheduler_patience': 10,\n",
    "    'log_dir': os.path.join('.training_logs/', run_tag),\n",
    "}\n",
    "\n",
    "multisurv.fit(**fit_args)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save model weights\n",
    "\n",
    "If desired."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['epoch23', 'epoch24', 'epoch30'])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multisurv.best_model_weights.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'epoch23': 0.7934656932377523,\n",
       " 'epoch24': 0.7941594265138251,\n",
       " 'epoch30': 0.7949994572834915}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multisurv.best_concord_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'epoch30': 0.7949994572834915}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multisurv.current_concord"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving model weights to file:\n",
      "    /mnt/dataA/multisurv_models/clinical_DNAm_lr0.005_20_cancers_epoch39_concord0.79.pth\n"
     ]
    }
   ],
   "source": [
    "multisurv.save_weights(saved_epoch='epoch39', prefix=run_tag, weight_dir=MODELS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check validation metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data modalities:\n",
      "   clinical\n",
      "   mRNA\n",
      "\n",
      "Dataset sizes (# patients):\n",
      "   train: 8880\n",
      "   val: 1109\n",
      "   test: 1092\n",
      "\n",
      "Batch size: 128\n"
     ]
    }
   ],
   "source": [
    "dataloaders = utils.get_dataloaders(data_location=DATA,\n",
    "                                    labels_file='data/labels.tsv',\n",
    "                                    modalities=data_modalities.value,\n",
    "                                    wsi_patch_size=299,\n",
    "                                    n_wsi_patches=5,\n",
    "#                                     exclude_patients=exclude_cancers,\n",
    "                                    return_patient_id=True,\n",
    "                                   )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collect patient predictions: 1092/1092\n",
      "\n",
      "C-index   0.808\n",
      "Ctd       0.811\n",
      "IBS       0.143\n",
      "INBLL     0.436\n"
     ]
    }
   ],
   "source": [
    "performance = utils.Evaluation(\n",
    "    model=multisurv, dataset=dataloaders['val'].dataset,\n",
    "    device=device)\n",
    "performance.compute_metrics()\n",
    "performance.show_results()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Watermark <a class='tocSkip'></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch      1.4.0\n",
      "ipywidgets 7.5.1\n",
      "pandas     1.0.1\n",
      "\n",
      "CPython 3.6.7\n",
      "IPython 7.11.1\n",
      "\n",
      "last updated: Tue Jul 28 2020\n"
     ]
    }
   ],
   "source": [
    "%watermark --iversions\n",
    "%watermark -v\n",
    "print()\n",
    "%watermark -u -n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Top of the page](#Top)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {
    "height": "197px",
    "width": "372px"
   },
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "236px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}