NCDB_G2_Glioma_36M_Survival.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 23042,
     "status": "ok",
     "timestamp": 1691626452315,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "RxtJ9G-aIZ_y",
    "outputId": "94d4dab3-24d3-4c62-f390-b38a3068a8a4"
   },
   "outputs": [],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 20960,
     "status": "ok",
     "timestamp": 1691626473260,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "h_PnaQ4NIuE0",
    "outputId": "d500b152-676b-4b63-98f1-6e5bef43afbc"
   },
   "outputs": [],
   "source": [
    "!pip install optuna\n",
    "!pip install shap\n",
    "!pip install pytorch-tabnet\n",
    "!pip install tabpfn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "JrZMQLTfIQPb"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import sklearn\n",
    "import sklearn.metrics\n",
    "from sklearn.metrics import roc_auc_score, roc_curve, precision_recall_curve, average_precision_score, auc, precision_score, recall_score, accuracy_score, balanced_accuracy_score, brier_score_loss, confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.utils import resample\n",
    "from math import sqrt\n",
    "from scipy import stats as st\n",
    "from random import randrange\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "\n",
    "from tabpfn import TabPFNClassifier\n",
    "from pytorch_tabnet.tab_model import TabNetClassifier\n",
    "import xgboost as xgb\n",
    "import lightgbm as lgb\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "import optuna\n",
    "from optuna.samplers import TPESampler\n",
    "\n",
    "import shap\n",
    "from sklearn.inspection import PartialDependenceDisplay"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UWbpPFMVHS-V"
   },
   "source": [
    "# Preparing Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "sDsi_6tnIQPc"
   },
   "outputs": [],
   "source": [
    "#Open csv file.\n",
    "\n",
    "data = pd.read_csv('/content/drive/MyDrive/NCDB-G2G3 Glioma/final_data_g2.csv', index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 267,
     "status": "ok",
     "timestamp": 1691626485898,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "hPmUaNyUN2p8",
    "outputId": "312e857b-4fb7-445c-fa29-ae7d8033c6b1"
   },
   "outputs": [],
   "source": [
    "#Define outcome of interest (3-month mortality).\n",
    "\n",
    "data.loc[(data['Vital Status'] == 'Dead') & (data['Last Contact or Death (Months from Diagnosis)'] < 36), 'OUTCOME'] = 1\n",
    "data.loc[(data['Vital Status'] == 'Dead') & (data['Last Contact or Death (Months from Diagnosis)'] >= 36), 'OUTCOME'] = 0\n",
    "data.loc[(data['Vital Status'] == 'Alive') & (data['Last Contact or Death (Months from Diagnosis)'] >= 36), 'OUTCOME'] = 0\n",
    "data = data.dropna(subset=['OUTCOME'])\n",
    "\n",
    "data['OUTCOME'].value_counts(normalize=False, dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "LhZeK6d8IQPd"
   },
   "outputs": [],
   "source": [
    "#Define predictor variables (x) and outcome of interest (y).\n",
    "\n",
    "outcomes = ['Vital Status', 'Last Contact or Death (Months from Diagnosis)', 'OUTCOME']\n",
    "\n",
    "x = data.drop(outcomes, axis = 1)\n",
    "y = data['OUTCOME']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 37,
     "status": "ok",
     "timestamp": 1691626485898,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "mt6ks1daIQPd",
    "outputId": "3b40aceb-f20a-46d2-cf92-47d78c84b84d"
   },
   "outputs": [],
   "source": [
    "#Check data shapes.\n",
    "\n",
    "print(y.shape)\n",
    "print(x.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 32,
     "status": "ok",
     "timestamp": 1691626485898,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "d0IwUyCsIQPe",
    "outputId": "5df29a5a-0170-4e71-dfaf-816b3f66bd45"
   },
   "outputs": [],
   "source": [
    "#Split data into initial train set and test set in 80:20 ratio.\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.2, random_state = 0)\n",
    "\n",
    "#Describe initial train set and test set.\n",
    "\n",
    "print(\"Number patients x_train dataset: \", x_train.shape[0])\n",
    "print(\"Number patients y_train dataset: \", y_train.shape[0])\n",
    "print(\"Number patients x_test dataset: \", x_test.shape[0])\n",
    "print(\"Number patients y_test dataset: \", y_test.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 28,
     "status": "ok",
     "timestamp": 1691626485899,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "WyNo7C2Pbzt2",
    "outputId": "bd00b663-bce9-4e96-c880-09faba6566e1"
   },
   "outputs": [],
   "source": [
    "#Split initial train set into final train set and validation set in 75:25 ratio.\n",
    "\n",
    "train_x, valid_x, train_y, valid_y = train_test_split(x_train, y_train, test_size = 0.25, random_state = 0)\n",
    "\n",
    "#Describe train and validation sets.\n",
    "\n",
    "print(\"Number patients train_x dataset: \", train_x.shape[0])\n",
    "print(\"Number patients train_y dataset: \", train_y.shape[0])\n",
    "print(\"Number patients valid_x dataset: \", valid_x.shape[0])\n",
    "print(\"Number patients valid_y dataset: \", valid_y.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 23,
     "status": "ok",
     "timestamp": 1691626485899,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "g2O0He35IQPe",
    "outputId": "bb75da27-58b3-4c1c-db7c-a1a051c733b1"
   },
   "outputs": [],
   "source": [
    "#Describe outcome of interest before resampling.\n",
    "\n",
    "print(\"Before resampling, counts of label '1': {}\".format(sum(train_y == 1)))\n",
    "print(\"Before resampling, counts of label '0': {} \\n\".format(sum(train_y == 0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "QvXczViaIQPe"
   },
   "outputs": [],
   "source": [
    "#Apply SMOTE.\n",
    "\n",
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "resampler = SMOTE(random_state = 0)\n",
    "train_x, train_y = resampler.fit_resample(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 16,
     "status": "ok",
     "timestamp": 1691626485899,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "mYw68tzFIQPe",
    "outputId": "43f956c2-5765-451c-b030-d5910db47f93"
   },
   "outputs": [],
   "source": [
    "#Describe outcome of interest after resampling.\n",
    "\n",
    "print(\"After resampling, counts of label '1': {}\".format(sum(train_y == 1)))\n",
    "print(\"After resampling, counts of label '0': {} \\n\".format(sum(train_y == 0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lrb8PN0dEOc1"
   },
   "outputs": [],
   "source": [
    "#Define function for AUROC with 95% confidence intervals.\n",
    "\n",
    "def auroc_ci(y_test, y_probs, positive=1, n_samples=1000, alpha=0.05):\n",
    "    auroc_values = []\n",
    "    for i in range(n_samples):\n",
    "        y_test_bs, y_probs_bs = resample(y_test, y_probs)\n",
    "\n",
    "        auroc = roc_auc_score(y_test_bs, y_probs_bs)\n",
    "        auroc_values.append(auroc)\n",
    "\n",
    "    mean_auroc = np.mean(auroc_values)\n",
    "    std_auroc = np.std(auroc_values)\n",
    "\n",
    "    lower_ci = mean_auroc - 1.96 * std_auroc\n",
    "    upper_ci = mean_auroc + 1.96 * std_auroc\n",
    "\n",
    "    return auroc, lower_ci, upper_ci"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "BLkyBrpf95-0"
   },
   "outputs": [],
   "source": [
    "#Define z-value for other confidence intervals.\n",
    "\n",
    "confidence = 0.95\n",
    "z_value = st.norm.ppf((1 + confidence) / 2.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UWX_CriXYlL_"
   },
   "source": [
    "#TabPFN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "OFQmwskRLrY4"
   },
   "source": [
    "No hyperparameter tuning is performed for TabPFN since the paper that introduced TabPFN claims no hyperparameter tuning is needed for it:\n",
    "\n",
    "\n",
    "*We present TabPFN, a trained Transformer that can do supervised classification for small tabular datasets in less than a second, needs no hyperparameter tuning and is competitive with state-of-the-art classification methods.*\n",
    "\n",
    "https://doi.org/10.48550/arXiv.2207.01848\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 162
    },
    "executionInfo": {
     "elapsed": 5756,
     "status": "ok",
     "timestamp": 1691626491980,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "F3gvvODJ8B2Z",
    "outputId": "18b81b1d-d9f5-4020-b1a9-aa5e8db0de98"
   },
   "outputs": [],
   "source": [
    "#Fit TabPFN.\n",
    "\n",
    "tabpfn = TabPFNClassifier(device='cpu', N_ensemble_configurations=2)\n",
    "\n",
    "tabpfn.fit(x_train.sample(n=1024, random_state=42), y_train.sample(n=1024, random_state=42), overwrite_warning = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 74
    },
    "executionInfo": {
     "elapsed": 8168,
     "status": "ok",
     "timestamp": 1691626500036,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "5oLbONin8YGA",
    "outputId": "957c473e-bfe6-4c82-d3d5-afb603df752b"
   },
   "outputs": [],
   "source": [
    "#Predict on the validation set, get predicted probabilities for calibration, and fit calibration function.\n",
    "\n",
    "calib_probs_tabpfn = tabpfn.predict_proba(valid_x.values)\n",
    "\n",
    "calib_model_tabpfn = LogisticRegression()\n",
    "calib_model_tabpfn.fit(calib_probs_tabpfn, valid_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 17127,
     "status": "ok",
     "timestamp": 1691626517160,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "UZNInXD39Tux",
    "outputId": "6d95e47d-06ef-44cb-ac4e-b4cbac0403b0"
   },
   "outputs": [],
   "source": [
    "#Make predictions on the test set based on the trained TabNet model.\n",
    "\n",
    "preds_tabpfn = tabpfn.predict(x_test.values)\n",
    "\n",
    "uncalibrated_probs_tabpfn = tabpfn.predict_proba(x_test.values)\n",
    "\n",
    "probs_tabpfn = calib_model_tabpfn.predict_proba(uncalibrated_probs_tabpfn)\n",
    "probs_tabpfn = probs_tabpfn[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2709,
     "status": "ok",
     "timestamp": 1691627521164,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "Sm8PNeSm9dtI",
    "outputId": "b60c596c-095d-441d-a95b-d1f2a09dbfd8"
   },
   "outputs": [],
   "source": [
    "#Evaluate TabPFN model.\n",
    "\n",
    "tabpfn_tn, tabpfn_fp, tabpfn_fn, tabpfn_tp = confusion_matrix(y_test, preds_tabpfn).ravel()\n",
    "tabpfn_cm = confusion_matrix(y_test, preds_tabpfn)\n",
    "\n",
    "tabpfn_sensitivity = round(tabpfn_tp/(tabpfn_tp+tabpfn_fn), 3)\n",
    "tabpfn_sensitivity_ci_length = z_value * np.sqrt((tabpfn_sensitivity * (1 - tabpfn_sensitivity)) / y_test.shape[0])\n",
    "tabpfn_sensitivity_ci_lower = round((tabpfn_sensitivity - tabpfn_sensitivity_ci_length), 3)\n",
    "tabpfn_sensitivity_ci_upper = round((tabpfn_sensitivity + tabpfn_sensitivity_ci_length), 3)\n",
    "tabpfn_sensitivity_str = str(tabpfn_sensitivity) + ' (' + str(tabpfn_sensitivity_ci_lower) + ' - ' + str(tabpfn_sensitivity_ci_upper) + ')'\n",
    "\n",
    "tabpfn_specificity = round(tabpfn_tn/(tabpfn_tn+tabpfn_fp), 3)\n",
    "tabpfn_specificity_ci_length = z_value * np.sqrt((tabpfn_specificity * (1 - tabpfn_specificity)) / y_test.shape[0])\n",
    "tabpfn_specificity_ci_lower = round((tabpfn_specificity - tabpfn_specificity_ci_length), 3)\n",
    "tabpfn_specificity_ci_upper = round((tabpfn_specificity + tabpfn_specificity_ci_length), 3)\n",
    "tabpfn_specificity_str = str(tabpfn_specificity) + ' (' + str(tabpfn_specificity_ci_lower) + ' - ' + str(tabpfn_specificity_ci_upper) + ')'\n",
    "\n",
    "tabpfn_auprc = round(average_precision_score(y_test, probs_tabpfn), 3)\n",
    "tabpfn_auprc_ci_length = z_value * np.sqrt((tabpfn_auprc * (1 - tabpfn_auprc)) / y_test.shape[0])\n",
    "tabpfn_auprc_ci_lower = round((tabpfn_auprc - tabpfn_auprc_ci_length), 3)\n",
    "tabpfn_auprc_ci_upper = round((tabpfn_auprc + tabpfn_auprc_ci_length), 3)\n",
    "tabpfn_auprc_str = str(tabpfn_auprc) + ' (' + str(tabpfn_auprc_ci_lower) + ' - ' + str(tabpfn_auprc_ci_upper) + ')'\n",
    "\n",
    "tabpfn_accuracy = round(accuracy_score(y_test, preds_tabpfn), 3)\n",
    "tabpfn_accuracy_ci_length = z_value * np.sqrt((tabpfn_accuracy * (1 - tabpfn_accuracy)) / y_test.shape[0])\n",
    "tabpfn_accuracy_ci_lower = round((tabpfn_accuracy - tabpfn_accuracy_ci_length), 3)\n",
    "tabpfn_accuracy_ci_upper = round((tabpfn_accuracy + tabpfn_accuracy_ci_length), 3)\n",
    "tabpfn_accuracy_str = str(tabpfn_accuracy) + ' (' + str(tabpfn_accuracy_ci_lower) + ' - ' + str(tabpfn_accuracy_ci_upper) + ')'\n",
    "\n",
    "tabpfn_auroc, tabpfn_auroc_ci_lower, tabpfn_auroc_ci_upper = auroc_ci(y_test, probs_tabpfn)\n",
    "tabpfn_auroc = round(tabpfn_auroc, 3)\n",
    "tabpfn_auroc_ci_lower = round(tabpfn_auroc_ci_lower, 3)\n",
    "tabpfn_auroc_ci_upper = round(tabpfn_auroc_ci_upper, 3)\n",
    "tabpfn_auroc_str = str(tabpfn_auroc) + ' (' + str(tabpfn_auroc_ci_lower) + ' - ' + str(tabpfn_auroc_ci_upper) + ')'\n",
    "\n",
    "tabpfn_brier = round(brier_score_loss(y_test, probs_tabpfn), 3)\n",
    "tabpfn_brier_ci_length = z_value * np.sqrt((tabpfn_brier * (1 - tabpfn_brier)) / y_test.shape[0])\n",
    "tabpfn_brier_ci_lower = round((tabpfn_brier - tabpfn_brier_ci_length), 3)\n",
    "tabpfn_brier_ci_upper = round((tabpfn_brier + tabpfn_brier_ci_length), 3)\n",
    "tabpfn_brier_str = str(tabpfn_brier) + ' (' + str(tabpfn_brier_ci_lower) + ' - ' + str(tabpfn_brier_ci_upper) + ')'\n",
    "\n",
    "tabpfn_results = [tabpfn_sensitivity_str, tabpfn_specificity_str, tabpfn_auprc_str, tabpfn_accuracy_str, tabpfn_auroc_str, tabpfn_brier_str]\n",
    "\n",
    "print(\"Sensitivity: \", (tabpfn_sensitivity_str))\n",
    "print(\"Specificity: \", (tabpfn_specificity_str))\n",
    "print('AUPRC: ', (tabpfn_auprc_str))\n",
    "print('Accuracy: ', (tabpfn_accuracy_str))\n",
    "print('AUROC: ', (tabpfn_auroc_str))\n",
    "print('Brier Score: ', (tabpfn_brier_str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "0wp5veHVFWIB"
   },
   "outputs": [],
   "source": [
    "#Recalculate precision recall curve for plotting purposes.\n",
    "\n",
    "tabpfn_precision_curve, tabpfn_recall_curve, _ = precision_recall_curve(y_test, probs_tabpfn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BY44IPIU7sBi"
   },
   "source": [
    "# TabNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 151868,
     "status": "ok",
     "timestamp": 1691626671777,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "beooviDI7uuS",
    "outputId": "4a5bac09-48d5-4de3-b6ad-d29aeeb7262c"
   },
   "outputs": [],
   "source": [
    "# Hyperparameter tuning for TabNet.\n",
    "\n",
    "def objective(trial):\n",
    "\n",
    "    param = {\n",
    "        \"n_d\": trial.suggest_int(\"n_d\", 8, 64),\n",
    "        \"n_a\": trial.suggest_int(\"n_a\", 8, 64),\n",
    "        \"n_steps\": trial.suggest_int(\"n_steps\", 1, 10),\n",
    "        \"gamma\": trial.suggest_float(\"gamma\", 1.0, 2.0),\n",
    "        \"n_independent\": trial.suggest_int(\"n_independent\", 1, 5),\n",
    "        \"n_shared\": trial.suggest_int(\"n_shared\", 1, 5),\n",
    "        \"lambda_sparse\": trial.suggest_loguniform(\"lambda_sparse\", 1e-6, 1e-3),\n",
    "        \"optimizer_params\": {\"lr\": trial.suggest_loguniform(\"lr\", 1e-5, 1e-2)},\n",
    "        \"seed\": 31,\n",
    "    }\n",
    "\n",
    "    tabnet_clf = TabNetClassifier(**param)\n",
    "\n",
    "    tabnet_clf.fit(\n",
    "        train_x.values,\n",
    "        train_y.values,\n",
    "        eval_set=[(valid_x.values, valid_y.values)],\n",
    "        eval_metric=[\"auc\"],\n",
    "        max_epochs=10,\n",
    "        patience=20,\n",
    "        batch_size=1024,\n",
    "        virtual_batch_size=128,\n",
    "        num_workers=0,\n",
    "        drop_last=False,\n",
    "    )\n",
    "\n",
    "    preds = tabnet_clf.predict_proba(valid_x.values)[:, 1]\n",
    "    pred_labels = np.rint(preds)\n",
    "    auc = sklearn.metrics.roc_auc_score(valid_y, pred_labels)\n",
    "\n",
    "    return auc\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    study = optuna.create_study(direction='maximize', sampler=TPESampler(seed=31))\n",
    "    study.optimize(objective, n_trials=10, timeout=600)\n",
    "\n",
    "    print(\"Number of finished trials: {}\".format(len(study.trials)))\n",
    "\n",
    "    print(\"Best trial:\")\n",
    "    trial = study.best_trial\n",
    "\n",
    "    print(\"  Value: {}\".format(trial.value))\n",
    "\n",
    "    print(\"  Params: \")\n",
    "    for key, value in trial.params.items():\n",
    "        print(\"    {}: {}\".format(key, value))\n",
    "\n",
    "    tabnet_params = {}\n",
    "\n",
    "    for key, value in trial.params.items():\n",
    "        tabnet_params[key] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 26,
     "status": "ok",
     "timestamp": 1691626671778,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "gXsTkX3n_i2y",
    "outputId": "f56a2ca9-fc08-4272-cbc3-2c829617a5b8"
   },
   "outputs": [],
   "source": [
    "#See TabNet hyperparameters.\n",
    "\n",
    "tabnet_params.pop('lr')\n",
    "\n",
    "print(tabnet_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 107845,
     "status": "ok",
     "timestamp": 1691626779615,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "pujw5iMQ_i2y",
    "outputId": "064af0a0-083e-4466-c56d-c2ddecab4266"
   },
   "outputs": [],
   "source": [
    "#Fit TabNet.\n",
    "\n",
    "tabnet = TabNetClassifier(**tabnet_params)\n",
    "\n",
    "tabnet.fit(train_x.values, train_y.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 111
    },
    "executionInfo": {
     "elapsed": 114,
     "status": "ok",
     "timestamp": 1691626779615,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "xYiPND2v_i2y",
    "outputId": "610828c1-8e32-4381-b8c8-e28bbbd2d0a3"
   },
   "outputs": [],
   "source": [
    "#Predict on the validation set, get predicted probabilities for calibration, and fit calibration function.\n",
    "\n",
    "calib_probs_tabnet = tabnet.predict_proba(valid_x.values)\n",
    "\n",
    "calib_model_tabnet = LogisticRegression()\n",
    "calib_model_tabnet.fit(calib_probs_tabnet, valid_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 294,
     "status": "ok",
     "timestamp": 1691626779809,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "y1AJq9eA_i2y",
    "outputId": "f215482e-1f3c-440e-c0fb-11d1c3b2ebb9"
   },
   "outputs": [],
   "source": [
    "#Make predictions on the test set based on the trained TabNet model.\n",
    "\n",
    "preds_tabnet = tabnet.predict(x_test.values)\n",
    "\n",
    "uncalibrated_probs_tabnet = tabnet.predict_proba(x_test.values)\n",
    "\n",
    "probs_tabnet = calib_model_tabnet.predict_proba(uncalibrated_probs_tabnet)\n",
    "probs_tabnet = probs_tabnet[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2200,
     "status": "ok",
     "timestamp": 1691626782001,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "AP-y4lEI_i2z",
    "outputId": "e5d52d1f-1dbd-417a-9bae-f06c6f92be78"
   },
   "outputs": [],
   "source": [
    "#Evaluate TabNet model.\n",
    "\n",
    "tabnet_tn, tabnet_fp, tabnet_fn, tabnet_tp = confusion_matrix(y_test, preds_tabnet).ravel()\n",
    "tabnet_cm = confusion_matrix(y_test, preds_tabnet)\n",
    "\n",
    "tabnet_sensitivity = round(tabnet_tp/(tabnet_tp+tabnet_fn), 3)\n",
    "tabnet_sensitivity_ci_length = z_value * np.sqrt((tabnet_sensitivity * (1 - tabnet_sensitivity)) / y_test.shape[0])\n",
    "tabnet_sensitivity_ci_lower = round((tabnet_sensitivity - tabnet_sensitivity_ci_length), 3)\n",
    "tabnet_sensitivity_ci_upper = round((tabnet_sensitivity + tabnet_sensitivity_ci_length), 3)\n",
    "tabnet_sensitivity_str = str(tabnet_sensitivity) + ' (' + str(tabnet_sensitivity_ci_lower) + ' - ' + str(tabnet_sensitivity_ci_upper) + ')'\n",
    "\n",
    "tabnet_specificity = round(tabnet_tn/(tabnet_tn+tabnet_fp), 3)\n",
    "tabnet_specificity_ci_length = z_value * np.sqrt((tabnet_specificity * (1 - tabnet_specificity)) / y_test.shape[0])\n",
    "tabnet_specificity_ci_lower = round((tabnet_specificity - tabnet_specificity_ci_length), 3)\n",
    "tabnet_specificity_ci_upper = round((tabnet_specificity + tabnet_specificity_ci_length), 3)\n",
    "tabnet_specificity_str = str(tabnet_specificity) + ' (' + str(tabnet_specificity_ci_lower) + ' - ' + str(tabnet_specificity_ci_upper) + ')'\n",
    "\n",
    "tabnet_auprc = round(average_precision_score(y_test, probs_tabnet), 3)\n",
    "tabnet_auprc_ci_length = z_value * np.sqrt((tabnet_auprc * (1 - tabnet_auprc)) / y_test.shape[0])\n",
    "tabnet_auprc_ci_lower = round((tabnet_auprc - tabnet_auprc_ci_length), 3)\n",
    "tabnet_auprc_ci_upper = round((tabnet_auprc + tabnet_auprc_ci_length), 3)\n",
    "tabnet_auprc_str = str(tabnet_auprc) + ' (' + str(tabnet_auprc_ci_lower) + ' - ' + str(tabnet_auprc_ci_upper) + ')'\n",
    "\n",
    "tabnet_accuracy = round(accuracy_score(y_test, preds_tabnet), 3)\n",
    "tabnet_accuracy_ci_length = z_value * np.sqrt((tabnet_accuracy * (1 - tabnet_accuracy)) / y_test.shape[0])\n",
    "tabnet_accuracy_ci_lower = round((tabnet_accuracy - tabnet_accuracy_ci_length), 3)\n",
    "tabnet_accuracy_ci_upper = round((tabnet_accuracy + tabnet_accuracy_ci_length), 3)\n",
    "tabnet_accuracy_str = str(tabnet_accuracy) + ' (' + str(tabnet_accuracy_ci_lower) + ' - ' + str(tabnet_accuracy_ci_upper) + ')'\n",
    "\n",
    "tabnet_auroc, tabnet_auroc_ci_lower, tabnet_auroc_ci_upper = auroc_ci(y_test, probs_tabnet)\n",
    "tabnet_auroc = round(tabnet_auroc, 3)\n",
    "tabnet_auroc_ci_lower = round(tabnet_auroc_ci_lower, 3)\n",
    "tabnet_auroc_ci_upper = round(tabnet_auroc_ci_upper, 3)\n",
    "tabnet_auroc_str = str(tabnet_auroc) + ' (' + str(tabnet_auroc_ci_lower) + ' - ' + str(tabnet_auroc_ci_upper) + ')'\n",
    "\n",
    "tabnet_brier = round(brier_score_loss(y_test, probs_tabnet), 3)\n",
    "tabnet_brier_ci_length = z_value * np.sqrt((tabnet_brier * (1 - tabnet_brier)) / y_test.shape[0])\n",
    "tabnet_brier_ci_lower = round((tabnet_brier - tabnet_brier_ci_length), 3)\n",
    "tabnet_brier_ci_upper = round((tabnet_brier + tabnet_brier_ci_length), 3)\n",
    "tabnet_brier_str = str(tabnet_brier) + ' (' + str(tabnet_brier_ci_lower) + ' - ' + str(tabnet_brier_ci_upper) + ')'\n",
    "\n",
    "tabnet_results = [tabnet_sensitivity_str, tabnet_specificity_str, tabnet_auprc_str, tabnet_accuracy_str, tabnet_auroc_str, tabnet_brier_str]\n",
    "\n",
    "print(\"Sensitivity: \", (tabnet_sensitivity_str))\n",
    "print(\"Specificity: \", (tabnet_specificity_str))\n",
    "print('AUPRC: ', (tabnet_auprc_str))\n",
    "print('Accuracy: ', (tabnet_accuracy_str))\n",
    "print('AUROC: ', (tabnet_auroc_str))\n",
    "print('Brier Score: ', (tabnet_brier_str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 16,
     "status": "ok",
     "timestamp": 1691626782001,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "tsAsKQ1e_i2z",
    "outputId": "54ffa546-0071-4a35-8345-b49a1c454215"
   },
   "outputs": [],
   "source": [
    "#Recalculate precision recall curve for plotting purposes.\n",
    "\n",
    "tabnet_precision_curve, tabnet_recall_curve, _ = precision_recall_curve(y_test, probs_tabnet)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "esuV5nR8IQPe"
   },
   "source": [
    "# XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 11087,
     "status": "ok",
     "timestamp": 1691626793074,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "dBWNs_MxIQPf",
    "outputId": "c373c297-2177-4e9b-ca46-292a6a1675d1"
   },
   "outputs": [],
   "source": [
    "#Hyperparameter tuning for XGBoost.\n",
    "\n",
    "def objective(trial):\n",
    "\n",
    "    dtrain = xgb.DMatrix(train_x, label=train_y)\n",
    "    dvalid = xgb.DMatrix(valid_x, label=valid_y)\n",
    "\n",
    "    param = {\n",
    "        \"seed\": 31,\n",
    "        \"verbosity\": 0,\n",
    "        \"objective\":  trial.suggest_categorical(\"objective\", [\"binary:logistic\"]),\n",
    "        \"eval_metric\": \"auc\",\n",
    "        \"booster\": trial.suggest_categorical(\"booster\", [\"gbtree\"]),\n",
    "        \"lambda\": trial.suggest_float(\"lambda\", 1e-8, 1.0, log=True),\n",
    "        \"alpha\": trial.suggest_float(\"alpha\", 1e-8, 1.0, log=True),\n",
    "        \"max_depth\" : trial.suggest_int(\"max_depth\", 1, 9),\n",
    "        \"eta\" : trial.suggest_float(\"eta\", 1e-8, 1.0, log=True),\n",
    "        \"gamma\" : trial.suggest_float(\"gamma\", 1e-8, 1.0, log=True),\n",
    "        \"grow_policy\" : trial.suggest_categorical(\"grow_policy\", [\"depthwise\", \"lossguide\"])\n",
    "    }\n",
    "\n",
    "\n",
    "    # Add a callback for pruning.\n",
    "    pruning_callback = optuna.integration.XGBoostPruningCallback(trial, \"validation-auc\")\n",
    "\n",
    "    bst = xgb.train(param, dtrain, evals=[(dvalid, \"validation\")], callbacks=[pruning_callback])\n",
    "    preds = bst.predict(dvalid)\n",
    "    pred_labels = np.rint(preds)\n",
    "    auc = sklearn.metrics.roc_auc_score(valid_y, pred_labels)\n",
    "\n",
    "    return auc\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    study = optuna.create_study(\n",
    "        pruner=optuna.pruners.MedianPruner(n_warmup_steps=5), direction=\"maximize\", sampler = TPESampler(seed=31)\n",
    "    )\n",
    "    study.optimize(objective, n_trials=100)\n",
    "    print(\"Number of finished trials: {}\".format(len(study.trials)))\n",
    "\n",
    "    print(\"Best trial:\")\n",
    "    trial = study.best_trial\n",
    "\n",
    "    print(\"  Value: {}\".format(trial.value))\n",
    "\n",
    "    print(\"  Params: \")\n",
    "    for key, value in trial.params.items():\n",
    "        print(\"    {}: {}\".format(key, value))\n",
    "\n",
    "    xgb_params = {}\n",
    "\n",
    "    for key, value in trial.params.items():\n",
    "        xgb_params[key] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 40,
     "status": "ok",
     "timestamp": 1691626793075,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "algcmJCqkAya",
    "outputId": "7f5b1cf0-8315-47a0-a180-4e2663650cf8"
   },
   "outputs": [],
   "source": [
    "#See XGBoost hyperparameters.\n",
    "\n",
    "xgb_params['eval_metric'] = 'auc'\n",
    "xgb_params['verbosity'] = 0\n",
    "xgb_params['seed'] = 31\n",
    "\n",
    "print(xgb_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 244
    },
    "executionInfo": {
     "elapsed": 453,
     "status": "ok",
     "timestamp": 1691626793520,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "uZk8hbTUIQPg",
    "outputId": "5aee9420-8146-4a66-b6b3-11bb455e9f33"
   },
   "outputs": [],
   "source": [
    "#Fit XGBoost.\n",
    "\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "xgb = XGBClassifier(**xgb_params)\n",
    "\n",
    "xgb.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 74
    },
    "executionInfo": {
     "elapsed": 180,
     "status": "ok",
     "timestamp": 1691626793520,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "rUf-OtCoIQPg",
    "outputId": "e0610e2d-f044-46fb-dcab-f16960975855"
   },
   "outputs": [],
   "source": [
    "#Predict on the validation set, get predicted probabilities for calibration, and fit calibration function.\n",
    "\n",
    "calib_probs_xgb = xgb.predict_proba(valid_x)\n",
    "\n",
    "calib_model_xgb = LogisticRegression()\n",
    "calib_model_xgb.fit(calib_probs_xgb, valid_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 179,
     "status": "ok",
     "timestamp": 1691626793520,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "wq-OU3c29-4L",
    "outputId": "02505338-d25c-42fb-d324-ba673a4c20f0"
   },
   "outputs": [],
   "source": [
    "#Make predictions on the test set based on the trained and calibrated XGBoost model.\n",
    "\n",
    "preds_xgb = xgb.predict(x_test)\n",
    "\n",
    "uncalibrated_probs_xgb = xgb.predict_proba(x_test)\n",
    "\n",
    "probs_xgb = calib_model_xgb.predict_proba(uncalibrated_probs_xgb)\n",
    "probs_xgb = probs_xgb[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2368,
     "status": "ok",
     "timestamp": 1691626795715,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "5m3AkfQJIQPg",
    "outputId": "943759d5-2bda-48c4-ce49-322a8255432e"
   },
   "outputs": [],
   "source": [
    "#Evaluate XGBoost model.\n",
    "\n",
    "xgb_tn, xgb_fp, xgb_fn, xgb_tp = confusion_matrix(y_test, preds_xgb).ravel()\n",
    "xgb_cm = confusion_matrix(y_test, preds_xgb)\n",
    "\n",
    "xgb_sensitivity = round(xgb_tp/(xgb_tp+xgb_fn), 3)\n",
    "xgb_sensitivity_ci_length = z_value * np.sqrt((xgb_sensitivity * (1 - xgb_sensitivity)) / y_test.shape[0])\n",
    "xgb_sensitivity_ci_lower = round((xgb_sensitivity - xgb_sensitivity_ci_length), 3)\n",
    "xgb_sensitivity_ci_upper = round((xgb_sensitivity + xgb_sensitivity_ci_length), 3)\n",
    "xgb_sensitivity_str = str(xgb_sensitivity) + ' (' + str(xgb_sensitivity_ci_lower) + ' - ' + str(xgb_sensitivity_ci_upper) + ')'\n",
    "\n",
    "xgb_specificity = round(xgb_tn/(xgb_tn+xgb_fp), 3)\n",
    "xgb_specificity_ci_length = z_value * np.sqrt((xgb_specificity * (1 - xgb_specificity)) / y_test.shape[0])\n",
    "xgb_specificity_ci_lower = round((xgb_specificity - xgb_specificity_ci_length), 3)\n",
    "xgb_specificity_ci_upper = round((xgb_specificity + xgb_specificity_ci_length), 3)\n",
    "xgb_specificity_str = str(xgb_specificity) + ' (' + str(xgb_specificity_ci_lower) + ' - ' + str(xgb_specificity_ci_upper) + ')'\n",
    "\n",
    "xgb_auprc = round(average_precision_score(y_test, probs_xgb), 3)\n",
    "xgb_auprc_ci_length = z_value * np.sqrt((xgb_auprc * (1 - xgb_auprc)) / y_test.shape[0])\n",
    "xgb_auprc_ci_lower = round((xgb_auprc - xgb_auprc_ci_length), 3)\n",
    "xgb_auprc_ci_upper = round((xgb_auprc + xgb_auprc_ci_length), 3)\n",
    "xgb_auprc_str = str(xgb_auprc) + ' (' + str(xgb_auprc_ci_lower) + ' - ' + str(xgb_auprc_ci_upper) + ')'\n",
    "\n",
    "xgb_accuracy = round(accuracy_score(y_test, preds_xgb), 3)\n",
    "xgb_accuracy_ci_length = z_value * np.sqrt((xgb_accuracy * (1 - xgb_accuracy)) / y_test.shape[0])\n",
    "xgb_accuracy_ci_lower = round((xgb_accuracy - xgb_accuracy_ci_length), 3)\n",
    "xgb_accuracy_ci_upper = round((xgb_accuracy + xgb_accuracy_ci_length), 3)\n",
    "xgb_accuracy_str = str(xgb_accuracy) + ' (' + str(xgb_accuracy_ci_lower) + ' - ' + str(xgb_accuracy_ci_upper) + ')'\n",
    "\n",
    "xgb_auroc, xgb_auroc_ci_lower, xgb_auroc_ci_upper = auroc_ci(y_test, probs_xgb)\n",
    "xgb_auroc = round(xgb_auroc, 3)\n",
    "xgb_auroc_ci_lower = round(xgb_auroc_ci_lower, 3)\n",
    "xgb_auroc_ci_upper = round(xgb_auroc_ci_upper, 3)\n",
    "xgb_auroc_str = str(xgb_auroc) + ' (' + str(xgb_auroc_ci_lower) + ' - ' + str(xgb_auroc_ci_upper) + ')'\n",
    "\n",
    "xgb_brier = round(brier_score_loss(y_test, probs_xgb), 3)\n",
    "xgb_brier_ci_length = z_value * np.sqrt((xgb_brier * (1 - xgb_brier)) / y_test.shape[0])\n",
    "xgb_brier_ci_lower = round((xgb_brier - xgb_brier_ci_length), 3)\n",
    "xgb_brier_ci_upper = round((xgb_brier + xgb_brier_ci_length), 3)\n",
    "xgb_brier_str = str(xgb_brier) + ' (' + str(xgb_brier_ci_lower) + ' - ' + str(xgb_brier_ci_upper) + ')'\n",
    "\n",
    "xgb_results = [xgb_sensitivity_str, xgb_specificity_str, xgb_auprc_str, xgb_accuracy_str, xgb_auroc_str, xgb_brier_str]\n",
    "\n",
    "print(\"Sensitivity: \", (xgb_sensitivity_str))\n",
    "print(\"Specificity: \", (xgb_specificity_str))\n",
    "print('AUPRC: ', (xgb_auprc_str))\n",
    "print('Accuracy: ', (xgb_accuracy_str))\n",
    "print('AUROC: ', (xgb_auroc_str))\n",
    "print('Brier Score: ', (xgb_brier_str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 38,
     "status": "ok",
     "timestamp": 1691626795715,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "Ch-wpxcQIQPg",
    "outputId": "549d40df-f0a5-4c3f-e629-be8b10bf8113"
   },
   "outputs": [],
   "source": [
    "#Recalculate precision recall curve for plotting purposes.\n",
    "\n",
    "xgb_precision_curve, xgb_recall_curve, _ = precision_recall_curve(y_test, probs_xgb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9VszU3k5IQPh"
   },
   "source": [
    "# LightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 30,
     "status": "ok",
     "timestamp": 1691626795716,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "vPMTdOzC3Iwc",
    "outputId": "b2e03323-940a-4c6b-9510-f6f04f81b4d1"
   },
   "outputs": [],
   "source": [
    "import lightgbm as lgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 22762,
     "status": "ok",
     "timestamp": 1691626818462,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "oR81p9HpIQPh",
    "outputId": "d3e698b2-b5ef-4dc4-cde6-4cbf7eff0061"
   },
   "outputs": [],
   "source": [
    "#Hyperparameter tuning for LightGBM.\n",
    "\n",
    "def objective(trial):\n",
    "    dtrain = lgb.Dataset(train_x, label=train_y)\n",
    "\n",
    "    param = {\n",
    "        \"objective\":  trial.suggest_categorical(\"objective\", [\"binary\"]),\n",
    "        \"metric\": \"binary_logloss\",\n",
    "        \"verbosity\": -1,\n",
    "        \"random_state\": 31,\n",
    "        \"boosting_type\":  trial.suggest_categorical(\"boosting_type\", [\"gbdt\"]),\n",
    "        \"lambda_l1\": trial.suggest_float(\"lambda_l1\", 1e-8, 10.0, log=True),\n",
    "        \"lambda_l2\": trial.suggest_float(\"lambda_l2\", 1e-8, 10.0, log=True),\n",
    "        \"num_leaves\": trial.suggest_int(\"num_leaves\", 2, 256),\n",
    "        \"feature_fraction\": trial.suggest_float(\"feature_fraction\", 0.4, 1.0),\n",
    "        \"bagging_fraction\": trial.suggest_float(\"bagging_fraction\", 0.4, 1.0),\n",
    "        \"bagging_freq\": trial.suggest_int(\"bagging_freq\", 1, 7),\n",
    "        \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 5, 100),\n",
    "    }\n",
    "\n",
    "    gbm = lgb.train(param, dtrain)\n",
    "    preds = gbm.predict(valid_x)\n",
    "    pred_labels = np.rint(preds)\n",
    "    auc = sklearn.metrics.roc_auc_score(valid_y, pred_labels)\n",
    "    return auc\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    study = optuna.create_study(direction=\"maximize\", sampler = TPESampler(seed=31))\n",
    "    study.optimize(objective, n_trials=100)\n",
    "\n",
    "    print(\"Number of finished trials: {}\".format(len(study.trials)))\n",
    "\n",
    "    print(\"Best trial:\")\n",
    "    trial = study.best_trial\n",
    "\n",
    "    print(\"  Value: {}\".format(trial.value))\n",
    "\n",
    "    print(\"  Params: \")\n",
    "    for key, value in trial.params.items():\n",
    "        print(\"    {}: {}\".format(key, value))\n",
    "\n",
    "    lgb_params = {}\n",
    "\n",
    "    for key, value in trial.params.items():\n",
    "        lgb_params[key] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 141,
     "status": "ok",
     "timestamp": 1691626818462,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "h7P7Mw7UkC-y",
    "outputId": "19afe665-4b99-404c-de81-b5783293eb51"
   },
   "outputs": [],
   "source": [
    "#See LightGBM hyperparameters.\n",
    "\n",
    "lgb_params['metric'] = 'binary_logloss'\n",
    "lgb_params['verbosity'] = -1\n",
    "lgb_params['random_state'] = 31\n",
    "\n",
    "print(lgb_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 247
    },
    "executionInfo": {
     "elapsed": 107,
     "status": "ok",
     "timestamp": 1691626818463,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "UAidUSs1IQPh",
    "outputId": "32d1dc99-1b28-4208-bbf7-6d9674dd76eb"
   },
   "outputs": [],
   "source": [
    "#Fit LightGBM.\n",
    "\n",
    "import lightgbm as lgb\n",
    "\n",
    "lgb = lgb.LGBMClassifier(**lgb_params)\n",
    "\n",
    "lgb.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 111
    },
    "executionInfo": {
     "elapsed": 368,
     "status": "ok",
     "timestamp": 1691626818736,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "JlKe1z0RIQPh",
    "outputId": "543ba93e-74aa-40ad-a916-196c718002c0"
   },
   "outputs": [],
   "source": [
    "#Predict on the validation set, get predicted probabilities for calibration, and fit calibration function.\n",
    "\n",
    "calib_probs_lgb = lgb.predict_proba(valid_x)\n",
    "\n",
    "calib_model_lgb = LogisticRegression()\n",
    "calib_model_lgb.fit(calib_probs_lgb, valid_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 95,
     "status": "ok",
     "timestamp": 1691626818737,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "mPyYlwGX-HkZ",
    "outputId": "1c35df15-1ddd-41fe-f607-64cd63de9435"
   },
   "outputs": [],
   "source": [
    "#Make predictions on the test set based on the trained and calibrated LightGBM model.\n",
    "\n",
    "preds_lgb = lgb.predict(x_test)\n",
    "\n",
    "uncalibrated_probs_lgb = lgb.predict_proba(x_test)\n",
    "uncalibrated_probs_lgb = uncalibrated_probs_lgb\n",
    "\n",
    "probs_lgb = calib_model_lgb.predict_proba(uncalibrated_probs_lgb)\n",
    "probs_lgb = probs_lgb[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 3546,
     "status": "ok",
     "timestamp": 1691627506444,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "r259oDdIIQPi",
    "outputId": "76d6a677-e806-41ac-e82d-34a4dd5a43fc"
   },
   "outputs": [],
   "source": [
    "#Evaluate LightGBM model.\n",
    "\n",
    "lgb_tn, lgb_fp, lgb_fn, lgb_tp = confusion_matrix(y_test, preds_lgb).ravel()\n",
    "lgb_cm = confusion_matrix(y_test, preds_lgb)\n",
    "\n",
    "lgb_sensitivity = round(lgb_tp/(lgb_tp+lgb_fn), 3)\n",
    "lgb_sensitivity_ci_length = z_value * np.sqrt((lgb_sensitivity * (1 - lgb_sensitivity)) / y_test.shape[0])\n",
    "lgb_sensitivity_ci_lower = round((lgb_sensitivity - lgb_sensitivity_ci_length), 3)\n",
    "lgb_sensitivity_ci_upper = round((lgb_sensitivity + lgb_sensitivity_ci_length), 3)\n",
    "lgb_sensitivity_str = str(lgb_sensitivity) + ' (' + str(lgb_sensitivity_ci_lower) + ' - ' + str(lgb_sensitivity_ci_upper) + ')'\n",
    "\n",
    "lgb_specificity = round(lgb_tn/(lgb_tn+lgb_fp), 3)\n",
    "lgb_specificity_ci_length = z_value * np.sqrt((lgb_specificity * (1 - lgb_specificity)) / y_test.shape[0])\n",
    "lgb_specificity_ci_lower = round((lgb_specificity - lgb_specificity_ci_length), 3)\n",
    "lgb_specificity_ci_upper = round((lgb_specificity + lgb_specificity_ci_length), 3)\n",
    "lgb_specificity_str = str(lgb_specificity) + ' (' + str(lgb_specificity_ci_lower) + ' - ' + str(lgb_specificity_ci_upper) + ')'\n",
    "\n",
    "lgb_auprc = round(average_precision_score(y_test, probs_lgb), 3)\n",
    "lgb_auprc_ci_length = z_value * np.sqrt((lgb_auprc * (1 - lgb_auprc)) / y_test.shape[0])\n",
    "lgb_auprc_ci_lower = round((lgb_auprc - lgb_auprc_ci_length), 3)\n",
    "lgb_auprc_ci_upper = round((lgb_auprc + lgb_auprc_ci_length), 3)\n",
    "lgb_auprc_str = str(lgb_auprc) + ' (' + str(lgb_auprc_ci_lower) + ' - ' + str(lgb_auprc_ci_upper) + ')'\n",
    "\n",
    "lgb_accuracy = round(accuracy_score(y_test, preds_lgb), 3)\n",
    "lgb_accuracy_ci_length = z_value * np.sqrt((lgb_accuracy * (1 - lgb_accuracy)) / y_test.shape[0])\n",
    "lgb_accuracy_ci_lower = round((lgb_accuracy - lgb_accuracy_ci_length), 3)\n",
    "lgb_accuracy_ci_upper = round((lgb_accuracy + lgb_accuracy_ci_length), 3)\n",
    "lgb_accuracy_str = str(lgb_accuracy) + ' (' + str(lgb_accuracy_ci_lower) + ' - ' + str(lgb_accuracy_ci_upper) + ')'\n",
    "\n",
    "lgb_auroc, lgb_auroc_ci_lower, lgb_auroc_ci_upper = auroc_ci(y_test, probs_lgb)\n",
    "lgb_auroc = round(lgb_auroc, 3)\n",
    "lgb_auroc_ci_lower = round(lgb_auroc_ci_lower, 3)\n",
    "lgb_auroc_ci_upper = round(lgb_auroc_ci_upper, 3)\n",
    "lgb_auroc_str = str(lgb_auroc) + ' (' + str(lgb_auroc_ci_lower) + ' - ' + str(lgb_auroc_ci_upper) + ')'\n",
    "\n",
    "lgb_brier = round(brier_score_loss(y_test, probs_lgb), 3)\n",
    "lgb_brier_ci_length = z_value * np.sqrt((lgb_brier * (1 - lgb_brier)) / y_test.shape[0])\n",
    "lgb_brier_ci_lower = round((lgb_brier - lgb_brier_ci_length), 3)\n",
    "lgb_brier_ci_upper = round((lgb_brier + lgb_brier_ci_length), 3)\n",
    "lgb_brier_str = str(lgb_brier) + ' (' + str(lgb_brier_ci_lower) + ' - ' + str(lgb_brier_ci_upper) + ')'\n",
    "\n",
    "lgb_results = [lgb_sensitivity_str, lgb_specificity_str, lgb_auprc_str, lgb_accuracy_str, lgb_auroc_str, lgb_brier_str]\n",
    "\n",
    "print(\"Sensitivity: \", (lgb_sensitivity_str))\n",
    "print(\"Specificity: \", (lgb_specificity_str))\n",
    "print('AUPRC: ', (lgb_auprc_str))\n",
    "print('Accuracy: ', (lgb_accuracy_str))\n",
    "print('AUROC: ', (lgb_auroc_str))\n",
    "print('Brier Score: ', (lgb_brier_str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 41,
     "status": "ok",
     "timestamp": 1691626821216,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "PXzXI9O-IQPi",
    "outputId": "8f2028db-448d-4be0-b967-34c255b516ae"
   },
   "outputs": [],
   "source": [
    "#Recalculate precision recall curve for plotting purposes.\n",
    "\n",
    "lgb_precision_curve, lgb_recall_curve, _ = precision_recall_curve(y_test, probs_lgb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mQRRrBp-IQPk"
   },
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 600991,
     "status": "ok",
     "timestamp": 1691627422176,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "kFTmQMQxIQPk",
    "outputId": "fac3088a-6df1-4c0a-f826-841989979d27"
   },
   "outputs": [],
   "source": [
    "#Hyperparameter tuning for Random Forest.\n",
    "\n",
    "def objective(trial):\n",
    "\n",
    "    param = {\n",
    "        \"criterion\": trial.suggest_categorical(\"criterion\", [\"gini\", \"entropy\"]),\n",
    "        \"random_state\": 31,\n",
    "        \"max_features\": trial.suggest_categorical(\"max_features\", [\"auto\", \"sqrt\",\"log2\", None]),\n",
    "        \"max_depth\": trial.suggest_int(\"max_depth\", 1, 100),\n",
    "        \"n_estimators\": trial.suggest_int(\"n_estimators\", 100, 2000, 100),\n",
    "        \"min_samples_leaf\": trial.suggest_int(\"min_samples_leaf\", 1, 4, 1),\n",
    "        \"min_samples_split\": trial.suggest_int(\"min_samples_split\", 2, 10, 1),\n",
    "    }\n",
    "\n",
    "    rf = RandomForestClassifier(**param)\n",
    "\n",
    "    rf.fit(\n",
    "        train_x,\n",
    "        train_y,\n",
    "    )\n",
    "\n",
    "    preds = rf.predict(valid_x)\n",
    "    pred_labels = np.rint(preds)\n",
    "    auc = sklearn.metrics.roc_auc_score(valid_y, pred_labels)\n",
    "\n",
    "    return auc\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    study = optuna.create_study(direction='maximize', sampler = TPESampler(seed=31))\n",
    "    study.optimize(objective, n_trials=100, timeout=600)\n",
    "\n",
    "    print(\"Number of finished trials: {}\".format(len(study.trials)))\n",
    "\n",
    "    print(\"Best trial:\")\n",
    "    trial = study.best_trial\n",
    "\n",
    "    print(\"  Value: {}\".format(trial.value))\n",
    "\n",
    "    print(\"  Params: \")\n",
    "    for key, value in trial.params.items():\n",
    "        print(\"    {}: {}\".format(key, value))\n",
    "\n",
    "    rf_params = {}\n",
    "\n",
    "    for key, value in trial.params.items():\n",
    "        rf_params[key] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 36,
     "status": "ok",
     "timestamp": 1691627422177,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "yTqnQTzgkG3R",
    "outputId": "d8b2282f-aa70-400a-c599-3c1cddff7b9b"
   },
   "outputs": [],
   "source": [
    "#See Random Forest hyperparameters.\n",
    "\n",
    "rf_params['random_state'] = 31\n",
    "\n",
    "print(rf_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 91
    },
    "executionInfo": {
     "elapsed": 6014,
     "status": "ok",
     "timestamp": 1691627428181,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "o2nnbDAsIQPk",
    "outputId": "0be40a7d-16cc-4211-cd34-54de08a7e36d"
   },
   "outputs": [],
   "source": [
    "#Fit Random Forest.\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "rf = RandomForestClassifier(**rf_params)\n",
    "\n",
    "rf.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 111
    },
    "executionInfo": {
     "elapsed": 479,
     "status": "ok",
     "timestamp": 1691627428552,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "pSsBliFuIQPk",
    "outputId": "aa5b4890-f557-4329-b757-97aab3b189d9"
   },
   "outputs": [],
   "source": [
    "#Predict on the validation set, get predicted probabilities for calibration, and fit calibration function.\n",
    "\n",
    "calib_probs_rf = rf.predict_proba(valid_x)\n",
    "\n",
    "calib_model_rf = LogisticRegression()\n",
    "calib_model_rf.fit(calib_probs_rf, valid_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 814,
     "status": "ok",
     "timestamp": 1691627429273,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "o3Ru1zLF-YER",
    "outputId": "9006acd3-fe3c-4468-85b4-d5f4b38f1112"
   },
   "outputs": [],
   "source": [
    "#Make predictions on the test set based on the trained Random Forest model.\n",
    "\n",
    "preds_rf = rf.predict(x_test)\n",
    "\n",
    "uncalibrated_probs_rf = rf.predict_proba(x_test)\n",
    "\n",
    "probs_rf = calib_model_rf.predict_proba(uncalibrated_probs_rf)\n",
    "probs_rf = probs_rf[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "haQcIGjoIQPk"
   },
   "outputs": [],
   "source": [
    "#Evaluate Random Forest model.\n",
    "\n",
    "rf_tn, rf_fp, rf_fn, rf_tp = confusion_matrix(y_test, preds_rf).ravel()\n",
    "rf_cm = confusion_matrix(y_test, preds_rf)\n",
    "\n",
    "rf_sensitivity = round(rf_tp/(rf_tp+rf_fn), 3)\n",
    "rf_sensitivity_ci_length = z_value * np.sqrt((rf_sensitivity * (1 - rf_sensitivity)) / y_test.shape[0])\n",
    "rf_sensitivity_ci_lower = round((rf_sensitivity - rf_sensitivity_ci_length), 3)\n",
    "rf_sensitivity_ci_upper = round((rf_sensitivity + rf_sensitivity_ci_length), 3)\n",
    "rf_sensitivity_str = str(rf_sensitivity) + ' (' + str(rf_sensitivity_ci_lower) + ' - ' + str(rf_sensitivity_ci_upper) + ')'\n",
    "\n",
    "rf_specificity = round(rf_tn/(rf_tn+rf_fp), 3)\n",
    "rf_specificity_ci_length = z_value * np.sqrt((rf_specificity * (1 - rf_specificity)) / y_test.shape[0])\n",
    "rf_specificity_ci_lower = round((rf_specificity - rf_specificity_ci_length), 3)\n",
    "rf_specificity_ci_upper = round((rf_specificity + rf_specificity_ci_length), 3)\n",
    "rf_specificity_str = str(rf_specificity) + ' (' + str(rf_specificity_ci_lower) + ' - ' + str(rf_specificity_ci_upper) + ')'\n",
    "\n",
    "rf_auprc = round(average_precision_score(y_test, probs_rf), 3)\n",
    "rf_auprc_ci_length = z_value * np.sqrt((rf_auprc * (1 - rf_auprc)) / y_test.shape[0])\n",
    "rf_auprc_ci_lower = round((rf_auprc - rf_auprc_ci_length), 3)\n",
    "rf_auprc_ci_upper = round((rf_auprc + rf_auprc_ci_length), 3)\n",
    "rf_auprc_str = str(rf_auprc) + ' (' + str(rf_auprc_ci_lower) + ' - ' + str(rf_auprc_ci_upper) + ')'\n",
    "\n",
    "rf_accuracy = round(accuracy_score(y_test, preds_rf), 3)\n",
    "rf_accuracy_ci_length = z_value * np.sqrt((rf_accuracy * (1 - rf_accuracy)) / y_test.shape[0])\n",
    "rf_accuracy_ci_lower = round((rf_accuracy - rf_accuracy_ci_length), 3)\n",
    "rf_accuracy_ci_upper = round((rf_accuracy + rf_accuracy_ci_length), 3)\n",
    "rf_accuracy_str = str(rf_accuracy) + ' (' + str(rf_accuracy_ci_lower) + ' - ' + str(rf_accuracy_ci_upper) + ')'\n",
    "\n",
    "rf_auroc, rf_auroc_ci_lower, rf_auroc_ci_upper = auroc_ci(y_test, probs_rf)\n",
    "rf_auroc = round(rf_auroc, 3)\n",
    "rf_auroc_ci_lower = round(rf_auroc_ci_lower, 3)\n",
    "rf_auroc_ci_upper = round(rf_auroc_ci_upper, 3)\n",
    "rf_auroc_str = str(rf_auroc) + ' (' + str(rf_auroc_ci_lower) + ' - ' + str(rf_auroc_ci_upper) + ')'\n",
    "\n",
    "rf_brier = round(brier_score_loss(y_test, probs_rf), 3)\n",
    "rf_brier_ci_length = z_value * np.sqrt((rf_brier * (1 - rf_brier)) / y_test.shape[0])\n",
    "rf_brier_ci_lower = round((rf_brier - rf_brier_ci_length), 3)\n",
    "rf_brier_ci_upper = round((rf_brier + rf_brier_ci_length), 3)\n",
    "rf_brier_str = str(rf_brier) + ' (' + str(rf_brier_ci_lower) + ' - ' + str(rf_brier_ci_upper) + ')'\n",
    "\n",
    "rf_results = [rf_sensitivity_str, rf_specificity_str, rf_auprc_str, rf_accuracy_str, rf_auroc_str, rf_brier_str]\n",
    "\n",
    "print(\"Sensitivity: \", (rf_sensitivity_str))\n",
    "print(\"Specificity: \", (rf_specificity_str))\n",
    "print('AUPRC: ', (rf_auprc_str))\n",
    "print('Accuracy: ', (rf_accuracy_str))\n",
    "print('AUROC: ', (rf_auroc_str))\n",
    "print('Brier Score: ', (rf_brier_str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 17,
     "status": "ok",
     "timestamp": 1691627431594,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "Drrp3KEQIQPk",
    "outputId": "8e621b3d-77cd-4c3f-8042-0a5c860bf951"
   },
   "outputs": [],
   "source": [
    "#Recalculate precision recall curve for plotting purposes.\n",
    "\n",
    "rf_precision_curve, rf_recall_curve, _ = precision_recall_curve(y_test, probs_rf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "fLvNgketIQPl"
   },
   "source": [
    "# ROC and PR Curves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 2123,
     "status": "ok",
     "timestamp": 1691627945181,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "0mMqO-euIQPl",
    "outputId": "dbfe1760-a6e1-4d67-947d-a9c2b688259e"
   },
   "outputs": [],
   "source": [
    "#Plot ROC curve.\n",
    "\n",
    "f = pyplot.figure()\n",
    "f.set_figwidth(12)\n",
    "f.set_figheight(12)\n",
    "\n",
    "tabpfn_fpr, tabpfn_tpr, _ = roc_curve(y_test, probs_tabpfn)\n",
    "tabpfn_label = 'TabPFN AUROC: ' + tabpfn_auroc_str\n",
    "pyplot.plot(tabpfn_fpr, tabpfn_tpr, label = tabpfn_label, color = 'b', linewidth = 3.5, alpha = 0.75)\n",
    "\n",
    "tabnet_fpr, tabnet_tpr, _ = roc_curve(y_test, probs_tabnet)\n",
    "tabnet_label = 'TabNet AUROC: ' + tabnet_auroc_str\n",
    "pyplot.plot(tabnet_fpr, tabnet_tpr, label = tabnet_label, color = 'g', linewidth = 3.5, alpha = 0.75)\n",
    "\n",
    "xgb_fpr, xgb_tpr, _ = roc_curve(y_test, probs_xgb)\n",
    "xgb_label = 'XGBoost AUROC: ' + xgb_auroc_str\n",
    "pyplot.plot(xgb_fpr, xgb_tpr, label = xgb_label, color = 'r', linewidth = 3.5, alpha = 0.75)\n",
    "\n",
    "lgb_fpr, lgb_tpr, _ = roc_curve(y_test, probs_lgb)\n",
    "lgb_label = 'LightGBM AUROC: ' + lgb_auroc_str\n",
    "pyplot.plot(lgb_fpr, lgb_tpr, label = lgb_label, color='c', linewidth = 3.5, alpha = 0.75)\n",
    "\n",
    "rf_fpr, rf_tpr, _ = roc_curve(y_test, probs_rf)\n",
    "rf_label = 'Random Forest AUROC: ' + rf_auroc_str\n",
    "pyplot.plot(rf_fpr, rf_tpr, label = rf_label, color = 'm', linewidth = 3.5, alpha = 0.75)\n",
    "\n",
    "pyplot.plot([0, 1], [0, 1], linestyle = '--', linewidth=2)\n",
    "\n",
    "pyplot.title('C', x = -0.075, y = 1.005, fontsize = 75, pad = 20)\n",
    "pyplot.xlabel('False Positive Rate', fontsize = 22, fontweight = 'heavy', labelpad = 16)\n",
    "pyplot.ylabel('True Positive Rate', fontsize = 22, fontweight = 'heavy', labelpad = 16)\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", labelsize = 16)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", labelsize = 16)\n",
    "\n",
    "leg = pyplot.legend(loc = 'lower right', fontsize = 18)\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g2_36m_roc.png', dpi=300)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 1832,
     "status": "ok",
     "timestamp": 1691627952577,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "GpotrYFBIQPl",
    "outputId": "a976a182-b66a-4515-afb0-33f80d234d17"
   },
   "outputs": [],
   "source": [
    "#Plot PR curve.\n",
    "\n",
    "f = pyplot.figure()\n",
    "f.set_figwidth(12)\n",
    "f.set_figheight(12)\n",
    "\n",
    "tabpfn_label = 'TabPFN AUPRC: ' + tabpfn_auprc_str\n",
    "tabnet_label = 'TabNet AUPRC: ' + tabnet_auprc_str\n",
    "xgb_label = 'XGBoost AUPRC: ' + xgb_auprc_str\n",
    "lgb_label = 'LightGBM AUPRC: ' + lgb_auprc_str\n",
    "rf_label = 'Random Forest AUPRC: ' + rf_auprc_str\n",
    "\n",
    "pyplot.plot(tabpfn_recall_curve, tabpfn_precision_curve, label = tabpfn_label, color = 'b', linewidth = 3.5, alpha = 0.75)\n",
    "pyplot.plot(tabnet_recall_curve, tabnet_precision_curve, label = tabnet_label, color = 'g', linewidth = 3.5, alpha = 0.75)\n",
    "pyplot.plot(xgb_recall_curve, xgb_precision_curve, label = xgb_label, color = 'r', linewidth = 3.5, alpha = 0.75)\n",
    "pyplot.plot(lgb_recall_curve, lgb_precision_curve, label = lgb_label, color = 'c', linewidth = 3.5, alpha = 0.75)\n",
    "pyplot.plot(rf_recall_curve, rf_precision_curve, label = rf_label, color = 'm', linewidth = 3.5, alpha = 0.75)\n",
    "\n",
    "pyplot.title('C', x = -0.075, y = 1.005, fontsize = 75, pad = 20)\n",
    "pyplot.xlabel('Recall', fontsize = 18, fontweight = 'heavy', labelpad = 16)\n",
    "pyplot.ylabel('Precision', fontsize = 22, fontweight = 'heavy', labelpad = 16)\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", labelsize = 16)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", labelsize = 16)\n",
    "\n",
    "leg = pyplot.legend(loc = 'lower left', fontsize = 18)\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g2_36m_prc.png', dpi=300)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rOySRdcDIQPl"
   },
   "source": [
    "# Results Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 350
    },
    "executionInfo": {
     "elapsed": 129,
     "status": "ok",
     "timestamp": 1691627954167,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "kznujF4kIQPl",
    "outputId": "842bf51f-f5f7-40ff-cb06-da6ce7216993"
   },
   "outputs": [],
   "source": [
    "results = {'TabPFN':tabpfn_results, 'TabNet':tabnet_results, 'XGBoost':xgb_results, 'LightGBM':lgb_results, 'Random Forest':rf_results}\n",
    "\n",
    "results = pd.DataFrame(results, columns = ['TabPFN', 'TabNet', 'XGBoost', 'LightGBM', 'Random Forest'])\n",
    "results = results.T\n",
    "\n",
    "results.columns = ['Sensitivity (95% CI)', 'Specificity (95% CI)',  'AUPRC (95% CI)', 'Accuracy (95% CI)', 'AUROC (95% CI)', 'Brier Score (95% CI)']\n",
    "\n",
    "results.to_csv('/content/drive/MyDrive/NCDB-G2G3 Glioma/g2_36m_results.csv')\n",
    "\n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uAD5jLGchIp4"
   },
   "source": [
    "# Confusion Matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 716
    },
    "executionInfo": {
     "elapsed": 1107,
     "status": "ok",
     "timestamp": 1691627987803,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "BDEMrQ5khLfg",
    "outputId": "88337c92-e823-46d0-e736-7f13a837ea0c"
   },
   "outputs": [],
   "source": [
    "tabpfn_cm = confusion_matrix(y_test, preds_tabpfn)\n",
    "\n",
    "f = pyplot.figure(figsize=(8,8))\n",
    "\n",
    "sns.heatmap(tabpfn_cm, annot=True, fmt='d', cmap='Blues', cbar=False, annot_kws={\"size\": 16}, linewidths=1, linecolor='black')\n",
    "\n",
    "labels = ['Survival', 'Mortality']\n",
    "pyplot.xticks([0.5,1.5], labels, fontsize=16, fontweight='heavy')\n",
    "pyplot.yticks([0.5,1.5], labels, fontsize=16, fontweight='heavy', va='center')\n",
    "\n",
    "pyplot.xlabel('Predicted', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "pyplot.ylabel('Truth', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", pad=10)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", pad=10)\n",
    "pyplot.title('A', x = -0.095, y = 1.005, fontsize = 75, pad = 20)\n",
    "\n",
    "pyplot.subplots_adjust(left=0.20, right=0.85, bottom=0.20, top=0.80)\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g2_36m_cm_tabpfn.png', dpi=300)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 753
    },
    "executionInfo": {
     "elapsed": 907,
     "status": "ok",
     "timestamp": 1691627988598,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "MRXGPobHhNEX",
    "outputId": "a4a5524e-4f6e-4779-f58b-56329f01dc70"
   },
   "outputs": [],
   "source": [
    "tabnet_cm = confusion_matrix(y_test, preds_tabnet)\n",
    "\n",
    "f = pyplot.figure(figsize=(8,8))\n",
    "\n",
    "sns.heatmap(tabnet_cm, annot=True, fmt='d', cmap='Blues', cbar=False, annot_kws={\"size\": 16}, linewidths=1, linecolor='black')\n",
    "\n",
    "labels = ['Survival', 'Mortality']\n",
    "pyplot.xticks([0.5,1.5], labels, fontsize=16, fontweight='heavy')\n",
    "pyplot.yticks([0.5,1.5], labels, fontsize=16, fontweight='heavy', va='center')\n",
    "\n",
    "pyplot.xlabel('Predicted', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "pyplot.ylabel('Truth', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", pad=10)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", pad=10)\n",
    "pyplot.title('B', x = -0.095, y = 1.005, fontsize = 75, pad = 20)\n",
    "\n",
    "pyplot.subplots_adjust(left=0.20, right=0.85, bottom=0.20, top=0.80)\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g2_36m_cm_tabnet.png', dpi=300)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 753
    },
    "executionInfo": {
     "elapsed": 966,
     "status": "ok",
     "timestamp": 1691627989474,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "xLZLslcChShm",
    "outputId": "63563191-bec8-415e-d3ce-eb622e6975bd"
   },
   "outputs": [],
   "source": [
    "xgb_cm = confusion_matrix(y_test, preds_xgb)\n",
    "\n",
    "f = pyplot.figure(figsize=(8,8))\n",
    "\n",
    "sns.heatmap(xgb_cm, annot=True, fmt='d', cmap='Blues', cbar=False, annot_kws={\"size\": 16}, linewidths=1, linecolor='black')\n",
    "\n",
    "labels = ['Survival', 'Mortality']\n",
    "pyplot.xticks([0.5,1.5], labels, fontsize=16, fontweight='heavy')\n",
    "pyplot.yticks([0.5,1.5], labels, fontsize=16, fontweight='heavy', va='center')\n",
    "\n",
    "pyplot.xlabel('Predicted', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "pyplot.ylabel('Truth', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", pad=10)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", pad=10)\n",
    "pyplot.title('C', x = -0.095, y = 1.005, fontsize = 75, pad = 20)\n",
    "\n",
    "pyplot.subplots_adjust(left=0.20, right=0.85, bottom=0.20, top=0.80)\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g2_36m_cm_xgb.png', dpi=300)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 753
    },
    "executionInfo": {
     "elapsed": 208,
     "status": "ok",
     "timestamp": 1691627989474,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "QMNzhoAAhUnc",
    "outputId": "89f1bd9f-47cd-4890-c5e4-7abcf548887f"
   },
   "outputs": [],
   "source": [
    "lgb_cm = confusion_matrix(y_test, preds_lgb)\n",
    "\n",
    "f = pyplot.figure(figsize=(8,8))\n",
    "\n",
    "sns.heatmap(lgb_cm, annot=True, fmt='d', cmap='Blues', cbar=False, annot_kws={\"size\": 16}, linewidths=1, linecolor='black')\n",
    "\n",
    "labels = ['Survival', 'Mortality']\n",
    "pyplot.xticks([0.5,1.5], labels, fontsize=16, fontweight='heavy')\n",
    "pyplot.yticks([0.5,1.5], labels, fontsize=16, fontweight='heavy', va='center')\n",
    "\n",
    "pyplot.xlabel('Predicted', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "pyplot.ylabel('Truth', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", pad=10)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", pad=10)\n",
    "pyplot.title('B', x = -0.095, y = 1.005, fontsize = 75, pad = 20)\n",
    "\n",
    "pyplot.subplots_adjust(left=0.20, right=0.85, bottom=0.20, top=0.80)\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g2_36m_cm_lgb.png', dpi=300)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 753
    },
    "executionInfo": {
     "elapsed": 457,
     "status": "ok",
     "timestamp": 1691627989743,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "-Rn8yc5QhWqo",
    "outputId": "f69e8d55-1f72-4f5c-8beb-8c09c293df34"
   },
   "outputs": [],
   "source": [
    "rf_cm = confusion_matrix(y_test, preds_rf)\n",
    "\n",
    "f = pyplot.figure(figsize=(8,8))\n",
    "\n",
    "sns.heatmap(rf_cm, annot=True, fmt='d', cmap='Blues', cbar=False, annot_kws={\"size\": 16}, linewidths=1, linecolor='black')\n",
    "\n",
    "labels = ['Survival', 'Mortality']\n",
    "pyplot.xticks([0.5,1.5], labels, fontsize=16, fontweight='heavy')\n",
    "pyplot.yticks([0.5,1.5], labels, fontsize=16, fontweight='heavy', va='center')\n",
    "\n",
    "pyplot.xlabel('Predicted', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "pyplot.ylabel('Truth', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", pad=10)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", pad=10)\n",
    "pyplot.title('D', x = -0.095, y = 1.005, fontsize = 75, pad = 20)\n",
    "\n",
    "pyplot.subplots_adjust(left=0.20, right=0.85, bottom=0.20, top=0.80)\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g2_36m_cm_rf.png', dpi=300)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qYnu4PnCIQPm"
   },
   "source": [
    "# SHAP Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 97,
     "status": "ok",
     "timestamp": 1691627989743,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "BuvljQiYu-Xb",
    "outputId": "195eb284-5580-48c7-eb3d-86625d936fe3"
   },
   "outputs": [],
   "source": [
    "import textwrap\n",
    "def wrap_labels(ax, width, break_long_words=False):\n",
    "    labels = []\n",
    "    for label in ax.get_yticklabels():\n",
    "        text = label.get_text()\n",
    "        labels.append(textwrap.fill(text, width=width,\n",
    "                                    break_long_words=break_long_words))\n",
    "    ax.set_yticklabels(labels, rotation=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 92,
     "status": "ok",
     "timestamp": 1691627989743,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "gjL0ZhOZJh-b"
   },
   "outputs": [],
   "source": [
    "feature_names = x_test.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true,
     "base_uri": "https://localhost:8080/"
    },
    "id": "GTKndsLJzH7h"
   },
   "outputs": [],
   "source": [
    "#Calculate SHAP values for TabPFN.\n",
    "\n",
    "tabpfn_explainer = shap.Explainer(tabpfn.predict, x_test)\n",
    "tabpfn_shap_values = tabpfn_explainer(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "Lw5zXxxQzH7p"
   },
   "outputs": [],
   "source": [
    "#Plot SHAP bar plot for TabPFN.\n",
    "\n",
    "shap.plots.bar(tabpfn_shap_values, max_display = 15, show=False)\n",
    "\n",
    "fig = pyplot.gcf()\n",
    "ax = pyplot.gca()\n",
    "fig.set_figheight(12)\n",
    "fig.set_figwidth(5)\n",
    "\n",
    "pyplot.title('A', x = -0.5, y = 1, fontsize = 50, pad = 20)\n",
    "pyplot.xlabel(\"Mean |SHAP Value|\", fontsize =12, fontweight = 'heavy', labelpad = 8)\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", labelsize = 12)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", labelsize = 12)\n",
    "\n",
    "wrap_labels(ax, 30)\n",
    "ax.figure\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g2_36m_shap_tabpfn.png', dpi=300, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "XotWpfehXbCw"
   },
   "outputs": [],
   "source": [
    "#Calculate SHAP values for TabNet.\n",
    "\n",
    "tabnet_explainer = shap.Explainer(tabnet.predict, x_test.values, feature_names=feature_names)\n",
    "tabnet_shap_values = tabnet_explainer(x_test.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "tzHasDCwXoxs"
   },
   "outputs": [],
   "source": [
    "#Plot SHAP bar plot for TabNet.\n",
    "\n",
    "shap.plots.bar(tabnet_shap_values, max_display = 15, show=False)\n",
    "\n",
    "fig = pyplot.gcf()\n",
    "ax = pyplot.gca()\n",
    "fig.set_figheight(12)\n",
    "fig.set_figwidth(5)\n",
    "\n",
    "pyplot.title('B', x = -0.5, y = 1, fontsize = 50, pad = 20)\n",
    "pyplot.xlabel(\"Mean |SHAP Value|\", fontsize =12, fontweight = 'heavy', labelpad = 8)\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", labelsize = 12)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", labelsize = 12)\n",
    "\n",
    "wrap_labels(ax, 30)\n",
    "ax.figure\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g2_36m_shap_tabnet.png', dpi=300, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "C09esVZrIQPm"
   },
   "outputs": [],
   "source": [
    "#Calculate SHAP values for XGBoost.\n",
    "\n",
    "xgb_explainer = shap.Explainer(xgb.predict, x_test)\n",
    "xgb_shap_values = xgb_explainer(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "6O74ff97qW2H"
   },
   "outputs": [],
   "source": [
    "#Plot SHAP bar plot for XGBoost.\n",
    "\n",
    "shap.plots.bar(xgb_shap_values, max_display = 15, show=False)\n",
    "\n",
    "fig = pyplot.gcf()\n",
    "ax = pyplot.gca()\n",
    "fig.set_figheight(12)\n",
    "fig.set_figwidth(5)\n",
    "\n",
    "pyplot.title('C', x = -0.5, y = 1, fontsize = 50, pad = 20)\n",
    "pyplot.xlabel(\"Mean |SHAP Value|\", fontsize =12, fontweight = 'heavy', labelpad = 8)\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", labelsize = 12)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", labelsize = 12)\n",
    "\n",
    "wrap_labels(ax, 30)\n",
    "ax.figure\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g2_36m_shap_xgb.png', dpi=300, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "HBQmzC4-IQPm"
   },
   "outputs": [],
   "source": [
    "#Calculate SHAP values for LightGBM.\n",
    "\n",
    "lgb_explainer = shap.Explainer(lgb.predict, x_test)\n",
    "lgb_shap_values = lgb_explainer(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "4V-NEfx_xysV"
   },
   "outputs": [],
   "source": [
    "#Plot SHAP bar plot for LightGBM.\n",
    "\n",
    "shap.plots.bar(lgb_shap_values, max_display = 15, show=False)\n",
    "\n",
    "fig = pyplot.gcf()\n",
    "ax = pyplot.gca()\n",
    "fig.set_figheight(12)\n",
    "fig.set_figwidth(5)\n",
    "\n",
    "pyplot.title('B', x = -0.5, y = 1, fontsize = 50, pad = 20)\n",
    "pyplot.xlabel(\"Mean |SHAP Value|\", fontsize =12, fontweight = 'heavy', labelpad = 8)\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", labelsize = 12)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", labelsize = 12)\n",
    "\n",
    "wrap_labels(ax, 30)\n",
    "ax.figure\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g2_36m_shap_lgb.png', dpi=300, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "Vj3B1IeWIQPm"
   },
   "outputs": [],
   "source": [
    "#Calculate SHAP values for Random Forest.\n",
    "\n",
    "rf_explainer = shap.Explainer(rf.predict, x_test)\n",
    "rf_shap_values = rf_explainer(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "x2P2or1bx25u"
   },
   "outputs": [],
   "source": [
    "#Plot SHAP bar plot for Random Forest.\n",
    "\n",
    "shap.plots.bar(rf_shap_values, max_display = 15, show=False)\n",
    "\n",
    "fig = pyplot.gcf()\n",
    "ax = pyplot.gca()\n",
    "fig.set_figheight(12)\n",
    "fig.set_figwidth(5)\n",
    "\n",
    "pyplot.title('D', x = -0.5, y = 1, fontsize = 50, pad = 20)\n",
    "pyplot.xlabel(\"Mean |SHAP Value|\", fontsize =12, fontweight = 'heavy', labelpad = 8)\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", labelsize = 12)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", labelsize = 12)\n",
    "\n",
    "wrap_labels(ax, 30)\n",
    "ax.figure\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g2_36m_shap_rf.png', dpi=300, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cwYwpeSZz_-x"
   },
   "source": [
    "#Partial Dependency Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "cCezSsG3XK1c"
   },
   "outputs": [],
   "source": [
    "column_filters = {\n",
    "    'Age': (18, 90),\n",
    "}\n",
    "\n",
    "for column, (min_val, max_val) in column_filters.items():\n",
    "    if column in x_test.columns:\n",
    "        x_test = x_test[(x_test[column] > min_val) & (x_test[column] < max_val)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "jLWR5lmo01KM"
   },
   "outputs": [],
   "source": [
    "pyplot.rcParams[\"figure.figsize\"] = (15, 15)\n",
    "pyplot.rcParams[\"figure.dpi\"] = 300\n",
    "pyplot.rcParams['axes.labelweight'] = 'bold'\n",
    "pyplot.rcParams['axes.labelsize'] = 12\n",
    "pyplot.rcParams['axes.labelpad'] = 6\n",
    "pyplot.rcParams['font.weight'] = 'normal'\n",
    "pyplot.rcParams['lines.linewidth'] = 2.5\n",
    "pyplot.rcParams['xtick.labelsize'] = 8\n",
    "pyplot.rcParams['ytick.labelsize'] = 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "4mCUeepoYdNJ"
   },
   "outputs": [],
   "source": [
    "categorical_features = ['Facility Type', 'Facility Location', 'Sex', 'Race', 'Hispanic Ethnicity', 'Histology',\n",
    "                        'Primary Payor', 'Percent No High School Education Quartiles', 'Census Median Income Quartiles',\n",
    "                        'Rurality', 'Charlson-Deyo Score', 'Tumor Localization', 'Laterality', 'Grade',\n",
    "                        'Diagnostic Biopsy', 'Ki-67 Labeling Index', 'Karnofsky Performance Scale',\n",
    "                        'MGMT Methylation', 'Focality', 'Tumor Extension', 'Tumor Size', 'Radiation Treatment-Surgery Sequence',\n",
    "                        'Chemotherapy', 'Immunotherapy', 'Systemic Therapy-Surgery Sequence', 'Extent of Resection',\n",
    "                        'Resective Surgery', 'Radiation Treatment', '1p19q Co-Deletion', 'Histology']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "UNB6VbOg5KKx"
   },
   "outputs": [],
   "source": [
    "#Calculate the mean absolute SHAP values for each feature.\n",
    "tabpfn_mean_abs_shap_values = np.mean(np.abs(tabpfn_shap_values.values), axis=0)\n",
    "\n",
    "#Create a DataFrame to map feature names to their mean absolute SHAP values.\n",
    "tabpfn_shap_summary = pd.DataFrame(list(zip(feature_names, tabpfn_mean_abs_shap_values)), columns=['Feature', 'Mean SHAP'])\n",
    "\n",
    "#Sort the DataFrame by 'Mean SHAP' in descending order.\n",
    "tabpfn_shap_summary_sorted = tabpfn_shap_summary.sort_values('Mean SHAP', ascending=False)\n",
    "\n",
    "#Get the names of the features.\n",
    "tabpfn_features = tabpfn_shap_summary_sorted['Feature'].tolist()\n",
    "tabpfn_features = tabpfn_features[:9]\n",
    "\n",
    "tabpfn_categorical_features = [item for item in tabpfn_features if item in categorical_features]\n",
    "\n",
    "PartialDependenceDisplay.from_estimator(tabpfn, x_test.sample(frac=0.05, random_state=42), tabpfn_features, categorical_features = tabpfn_categorical_features)\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g2_36m_pdp_tabpfn.png', dpi=300, bbox_inches='tight')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "bLDT7TOZ5JFi"
   },
   "outputs": [],
   "source": [
    "#Calculate the mean absolute SHAP values for each feature.\n",
    "tabnet_mean_abs_shap_values = np.mean(np.abs(tabnet_shap_values.values), axis=0)\n",
    "\n",
    "#Create a DataFrame to map feature names to their mean absolute SHAP values.\n",
    "tabnet_shap_summary = pd.DataFrame(list(zip(feature_names, tabnet_mean_abs_shap_values)), columns=['Feature', 'Mean SHAP'])\n",
    "\n",
    "#Sort the DataFrame by 'Mean SHAP' in descending order.\n",
    "tabnet_shap_summary_sorted = tabnet_shap_summary.sort_values('Mean SHAP', ascending=False)\n",
    "\n",
    "#Get the names of the features.\n",
    "tabnet_features = tabnet_shap_summary_sorted['Feature'].tolist()\n",
    "tabnet_features = tabnet_features[:9]\n",
    "\n",
    "tabnet_categorical_features = [item for item in tabnet_features if item in categorical_features]\n",
    "\n",
    "PartialDependenceDisplay.from_estimator(xgb, x_test, tabnet_features, categorical_features = tabnet_categorical_features)\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g2_36m_pdp_tabnet.png', dpi=300, bbox_inches='tight')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "frN2YQUD0IWP"
   },
   "outputs": [],
   "source": [
    "#Calculate the mean absolute SHAP values for each feature.\n",
    "xgb_mean_abs_shap_values = np.mean(np.abs(xgb_shap_values.values), axis=0)\n",
    "\n",
    "#Create a DataFrame to map feature names to their mean absolute SHAP values.\n",
    "xgb_shap_summary = pd.DataFrame(list(zip(feature_names, xgb_mean_abs_shap_values)), columns=['Feature', 'Mean SHAP'])\n",
    "\n",
    "#Sort the DataFrame by 'Mean SHAP' in descending order.\n",
    "xgb_shap_summary_sorted = xgb_shap_summary.sort_values('Mean SHAP', ascending=False)\n",
    "\n",
    "#Get the names of the features.\n",
    "xgb_features = xgb_shap_summary_sorted['Feature'].tolist()\n",
    "xgb_features = xgb_features[:9]\n",
    "\n",
    "xgb_categorical_features = [item for item in xgb_features if item in categorical_features]\n",
    "\n",
    "PartialDependenceDisplay.from_estimator(xgb, x_test, xgb_features, categorical_features = xgb_categorical_features)\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g2_36m_pdp_xgb.png', dpi=300, bbox_inches='tight')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "P9ZSSzpP5MbC"
   },
   "outputs": [],
   "source": [
    "#Calculate the mean absolute SHAP values for each feature.\n",
    "lgb_mean_abs_shap_values = np.mean(np.abs(lgb_shap_values.values), axis=0)\n",
    "\n",
    "#Create a DataFrame to map feature names to their mean absolute SHAP values.\n",
    "lgb_shap_summary = pd.DataFrame(list(zip(feature_names, lgb_mean_abs_shap_values)), columns=['Feature', 'Mean SHAP'])\n",
    "\n",
    "#Sort the DataFrame by 'Mean SHAP' in descending order.\n",
    "lgb_shap_summary_sorted = lgb_shap_summary.sort_values('Mean SHAP', ascending=False)\n",
    "\n",
    "#Get the names of the features.\n",
    "lgb_features = lgb_shap_summary_sorted['Feature'].tolist()\n",
    "lgb_features = lgb_features[:9]\n",
    "\n",
    "lgb_categorical_features = [item for item in lgb_features if item in categorical_features]\n",
    "\n",
    "PartialDependenceDisplay.from_estimator(lgb, x_test, lgb_features, categorical_features = lgb_categorical_features)\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g2_36m_pdp_lgb.png', dpi=300, bbox_inches='tight')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "aq_m3WU_5NKq"
   },
   "outputs": [],
   "source": [
    "#Calculate the mean absolute SHAP values for each feature.\n",
    "rf_mean_abs_shap_values = np.mean(np.abs(rf_shap_values.values), axis=0)\n",
    "\n",
    "#Create a DataFrame to map feature names to their mean absolute SHAP values.\n",
    "rf_shap_summary = pd.DataFrame(list(zip(feature_names, rf_mean_abs_shap_values)), columns=['Feature', 'Mean SHAP'])\n",
    "\n",
    "#Sort the DataFrame by 'Mean SHAP' in descending order.\n",
    "rf_shap_summary_sorted = rf_shap_summary.sort_values('Mean SHAP', ascending=False)\n",
    "\n",
    "#Get the names of the features.\n",
    "rf_features = rf_shap_summary_sorted['Feature'].tolist()\n",
    "rf_features = rf_features[:9]\n",
    "\n",
    "rf_categorical_features = [item for item in rf_features if item in categorical_features]\n",
    "\n",
    "PartialDependenceDisplay.from_estimator(rf, x_test, rf_features, categorical_features = rf_categorical_features)\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g2_36m_pdp_rf.png', dpi=300, bbox_inches='tight')\n",
    "pyplot.show()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "machine_shape": "hm",
   "name": "",
   "provenance": [
    {
     "file_id": "1YsXYFGUOBoaH2tWCgj1lljBtMN8iwzxH",
     "timestamp": 1691626231904
    },
    {
     "file_id": "1M02d2-HBRrt5xmA-5x863g67zB9zloB3",
     "timestamp": 1691620548048
    },
    {
     "file_id": "14TfLVbewZLcEADsU9RM-JZ4Jrc-obgN-",
     "timestamp": 1691610327686
    },
    {
     "file_id": "1Eo4jPVLNOwgUyqgXVFTcpiRbGrIuXwk5",
     "timestamp": 1690313290635
    },
    {
     "file_id": "1ehzQDERWA7oATVlIze0f5fvBuuSk3Il-",
     "timestamp": 1690235650728
    },
    {
     "file_id": "10kjTiT6ZbSYDMQVY6jE58AJ9bNIvu_Z4",
     "timestamp": 1680804894233
    },
    {
     "file_id": "1SyxRVArbYdS2kIxHOoZLuxtWYNsZL3ei",
     "timestamp": 1675315289522
    },
    {
     "file_id": "1rquc9s2K5gcWtyx-P9n0ie3ooY4kzveD",
     "timestamp": 1673820526724
    },
    {
     "file_id": "1sd9nBAoznz2E976naQ2pJvARRLH-vRK7",
     "timestamp": 1673714326538
    },
    {
     "file_id": "1tydPpyRVpKMFnUtRbvvIdQTTUtRQ1SJK",
     "timestamp": 1672252113955
    },
    {
     "file_id": "1FJ2Tp28HBJo_krRjesYEkutbjOcsodmu",
     "timestamp": 1672251686516
    },
    {
     "file_id": "1vmOc15wf8HLrh9xcXe9IRvqZIrBNUFp_",
     "timestamp": 1671647832509
    },
    {
     "file_id": "1jvG_2brh3p2BhjDqh6LIyEOlVCFKYSX6",
     "timestamp": 1671034103968
    },
    {
     "file_id": "1YKiMGVsPQlDRLgB2d3Svjjsp96BkJxlX",
     "timestamp": 1670597416903
    },
    {
     "file_id": "1gG7ScMqQA2JnmntazVsJaQStNW3qHl9R",
     "timestamp": 1670198888608
    },
    {
     "file_id": "1V4GD1xQ2LBQhBny_OS3iBlGyn9mh7oWM",
     "timestamp": 1669993880849
    },
    {
     "file_id": "1euhY_qXOTy9qT_vn0Q6Kb2eEUOCHom8m",
     "timestamp": 1669310009337
    },
    {
     "file_id": "1ksEkZ_gSIaosRd56LkYx54z5oGNBgq2C",
     "timestamp": 1668779215067
    },
    {
     "file_id": "1o0f_u5loubhXZTvFpGUCA_m2gpTKQ6EL",
     "timestamp": 1668619772114
    }
   ],
   "toc_visible": true,
   "version": ""
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}