NCDB_G3_Glioma_12M_Survival.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 20576,
     "status": "ok",
     "timestamp": 1691674708042,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "RxtJ9G-aIZ_y",
    "outputId": "19e70969-104d-4524-827a-1d049637dd06"
   },
   "outputs": [],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 23195,
     "status": "ok",
     "timestamp": 1691674731223,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "h_PnaQ4NIuE0",
    "outputId": "323c9bb3-7113-440f-b8a8-ff96097ee8ae"
   },
   "outputs": [],
   "source": [
    "!pip install optuna\n",
    "!pip install shap\n",
    "!pip install pytorch-tabnet\n",
    "!pip install tabpfn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "JrZMQLTfIQPb"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import sklearn\n",
    "import sklearn.metrics\n",
    "from sklearn.metrics import roc_auc_score, roc_curve, precision_recall_curve, average_precision_score, auc, precision_score, recall_score, accuracy_score, balanced_accuracy_score, brier_score_loss, confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.utils import resample\n",
    "from math import sqrt\n",
    "from scipy import stats as st\n",
    "from random import randrange\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "\n",
    "from tabpfn import TabPFNClassifier\n",
    "from pytorch_tabnet.tab_model import TabNetClassifier\n",
    "import xgboost as xgb\n",
    "import lightgbm as lgb\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "import optuna\n",
    "from optuna.samplers import TPESampler\n",
    "\n",
    "import shap\n",
    "from sklearn.inspection import PartialDependenceDisplay"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UWbpPFMVHS-V"
   },
   "source": [
    "# Preparing Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "sDsi_6tnIQPc"
   },
   "outputs": [],
   "source": [
    "#Open csv file.\n",
    "\n",
    "data = pd.read_csv('/content/drive/MyDrive/NCDB-G2G3 Glioma/final_data_g3.csv', index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 236,
     "status": "ok",
     "timestamp": 1691674745374,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "hPmUaNyUN2p8",
    "outputId": "49012b99-6910-4d1e-a773-385a9e288746"
   },
   "outputs": [],
   "source": [
    "#Define outcome of interest (3-month mortality).\n",
    "\n",
    "data.loc[(data['Vital Status'] == 'Dead') & (data['Last Contact or Death (Months from Diagnosis)'] < 12), 'OUTCOME'] = 1\n",
    "data.loc[(data['Vital Status'] == 'Dead') & (data['Last Contact or Death (Months from Diagnosis)'] >= 12), 'OUTCOME'] = 0\n",
    "data.loc[(data['Vital Status'] == 'Alive') & (data['Last Contact or Death (Months from Diagnosis)'] >= 12), 'OUTCOME'] = 0\n",
    "data = data.dropna(subset=['OUTCOME'])\n",
    "\n",
    "data['OUTCOME'].value_counts(normalize=False, dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "LhZeK6d8IQPd"
   },
   "outputs": [],
   "source": [
    "#Define predictor variables (x) and outcome of interest (y).\n",
    "\n",
    "outcomes = ['Vital Status', 'Last Contact or Death (Months from Diagnosis)', 'OUTCOME']\n",
    "\n",
    "x = data.drop(outcomes, axis = 1)\n",
    "y = data['OUTCOME']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 21,
     "status": "ok",
     "timestamp": 1691674745375,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "mt6ks1daIQPd",
    "outputId": "f7604141-be26-4676-fd0b-4386d7c8de10"
   },
   "outputs": [],
   "source": [
    "#Check data shapes.\n",
    "\n",
    "print(y.shape)\n",
    "print(x.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 20,
     "status": "ok",
     "timestamp": 1691674745375,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "d0IwUyCsIQPe",
    "outputId": "5e14988f-c597-4d15-dcb7-b7e8a3b850f7"
   },
   "outputs": [],
   "source": [
    "#Split data into initial train set and test set in 80:20 ratio.\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.2, random_state = 0)\n",
    "\n",
    "#Describe initial train set and test set.\n",
    "\n",
    "print(\"Number patients x_train dataset: \", x_train.shape[0])\n",
    "print(\"Number patients y_train dataset: \", y_train.shape[0])\n",
    "print(\"Number patients x_test dataset: \", x_test.shape[0])\n",
    "print(\"Number patients y_test dataset: \", y_test.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 14,
     "status": "ok",
     "timestamp": 1691674745375,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "WyNo7C2Pbzt2",
    "outputId": "69759939-5fe2-4c40-8ad8-e9fc3e5e19b5"
   },
   "outputs": [],
   "source": [
    "#Split initial train set into final train set and validation set in 75:25 ratio.\n",
    "\n",
    "train_x, valid_x, train_y, valid_y = train_test_split(x_train, y_train, test_size = 0.25, random_state = 0)\n",
    "\n",
    "#Describe train and validation sets.\n",
    "\n",
    "print(\"Number patients train_x dataset: \", train_x.shape[0])\n",
    "print(\"Number patients train_y dataset: \", train_y.shape[0])\n",
    "print(\"Number patients valid_x dataset: \", valid_x.shape[0])\n",
    "print(\"Number patients valid_y dataset: \", valid_y.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 10,
     "status": "ok",
     "timestamp": 1691674745375,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "g2O0He35IQPe",
    "outputId": "1a515dc3-fd0d-42b8-986e-3a30a849041c"
   },
   "outputs": [],
   "source": [
    "#Describe outcome of interest before resampling.\n",
    "\n",
    "print(\"Before resampling, counts of label '1': {}\".format(sum(train_y == 1)))\n",
    "print(\"Before resampling, counts of label '0': {} \\n\".format(sum(train_y == 0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "QvXczViaIQPe"
   },
   "outputs": [],
   "source": [
    "#Apply SMOTE.\n",
    "\n",
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "resampler = SMOTE(random_state = 0)\n",
    "train_x, train_y = resampler.fit_resample(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 8,
     "status": "ok",
     "timestamp": 1691674745985,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "mYw68tzFIQPe",
    "outputId": "f981916b-3620-4df5-da6f-8953ae78e003"
   },
   "outputs": [],
   "source": [
    "#Describe outcome of interest after resampling.\n",
    "\n",
    "print(\"After resampling, counts of label '1': {}\".format(sum(train_y == 1)))\n",
    "print(\"After resampling, counts of label '0': {} \\n\".format(sum(train_y == 0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lrb8PN0dEOc1"
   },
   "outputs": [],
   "source": [
    "#Define function for AUROC with 95% confidence intervals.\n",
    "\n",
    "def auroc_ci(y_test, y_probs, positive=1, n_samples=1000, alpha=0.05):\n",
    "    auroc_values = []\n",
    "    for i in range(n_samples):\n",
    "        y_test_bs, y_probs_bs = resample(y_test, y_probs)\n",
    "\n",
    "        auroc = roc_auc_score(y_test_bs, y_probs_bs)\n",
    "        auroc_values.append(auroc)\n",
    "\n",
    "    mean_auroc = np.mean(auroc_values)\n",
    "    std_auroc = np.std(auroc_values)\n",
    "\n",
    "    lower_ci = mean_auroc - 1.96 * std_auroc\n",
    "    upper_ci = mean_auroc + 1.96 * std_auroc\n",
    "\n",
    "    return auroc, lower_ci, upper_ci"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "BLkyBrpf95-0"
   },
   "outputs": [],
   "source": [
    "#Define z-value for other confidence intervals.\n",
    "\n",
    "confidence = 0.95\n",
    "z_value = st.norm.ppf((1 + confidence) / 2.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UWX_CriXYlL_"
   },
   "source": [
    "#TabPFN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "OFQmwskRLrY4"
   },
   "source": [
    "No hyperparameter tuning is performed for TabPFN since the paper that introduced TabPFN claims no hyperparameter tuning is needed for it:\n",
    "\n",
    "\n",
    "*We present TabPFN, a trained Transformer that can do supervised classification for small tabular datasets in less than a second, needs no hyperparameter tuning and is competitive with state-of-the-art classification methods.*\n",
    "\n",
    "https://doi.org/10.48550/arXiv.2207.01848\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 142
    },
    "executionInfo": {
     "elapsed": 3808,
     "status": "ok",
     "timestamp": 1691674749788,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "F3gvvODJ8B2Z",
    "outputId": "8ebeb20b-2acc-467e-a75c-5787259abd17"
   },
   "outputs": [],
   "source": [
    "#Fit TabPFN.\n",
    "\n",
    "tabpfn = TabPFNClassifier(device='cpu', N_ensemble_configurations=2)\n",
    "\n",
    "tabpfn.fit(x_train.sample(n=1024, random_state=42), y_train.sample(n=1024, random_state=42), overwrite_warning = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 74
    },
    "executionInfo": {
     "elapsed": 11833,
     "status": "ok",
     "timestamp": 1691674761509,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "5oLbONin8YGA",
    "outputId": "332c340c-903e-432e-8528-ff17b25ab5f4"
   },
   "outputs": [],
   "source": [
    "#Predict on the validation set, get predicted probabilities for calibration, and fit calibration function.\n",
    "\n",
    "calib_probs_tabpfn = tabpfn.predict_proba(valid_x.values)\n",
    "\n",
    "calib_model_tabpfn = LogisticRegression()\n",
    "calib_model_tabpfn.fit(calib_probs_tabpfn, valid_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 19519,
     "status": "ok",
     "timestamp": 1691674781022,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "UZNInXD39Tux",
    "outputId": "90861308-6331-4b78-f5ff-7b42105c1dd6"
   },
   "outputs": [],
   "source": [
    "#Make predictions on the test set based on the trained TabNet model.\n",
    "\n",
    "preds_tabpfn = tabpfn.predict(x_test.values)\n",
    "\n",
    "uncalibrated_probs_tabpfn = tabpfn.predict_proba(x_test.values)\n",
    "\n",
    "probs_tabpfn = calib_model_tabpfn.predict_proba(uncalibrated_probs_tabpfn)\n",
    "probs_tabpfn = probs_tabpfn[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 3095,
     "status": "ok",
     "timestamp": 1691674784104,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "Sm8PNeSm9dtI",
    "outputId": "2cc80c20-8f95-4906-b5f0-f60ddc921c34"
   },
   "outputs": [],
   "source": [
    "#Evaluate TabPFN model.\n",
    "\n",
    "tabpfn_tn, tabpfn_fp, tabpfn_fn, tabpfn_tp = confusion_matrix(y_test, preds_tabpfn).ravel()\n",
    "tabpfn_cm = confusion_matrix(y_test, preds_tabpfn)\n",
    "\n",
    "tabpfn_sensitivity = round(tabpfn_tp/(tabpfn_tp+tabpfn_fn), 3)\n",
    "tabpfn_sensitivity_ci_length = z_value * np.sqrt((tabpfn_sensitivity * (1 - tabpfn_sensitivity)) / y_test.shape[0])\n",
    "tabpfn_sensitivity_ci_lower = round((tabpfn_sensitivity - tabpfn_sensitivity_ci_length), 3)\n",
    "tabpfn_sensitivity_ci_upper = round((tabpfn_sensitivity + tabpfn_sensitivity_ci_length), 3)\n",
    "tabpfn_sensitivity_str = str(tabpfn_sensitivity) + ' (' + str(tabpfn_sensitivity_ci_lower) + ' - ' + str(tabpfn_sensitivity_ci_upper) + ')'\n",
    "\n",
    "tabpfn_specificity = round(tabpfn_tn/(tabpfn_tn+tabpfn_fp), 3)\n",
    "tabpfn_specificity_ci_length = z_value * np.sqrt((tabpfn_specificity * (1 - tabpfn_specificity)) / y_test.shape[0])\n",
    "tabpfn_specificity_ci_lower = round((tabpfn_specificity - tabpfn_specificity_ci_length), 3)\n",
    "tabpfn_specificity_ci_upper = round((tabpfn_specificity + tabpfn_specificity_ci_length), 3)\n",
    "tabpfn_specificity_str = str(tabpfn_specificity) + ' (' + str(tabpfn_specificity_ci_lower) + ' - ' + str(tabpfn_specificity_ci_upper) + ')'\n",
    "\n",
    "tabpfn_auprc = round(average_precision_score(y_test, probs_tabpfn), 3)\n",
    "tabpfn_auprc_ci_length = z_value * np.sqrt((tabpfn_auprc * (1 - tabpfn_auprc)) / y_test.shape[0])\n",
    "tabpfn_auprc_ci_lower = round((tabpfn_auprc - tabpfn_auprc_ci_length), 3)\n",
    "tabpfn_auprc_ci_upper = round((tabpfn_auprc + tabpfn_auprc_ci_length), 3)\n",
    "tabpfn_auprc_str = str(tabpfn_auprc) + ' (' + str(tabpfn_auprc_ci_lower) + ' - ' + str(tabpfn_auprc_ci_upper) + ')'\n",
    "\n",
    "tabpfn_accuracy = round(accuracy_score(y_test, preds_tabpfn), 3)\n",
    "tabpfn_accuracy_ci_length = z_value * np.sqrt((tabpfn_accuracy * (1 - tabpfn_accuracy)) / y_test.shape[0])\n",
    "tabpfn_accuracy_ci_lower = round((tabpfn_accuracy - tabpfn_accuracy_ci_length), 3)\n",
    "tabpfn_accuracy_ci_upper = round((tabpfn_accuracy + tabpfn_accuracy_ci_length), 3)\n",
    "tabpfn_accuracy_str = str(tabpfn_accuracy) + ' (' + str(tabpfn_accuracy_ci_lower) + ' - ' + str(tabpfn_accuracy_ci_upper) + ')'\n",
    "\n",
    "tabpfn_auroc, tabpfn_auroc_ci_lower, tabpfn_auroc_ci_upper = auroc_ci(y_test, probs_tabpfn)\n",
    "tabpfn_auroc = round(tabpfn_auroc, 3)\n",
    "tabpfn_auroc_ci_lower = round(tabpfn_auroc_ci_lower, 3)\n",
    "tabpfn_auroc_ci_upper = round(tabpfn_auroc_ci_upper, 3)\n",
    "tabpfn_auroc_str = str(tabpfn_auroc) + ' (' + str(tabpfn_auroc_ci_lower) + ' - ' + str(tabpfn_auroc_ci_upper) + ')'\n",
    "\n",
    "tabpfn_brier = round(brier_score_loss(y_test, probs_tabpfn), 3)\n",
    "tabpfn_brier_ci_length = z_value * np.sqrt((tabpfn_brier * (1 - tabpfn_brier)) / y_test.shape[0])\n",
    "tabpfn_brier_ci_lower = round((tabpfn_brier - tabpfn_brier_ci_length), 3)\n",
    "tabpfn_brier_ci_upper = round((tabpfn_brier + tabpfn_brier_ci_length), 3)\n",
    "tabpfn_brier_str = str(tabpfn_brier) + ' (' + str(tabpfn_brier_ci_lower) + ' - ' + str(tabpfn_brier_ci_upper) + ')'\n",
    "\n",
    "tabpfn_results = [tabpfn_sensitivity_str, tabpfn_specificity_str, tabpfn_auprc_str, tabpfn_accuracy_str, tabpfn_auroc_str, tabpfn_brier_str]\n",
    "\n",
    "print(\"Sensitivity: \", (tabpfn_sensitivity_str))\n",
    "print(\"Specificity: \", (tabpfn_specificity_str))\n",
    "print('AUPRC: ', (tabpfn_auprc_str))\n",
    "print('Accuracy: ', (tabpfn_accuracy_str))\n",
    "print('AUROC: ', (tabpfn_auroc_str))\n",
    "print('Brier Score: ', (tabpfn_brier_str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 13,
     "status": "ok",
     "timestamp": 1691674784105,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "0wp5veHVFWIB",
    "outputId": "f2fec721-eb11-4b3c-b46d-33c2fcf1eb08"
   },
   "outputs": [],
   "source": [
    "#Recalculate precision recall curve for plotting purposes.\n",
    "\n",
    "tabpfn_precision_curve, tabpfn_recall_curve, _ = precision_recall_curve(y_test, probs_tabpfn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BY44IPIU7sBi"
   },
   "source": [
    "# TabNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 211941,
     "status": "ok",
     "timestamp": 1691674996037,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "beooviDI7uuS",
    "outputId": "3b912a6c-0ac4-4f49-a6c8-79b939cda544"
   },
   "outputs": [],
   "source": [
    "# Hyperparameter tuning for TabNet.\n",
    "\n",
    "def objective(trial):\n",
    "\n",
    "    param = {\n",
    "        \"n_d\": trial.suggest_int(\"n_d\", 8, 64),\n",
    "        \"n_a\": trial.suggest_int(\"n_a\", 8, 64),\n",
    "        \"n_steps\": trial.suggest_int(\"n_steps\", 1, 10),\n",
    "        \"gamma\": trial.suggest_float(\"gamma\", 1.0, 2.0),\n",
    "        \"n_independent\": trial.suggest_int(\"n_independent\", 1, 5),\n",
    "        \"n_shared\": trial.suggest_int(\"n_shared\", 1, 5),\n",
    "        \"lambda_sparse\": trial.suggest_loguniform(\"lambda_sparse\", 1e-6, 1e-3),\n",
    "        \"optimizer_params\": {\"lr\": trial.suggest_loguniform(\"lr\", 1e-5, 1e-2)},\n",
    "        \"seed\": 31,\n",
    "    }\n",
    "\n",
    "    tabnet_clf = TabNetClassifier(**param)\n",
    "\n",
    "    tabnet_clf.fit(\n",
    "        train_x.values,\n",
    "        train_y.values,\n",
    "        eval_set=[(valid_x.values, valid_y.values)],\n",
    "        eval_metric=[\"auc\"],\n",
    "        max_epochs=10,\n",
    "        patience=20,\n",
    "        batch_size=1024,\n",
    "        virtual_batch_size=128,\n",
    "        num_workers=0,\n",
    "        drop_last=False,\n",
    "    )\n",
    "\n",
    "    preds = tabnet_clf.predict_proba(valid_x.values)[:, 1]\n",
    "    pred_labels = np.rint(preds)\n",
    "    auc = sklearn.metrics.roc_auc_score(valid_y, pred_labels)\n",
    "\n",
    "    return auc\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    study = optuna.create_study(direction='maximize', sampler=TPESampler(seed=31))\n",
    "    study.optimize(objective, n_trials=10, timeout=600)\n",
    "\n",
    "    print(\"Number of finished trials: {}\".format(len(study.trials)))\n",
    "\n",
    "    print(\"Best trial:\")\n",
    "    trial = study.best_trial\n",
    "\n",
    "    print(\"  Value: {}\".format(trial.value))\n",
    "\n",
    "    print(\"  Params: \")\n",
    "    for key, value in trial.params.items():\n",
    "        print(\"    {}: {}\".format(key, value))\n",
    "\n",
    "    tabnet_params = {}\n",
    "\n",
    "    for key, value in trial.params.items():\n",
    "        tabnet_params[key] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 52,
     "status": "ok",
     "timestamp": 1691674996039,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "gXsTkX3n_i2y",
    "outputId": "e0cb38a3-5e14-4097-a512-6ff9e1f26161"
   },
   "outputs": [],
   "source": [
    "#See TabNet hyperparameters.\n",
    "\n",
    "tabnet_params.pop('lr')\n",
    "\n",
    "print(tabnet_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 73332,
     "status": "ok",
     "timestamp": 1691675069351,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "pujw5iMQ_i2y",
    "outputId": "d93d87eb-2c6d-4978-ac43-7b4b863f2e9c"
   },
   "outputs": [],
   "source": [
    "#Fit TabNet.\n",
    "\n",
    "tabnet = TabNetClassifier(**tabnet_params)\n",
    "\n",
    "tabnet.fit(train_x.values, train_y.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 91
    },
    "executionInfo": {
     "elapsed": 308,
     "status": "ok",
     "timestamp": 1691675069633,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "xYiPND2v_i2y",
    "outputId": "6e0922e3-58c6-4258-b130-440120c8fda6"
   },
   "outputs": [],
   "source": [
    "#Predict on the validation set, get predicted probabilities for calibration, and fit calibration function.\n",
    "\n",
    "calib_probs_tabnet = tabnet.predict_proba(valid_x.values)\n",
    "\n",
    "calib_model_tabnet = LogisticRegression()\n",
    "calib_model_tabnet.fit(calib_probs_tabnet, valid_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 226,
     "status": "ok",
     "timestamp": 1691675069760,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "y1AJq9eA_i2y",
    "outputId": "7c0745c5-b3c3-4577-dddf-7817db75ac0b"
   },
   "outputs": [],
   "source": [
    "#Make predictions on the test set based on the trained TabNet model.\n",
    "\n",
    "preds_tabnet = tabnet.predict(x_test.values)\n",
    "\n",
    "uncalibrated_probs_tabnet = tabnet.predict_proba(x_test.values)\n",
    "\n",
    "probs_tabnet = calib_model_tabnet.predict_proba(uncalibrated_probs_tabnet)\n",
    "probs_tabnet = probs_tabnet[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2558,
     "status": "ok",
     "timestamp": 1691675072315,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "AP-y4lEI_i2z",
    "outputId": "9269093b-5b51-4225-b1eb-d100edf7c583"
   },
   "outputs": [],
   "source": [
    "#Evaluate TabNet model.\n",
    "\n",
    "tabnet_tn, tabnet_fp, tabnet_fn, tabnet_tp = confusion_matrix(y_test, preds_tabnet).ravel()\n",
    "tabnet_cm = confusion_matrix(y_test, preds_tabnet)\n",
    "\n",
    "tabnet_sensitivity = round(tabnet_tp/(tabnet_tp+tabnet_fn), 3)\n",
    "tabnet_sensitivity_ci_length = z_value * np.sqrt((tabnet_sensitivity * (1 - tabnet_sensitivity)) / y_test.shape[0])\n",
    "tabnet_sensitivity_ci_lower = round((tabnet_sensitivity - tabnet_sensitivity_ci_length), 3)\n",
    "tabnet_sensitivity_ci_upper = round((tabnet_sensitivity + tabnet_sensitivity_ci_length), 3)\n",
    "tabnet_sensitivity_str = str(tabnet_sensitivity) + ' (' + str(tabnet_sensitivity_ci_lower) + ' - ' + str(tabnet_sensitivity_ci_upper) + ')'\n",
    "\n",
    "tabnet_specificity = round(tabnet_tn/(tabnet_tn+tabnet_fp), 3)\n",
    "tabnet_specificity_ci_length = z_value * np.sqrt((tabnet_specificity * (1 - tabnet_specificity)) / y_test.shape[0])\n",
    "tabnet_specificity_ci_lower = round((tabnet_specificity - tabnet_specificity_ci_length), 3)\n",
    "tabnet_specificity_ci_upper = round((tabnet_specificity + tabnet_specificity_ci_length), 3)\n",
    "tabnet_specificity_str = str(tabnet_specificity) + ' (' + str(tabnet_specificity_ci_lower) + ' - ' + str(tabnet_specificity_ci_upper) + ')'\n",
    "\n",
    "tabnet_auprc = round(average_precision_score(y_test, probs_tabnet), 3)\n",
    "tabnet_auprc_ci_length = z_value * np.sqrt((tabnet_auprc * (1 - tabnet_auprc)) / y_test.shape[0])\n",
    "tabnet_auprc_ci_lower = round((tabnet_auprc - tabnet_auprc_ci_length), 3)\n",
    "tabnet_auprc_ci_upper = round((tabnet_auprc + tabnet_auprc_ci_length), 3)\n",
    "tabnet_auprc_str = str(tabnet_auprc) + ' (' + str(tabnet_auprc_ci_lower) + ' - ' + str(tabnet_auprc_ci_upper) + ')'\n",
    "\n",
    "tabnet_accuracy = round(accuracy_score(y_test, preds_tabnet), 3)\n",
    "tabnet_accuracy_ci_length = z_value * np.sqrt((tabnet_accuracy * (1 - tabnet_accuracy)) / y_test.shape[0])\n",
    "tabnet_accuracy_ci_lower = round((tabnet_accuracy - tabnet_accuracy_ci_length), 3)\n",
    "tabnet_accuracy_ci_upper = round((tabnet_accuracy + tabnet_accuracy_ci_length), 3)\n",
    "tabnet_accuracy_str = str(tabnet_accuracy) + ' (' + str(tabnet_accuracy_ci_lower) + ' - ' + str(tabnet_accuracy_ci_upper) + ')'\n",
    "\n",
    "tabnet_auroc, tabnet_auroc_ci_lower, tabnet_auroc_ci_upper = auroc_ci(y_test, probs_tabnet)\n",
    "tabnet_auroc = round(tabnet_auroc, 3)\n",
    "tabnet_auroc_ci_lower = round(tabnet_auroc_ci_lower, 3)\n",
    "tabnet_auroc_ci_upper = round(tabnet_auroc_ci_upper, 3)\n",
    "tabnet_auroc_str = str(tabnet_auroc) + ' (' + str(tabnet_auroc_ci_lower) + ' - ' + str(tabnet_auroc_ci_upper) + ')'\n",
    "\n",
    "tabnet_brier = round(brier_score_loss(y_test, probs_tabnet), 3)\n",
    "tabnet_brier_ci_length = z_value * np.sqrt((tabnet_brier * (1 - tabnet_brier)) / y_test.shape[0])\n",
    "tabnet_brier_ci_lower = round((tabnet_brier - tabnet_brier_ci_length), 3)\n",
    "tabnet_brier_ci_upper = round((tabnet_brier + tabnet_brier_ci_length), 3)\n",
    "tabnet_brier_str = str(tabnet_brier) + ' (' + str(tabnet_brier_ci_lower) + ' - ' + str(tabnet_brier_ci_upper) + ')'\n",
    "\n",
    "tabnet_results = [tabnet_sensitivity_str, tabnet_specificity_str, tabnet_auprc_str, tabnet_accuracy_str, tabnet_auroc_str, tabnet_brier_str]\n",
    "\n",
    "print(\"Sensitivity: \", (tabnet_sensitivity_str))\n",
    "print(\"Specificity: \", (tabnet_specificity_str))\n",
    "print('AUPRC: ', (tabnet_auprc_str))\n",
    "print('Accuracy: ', (tabnet_accuracy_str))\n",
    "print('AUROC: ', (tabnet_auroc_str))\n",
    "print('Brier Score: ', (tabnet_brier_str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 16,
     "status": "ok",
     "timestamp": 1691675072315,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "tsAsKQ1e_i2z",
    "outputId": "2c441618-e66f-4105-e658-157924ea61fe"
   },
   "outputs": [],
   "source": [
    "#Recalculate precision recall curve for plotting purposes.\n",
    "\n",
    "tabnet_precision_curve, tabnet_recall_curve, _ = precision_recall_curve(y_test, probs_tabnet)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "esuV5nR8IQPe"
   },
   "source": [
    "# XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 13482,
     "status": "ok",
     "timestamp": 1691675085789,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "dBWNs_MxIQPf",
    "outputId": "e03c92e7-7094-4980-e195-392e375ab015"
   },
   "outputs": [],
   "source": [
    "#Hyperparameter tuning for XGBoost.\n",
    "\n",
    "def objective(trial):\n",
    "\n",
    "    dtrain = xgb.DMatrix(train_x, label=train_y)\n",
    "    dvalid = xgb.DMatrix(valid_x, label=valid_y)\n",
    "\n",
    "    param = {\n",
    "        \"seed\": 31,\n",
    "        \"verbosity\": 0,\n",
    "        \"objective\":  trial.suggest_categorical(\"objective\", [\"binary:logistic\"]),\n",
    "        \"eval_metric\": \"auc\",\n",
    "        \"booster\": trial.suggest_categorical(\"booster\", [\"gbtree\"]),\n",
    "        \"lambda\": trial.suggest_float(\"lambda\", 1e-8, 1.0, log=True),\n",
    "        \"alpha\": trial.suggest_float(\"alpha\", 1e-8, 1.0, log=True),\n",
    "        \"max_depth\" : trial.suggest_int(\"max_depth\", 1, 9),\n",
    "        \"eta\" : trial.suggest_float(\"eta\", 1e-8, 1.0, log=True),\n",
    "        \"gamma\" : trial.suggest_float(\"gamma\", 1e-8, 1.0, log=True),\n",
    "        \"grow_policy\" : trial.suggest_categorical(\"grow_policy\", [\"depthwise\", \"lossguide\"])\n",
    "    }\n",
    "\n",
    "\n",
    "    # Add a callback for pruning.\n",
    "    pruning_callback = optuna.integration.XGBoostPruningCallback(trial, \"validation-auc\")\n",
    "\n",
    "    bst = xgb.train(param, dtrain, evals=[(dvalid, \"validation\")], callbacks=[pruning_callback])\n",
    "    preds = bst.predict(dvalid)\n",
    "    pred_labels = np.rint(preds)\n",
    "    auc = sklearn.metrics.roc_auc_score(valid_y, pred_labels)\n",
    "\n",
    "    return auc\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    study = optuna.create_study(\n",
    "        pruner=optuna.pruners.MedianPruner(n_warmup_steps=5), direction=\"maximize\", sampler = TPESampler(seed=31)\n",
    "    )\n",
    "    study.optimize(objective, n_trials=100)\n",
    "    print(\"Number of finished trials: {}\".format(len(study.trials)))\n",
    "\n",
    "    print(\"Best trial:\")\n",
    "    trial = study.best_trial\n",
    "\n",
    "    print(\"  Value: {}\".format(trial.value))\n",
    "\n",
    "    print(\"  Params: \")\n",
    "    for key, value in trial.params.items():\n",
    "        print(\"    {}: {}\".format(key, value))\n",
    "\n",
    "    xgb_params = {}\n",
    "\n",
    "    for key, value in trial.params.items():\n",
    "        xgb_params[key] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 42,
     "status": "ok",
     "timestamp": 1691675085790,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "algcmJCqkAya",
    "outputId": "e8a7b15d-8d82-44ac-f862-b45c5911677f"
   },
   "outputs": [],
   "source": [
    "#See XGBoost hyperparameters.\n",
    "\n",
    "xgb_params['eval_metric'] = 'auc'\n",
    "xgb_params['verbosity'] = 0\n",
    "xgb_params['seed'] = 31\n",
    "\n",
    "print(xgb_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 244
    },
    "executionInfo": {
     "elapsed": 375,
     "status": "ok",
     "timestamp": 1691675086154,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "uZk8hbTUIQPg",
    "outputId": "4cf06d0e-5711-4462-e280-178e350ce65c"
   },
   "outputs": [],
   "source": [
    "#Fit XGBoost.\n",
    "\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "xgb = XGBClassifier(**xgb_params)\n",
    "\n",
    "xgb.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 74
    },
    "executionInfo": {
     "elapsed": 186,
     "status": "ok",
     "timestamp": 1691675086154,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "rUf-OtCoIQPg",
    "outputId": "9bdb3320-c293-44dd-999d-0c746219576a"
   },
   "outputs": [],
   "source": [
    "#Predict on the validation set, get predicted probabilities for calibration, and fit calibration function.\n",
    "\n",
    "calib_probs_xgb = xgb.predict_proba(valid_x)\n",
    "\n",
    "calib_model_xgb = LogisticRegression()\n",
    "calib_model_xgb.fit(calib_probs_xgb, valid_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 185,
     "status": "ok",
     "timestamp": 1691675086154,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "wq-OU3c29-4L",
    "outputId": "21b029b3-2fca-4124-8e22-d8e1680647d9"
   },
   "outputs": [],
   "source": [
    "#Make predictions on the test set based on the trained and calibrated XGBoost model.\n",
    "\n",
    "preds_xgb = xgb.predict(x_test)\n",
    "\n",
    "uncalibrated_probs_xgb = xgb.predict_proba(x_test)\n",
    "\n",
    "probs_xgb = calib_model_xgb.predict_proba(uncalibrated_probs_xgb)\n",
    "probs_xgb = probs_xgb[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2574,
     "status": "ok",
     "timestamp": 1691675088755,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "5m3AkfQJIQPg",
    "outputId": "bdc9294c-7c48-4629-e778-03bfd172e2c9"
   },
   "outputs": [],
   "source": [
    "#Evaluate XGBoost model.\n",
    "\n",
    "xgb_tn, xgb_fp, xgb_fn, xgb_tp = confusion_matrix(y_test, preds_xgb).ravel()\n",
    "xgb_cm = confusion_matrix(y_test, preds_xgb)\n",
    "\n",
    "xgb_sensitivity = round(xgb_tp/(xgb_tp+xgb_fn), 3)\n",
    "xgb_sensitivity_ci_length = z_value * np.sqrt((xgb_sensitivity * (1 - xgb_sensitivity)) / y_test.shape[0])\n",
    "xgb_sensitivity_ci_lower = round((xgb_sensitivity - xgb_sensitivity_ci_length), 3)\n",
    "xgb_sensitivity_ci_upper = round((xgb_sensitivity + xgb_sensitivity_ci_length), 3)\n",
    "xgb_sensitivity_str = str(xgb_sensitivity) + ' (' + str(xgb_sensitivity_ci_lower) + ' - ' + str(xgb_sensitivity_ci_upper) + ')'\n",
    "\n",
    "xgb_specificity = round(xgb_tn/(xgb_tn+xgb_fp), 3)\n",
    "xgb_specificity_ci_length = z_value * np.sqrt((xgb_specificity * (1 - xgb_specificity)) / y_test.shape[0])\n",
    "xgb_specificity_ci_lower = round((xgb_specificity - xgb_specificity_ci_length), 3)\n",
    "xgb_specificity_ci_upper = round((xgb_specificity + xgb_specificity_ci_length), 3)\n",
    "xgb_specificity_str = str(xgb_specificity) + ' (' + str(xgb_specificity_ci_lower) + ' - ' + str(xgb_specificity_ci_upper) + ')'\n",
    "\n",
    "xgb_auprc = round(average_precision_score(y_test, probs_xgb), 3)\n",
    "xgb_auprc_ci_length = z_value * np.sqrt((xgb_auprc * (1 - xgb_auprc)) / y_test.shape[0])\n",
    "xgb_auprc_ci_lower = round((xgb_auprc - xgb_auprc_ci_length), 3)\n",
    "xgb_auprc_ci_upper = round((xgb_auprc + xgb_auprc_ci_length), 3)\n",
    "xgb_auprc_str = str(xgb_auprc) + ' (' + str(xgb_auprc_ci_lower) + ' - ' + str(xgb_auprc_ci_upper) + ')'\n",
    "\n",
    "xgb_accuracy = round(accuracy_score(y_test, preds_xgb), 3)\n",
    "xgb_accuracy_ci_length = z_value * np.sqrt((xgb_accuracy * (1 - xgb_accuracy)) / y_test.shape[0])\n",
    "xgb_accuracy_ci_lower = round((xgb_accuracy - xgb_accuracy_ci_length), 3)\n",
    "xgb_accuracy_ci_upper = round((xgb_accuracy + xgb_accuracy_ci_length), 3)\n",
    "xgb_accuracy_str = str(xgb_accuracy) + ' (' + str(xgb_accuracy_ci_lower) + ' - ' + str(xgb_accuracy_ci_upper) + ')'\n",
    "\n",
    "xgb_auroc, xgb_auroc_ci_lower, xgb_auroc_ci_upper = auroc_ci(y_test, probs_xgb)\n",
    "xgb_auroc = round(xgb_auroc, 3)\n",
    "xgb_auroc_ci_lower = round(xgb_auroc_ci_lower, 3)\n",
    "xgb_auroc_ci_upper = round(xgb_auroc_ci_upper, 3)\n",
    "xgb_auroc_str = str(xgb_auroc) + ' (' + str(xgb_auroc_ci_lower) + ' - ' + str(xgb_auroc_ci_upper) + ')'\n",
    "\n",
    "xgb_brier = round(brier_score_loss(y_test, probs_xgb), 3)\n",
    "xgb_brier_ci_length = z_value * np.sqrt((xgb_brier * (1 - xgb_brier)) / y_test.shape[0])\n",
    "xgb_brier_ci_lower = round((xgb_brier - xgb_brier_ci_length), 3)\n",
    "xgb_brier_ci_upper = round((xgb_brier + xgb_brier_ci_length), 3)\n",
    "xgb_brier_str = str(xgb_brier) + ' (' + str(xgb_brier_ci_lower) + ' - ' + str(xgb_brier_ci_upper) + ')'\n",
    "\n",
    "xgb_results = [xgb_sensitivity_str, xgb_specificity_str, xgb_auprc_str, xgb_accuracy_str, xgb_auroc_str, xgb_brier_str]\n",
    "\n",
    "print(\"Sensitivity: \", (xgb_sensitivity_str))\n",
    "print(\"Specificity: \", (xgb_specificity_str))\n",
    "print('AUPRC: ', (xgb_auprc_str))\n",
    "print('Accuracy: ', (xgb_accuracy_str))\n",
    "print('AUROC: ', (xgb_auroc_str))\n",
    "print('Brier Score: ', (xgb_brier_str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 26,
     "status": "ok",
     "timestamp": 1691675088755,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "Ch-wpxcQIQPg",
    "outputId": "39c7d4f3-514c-4483-ea7c-818d772b5ee6"
   },
   "outputs": [],
   "source": [
    "#Recalculate precision recall curve for plotting purposes.\n",
    "\n",
    "xgb_precision_curve, xgb_recall_curve, _ = precision_recall_curve(y_test, probs_xgb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9VszU3k5IQPh"
   },
   "source": [
    "# LightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 15,
     "status": "ok",
     "timestamp": 1691675088755,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "vPMTdOzC3Iwc",
    "outputId": "685f75b1-0703-435a-996a-820f4b646249"
   },
   "outputs": [],
   "source": [
    "import lightgbm as lgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 29350,
     "status": "ok",
     "timestamp": 1691675118096,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "oR81p9HpIQPh",
    "outputId": "6de11d31-e644-4936-c120-6bb715daccd1"
   },
   "outputs": [],
   "source": [
    "#Hyperparameter tuning for LightGBM.\n",
    "\n",
    "def objective(trial):\n",
    "    dtrain = lgb.Dataset(train_x, label=train_y)\n",
    "\n",
    "    param = {\n",
    "        \"objective\":  trial.suggest_categorical(\"objective\", [\"binary\"]),\n",
    "        \"metric\": \"binary_logloss\",\n",
    "        \"verbosity\": -1,\n",
    "        \"random_state\": 31,\n",
    "        \"boosting_type\":  trial.suggest_categorical(\"boosting_type\", [\"gbdt\"]),\n",
    "        \"lambda_l1\": trial.suggest_float(\"lambda_l1\", 1e-8, 10.0, log=True),\n",
    "        \"lambda_l2\": trial.suggest_float(\"lambda_l2\", 1e-8, 10.0, log=True),\n",
    "        \"num_leaves\": trial.suggest_int(\"num_leaves\", 2, 256),\n",
    "        \"feature_fraction\": trial.suggest_float(\"feature_fraction\", 0.4, 1.0),\n",
    "        \"bagging_fraction\": trial.suggest_float(\"bagging_fraction\", 0.4, 1.0),\n",
    "        \"bagging_freq\": trial.suggest_int(\"bagging_freq\", 1, 7),\n",
    "        \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 5, 100),\n",
    "    }\n",
    "\n",
    "    gbm = lgb.train(param, dtrain)\n",
    "    preds = gbm.predict(valid_x)\n",
    "    pred_labels = np.rint(preds)\n",
    "    auc = sklearn.metrics.roc_auc_score(valid_y, pred_labels)\n",
    "    return auc\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    study = optuna.create_study(direction=\"maximize\", sampler = TPESampler(seed=31))\n",
    "    study.optimize(objective, n_trials=100)\n",
    "\n",
    "    print(\"Number of finished trials: {}\".format(len(study.trials)))\n",
    "\n",
    "    print(\"Best trial:\")\n",
    "    trial = study.best_trial\n",
    "\n",
    "    print(\"  Value: {}\".format(trial.value))\n",
    "\n",
    "    print(\"  Params: \")\n",
    "    for key, value in trial.params.items():\n",
    "        print(\"    {}: {}\".format(key, value))\n",
    "\n",
    "    lgb_params = {}\n",
    "\n",
    "    for key, value in trial.params.items():\n",
    "        lgb_params[key] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 22,
     "status": "ok",
     "timestamp": 1691675118097,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "h7P7Mw7UkC-y",
    "outputId": "64a55986-beb7-4832-89ca-54eb76eb8331"
   },
   "outputs": [],
   "source": [
    "#See LightGBM hyperparameters.\n",
    "\n",
    "lgb_params['metric'] = 'binary_logloss'\n",
    "lgb_params['verbosity'] = -1\n",
    "lgb_params['random_state'] = 31\n",
    "\n",
    "print(lgb_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 227
    },
    "executionInfo": {
     "elapsed": 397,
     "status": "ok",
     "timestamp": 1691675118486,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "UAidUSs1IQPh",
    "outputId": "f934d5d3-c8d3-469f-a2fd-f3e06b1a913f"
   },
   "outputs": [],
   "source": [
    "#Fit LightGBM.\n",
    "\n",
    "import lightgbm as lgb\n",
    "\n",
    "lgb = lgb.LGBMClassifier(**lgb_params)\n",
    "\n",
    "lgb.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 91
    },
    "executionInfo": {
     "elapsed": 189,
     "status": "ok",
     "timestamp": 1691675118486,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "JlKe1z0RIQPh",
    "outputId": "7ff03a39-e2d5-42eb-8074-026b129e00c3"
   },
   "outputs": [],
   "source": [
    "#Predict on the validation set, get predicted probabilities for calibration, and fit calibration function.\n",
    "\n",
    "calib_probs_lgb = lgb.predict_proba(valid_x)\n",
    "\n",
    "calib_model_lgb = LogisticRegression()\n",
    "calib_model_lgb.fit(calib_probs_lgb, valid_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 185,
     "status": "ok",
     "timestamp": 1691675118487,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "mPyYlwGX-HkZ",
    "outputId": "14411683-3830-4f87-961c-eb90daf92a36"
   },
   "outputs": [],
   "source": [
    "#Make predictions on the test set based on the trained and calibrated LightGBM model.\n",
    "\n",
    "preds_lgb = lgb.predict(x_test)\n",
    "\n",
    "uncalibrated_probs_lgb = lgb.predict_proba(x_test)\n",
    "uncalibrated_probs_lgb = uncalibrated_probs_lgb\n",
    "\n",
    "probs_lgb = calib_model_lgb.predict_proba(uncalibrated_probs_lgb)\n",
    "probs_lgb = probs_lgb[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2765,
     "status": "ok",
     "timestamp": 1691684671159,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "r259oDdIIQPi",
    "outputId": "4761ee05-5cad-41d5-ff44-98898e395aba"
   },
   "outputs": [],
   "source": [
    "#Evaluate LightGBM model.\n",
    "\n",
    "lgb_tn, lgb_fp, lgb_fn, lgb_tp = confusion_matrix(y_test, preds_lgb).ravel()\n",
    "lgb_cm = confusion_matrix(y_test, preds_lgb)\n",
    "\n",
    "lgb_sensitivity = round(lgb_tp/(lgb_tp+lgb_fn), 3)\n",
    "lgb_sensitivity_ci_length = z_value * np.sqrt((lgb_sensitivity * (1 - lgb_sensitivity)) / y_test.shape[0])\n",
    "lgb_sensitivity_ci_lower = round((lgb_sensitivity - lgb_sensitivity_ci_length), 3)\n",
    "lgb_sensitivity_ci_upper = round((lgb_sensitivity + lgb_sensitivity_ci_length), 3)\n",
    "lgb_sensitivity_str = str(lgb_sensitivity) + ' (' + str(lgb_sensitivity_ci_lower) + ' - ' + str(lgb_sensitivity_ci_upper) + ')'\n",
    "\n",
    "lgb_specificity = round(lgb_tn/(lgb_tn+lgb_fp), 3)\n",
    "lgb_specificity_ci_length = z_value * np.sqrt((lgb_specificity * (1 - lgb_specificity)) / y_test.shape[0])\n",
    "lgb_specificity_ci_lower = round((lgb_specificity - lgb_specificity_ci_length), 3)\n",
    "lgb_specificity_ci_upper = round((lgb_specificity + lgb_specificity_ci_length), 3)\n",
    "lgb_specificity_str = str(lgb_specificity) + ' (' + str(lgb_specificity_ci_lower) + ' - ' + str(lgb_specificity_ci_upper) + ')'\n",
    "\n",
    "lgb_auprc = round(average_precision_score(y_test, probs_lgb), 3)\n",
    "lgb_auprc_ci_length = z_value * np.sqrt((lgb_auprc * (1 - lgb_auprc)) / y_test.shape[0])\n",
    "lgb_auprc_ci_lower = round((lgb_auprc - lgb_auprc_ci_length), 3)\n",
    "lgb_auprc_ci_upper = round((lgb_auprc + lgb_auprc_ci_length), 3)\n",
    "lgb_auprc_str = str(lgb_auprc) + ' (' + str(lgb_auprc_ci_lower) + ' - ' + str(lgb_auprc_ci_upper) + ')'\n",
    "\n",
    "lgb_accuracy = round(accuracy_score(y_test, preds_lgb), 3)\n",
    "lgb_accuracy_ci_length = z_value * np.sqrt((lgb_accuracy * (1 - lgb_accuracy)) / y_test.shape[0])\n",
    "lgb_accuracy_ci_lower = round((lgb_accuracy - lgb_accuracy_ci_length), 3)\n",
    "lgb_accuracy_ci_upper = round((lgb_accuracy + lgb_accuracy_ci_length), 3)\n",
    "lgb_accuracy_str = str(lgb_accuracy) + ' (' + str(lgb_accuracy_ci_lower) + ' - ' + str(lgb_accuracy_ci_upper) + ')'\n",
    "\n",
    "lgb_auroc, lgb_auroc_ci_lower, lgb_auroc_ci_upper = auroc_ci(y_test, probs_lgb)\n",
    "lgb_auroc = round(lgb_auroc, 3)\n",
    "lgb_auroc_ci_lower = round(lgb_auroc_ci_lower, 3)\n",
    "lgb_auroc_ci_upper = round(lgb_auroc_ci_upper, 3)\n",
    "lgb_auroc_str = str(lgb_auroc) + ' (' + str(lgb_auroc_ci_lower) + ' - ' + str(lgb_auroc_ci_upper) + ')'\n",
    "\n",
    "lgb_brier = round(brier_score_loss(y_test, probs_lgb), 3)\n",
    "lgb_brier_ci_length = z_value * np.sqrt((lgb_brier * (1 - lgb_brier)) / y_test.shape[0])\n",
    "lgb_brier_ci_lower = round((lgb_brier - lgb_brier_ci_length), 3)\n",
    "lgb_brier_ci_upper = round((lgb_brier + lgb_brier_ci_length), 3)\n",
    "lgb_brier_str = str(lgb_brier) + ' (' + str(lgb_brier_ci_lower) + ' - ' + str(lgb_brier_ci_upper) + ')'\n",
    "\n",
    "lgb_results = [lgb_sensitivity_str, lgb_specificity_str, lgb_auprc_str, lgb_accuracy_str, lgb_auroc_str, lgb_brier_str]\n",
    "\n",
    "print(\"Sensitivity: \", (lgb_sensitivity_str))\n",
    "print(\"Specificity: \", (lgb_specificity_str))\n",
    "print('AUPRC: ', (lgb_auprc_str))\n",
    "print('Accuracy: ', (lgb_accuracy_str))\n",
    "print('AUROC: ', (lgb_auroc_str))\n",
    "print('Brier Score: ', (lgb_brier_str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 25,
     "status": "ok",
     "timestamp": 1691675120922,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "PXzXI9O-IQPi",
    "outputId": "7b758448-3a54-40dd-8e18-add3c8e4e22a"
   },
   "outputs": [],
   "source": [
    "#Recalculate precision recall curve for plotting purposes.\n",
    "\n",
    "lgb_precision_curve, lgb_recall_curve, _ = precision_recall_curve(y_test, probs_lgb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mQRRrBp-IQPk"
   },
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true,
     "base_uri": "https://localhost:8080/"
    },
    "id": "kFTmQMQxIQPk",
    "outputId": "20c29007-263c-4aa0-cc56-ae642ea2e0df"
   },
   "outputs": [],
   "source": [
    "#Hyperparameter tuning for Random Forest.\n",
    "\n",
    "def objective(trial):\n",
    "\n",
    "    param = {\n",
    "        \"criterion\": trial.suggest_categorical(\"criterion\", [\"gini\", \"entropy\"]),\n",
    "        \"random_state\": 31,\n",
    "        \"max_features\": trial.suggest_categorical(\"max_features\", [\"auto\", \"sqrt\",\"log2\", None]),\n",
    "        \"max_depth\": trial.suggest_int(\"max_depth\", 1, 100),\n",
    "        \"n_estimators\": trial.suggest_int(\"n_estimators\", 100, 2000, 100),\n",
    "        \"min_samples_leaf\": trial.suggest_int(\"min_samples_leaf\", 1, 4, 1),\n",
    "        \"min_samples_split\": trial.suggest_int(\"min_samples_split\", 2, 10, 1),\n",
    "    }\n",
    "\n",
    "    rf = RandomForestClassifier(**param)\n",
    "\n",
    "    rf.fit(\n",
    "        train_x,\n",
    "        train_y,\n",
    "    )\n",
    "\n",
    "    preds = rf.predict(valid_x)\n",
    "    pred_labels = np.rint(preds)\n",
    "    auc = sklearn.metrics.roc_auc_score(valid_y, pred_labels)\n",
    "\n",
    "    return auc\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    study = optuna.create_study(direction='maximize', sampler = TPESampler(seed=31))\n",
    "    study.optimize(objective, n_trials=100, timeout=600)\n",
    "\n",
    "    print(\"Number of finished trials: {}\".format(len(study.trials)))\n",
    "\n",
    "    print(\"Best trial:\")\n",
    "    trial = study.best_trial\n",
    "\n",
    "    print(\"  Value: {}\".format(trial.value))\n",
    "\n",
    "    print(\"  Params: \")\n",
    "    for key, value in trial.params.items():\n",
    "        print(\"    {}: {}\".format(key, value))\n",
    "\n",
    "    rf_params = {}\n",
    "\n",
    "    for key, value in trial.params.items():\n",
    "        rf_params[key] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "yTqnQTzgkG3R",
    "outputId": "16815ccc-b199-452f-c1df-c71f563afa3c"
   },
   "outputs": [],
   "source": [
    "#See Random Forest hyperparameters.\n",
    "\n",
    "rf_params['random_state'] = 31\n",
    "\n",
    "print(rf_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "o2nnbDAsIQPk",
    "outputId": "92ff2908-1cd8-4dcb-ccc3-42c58bb81048"
   },
   "outputs": [],
   "source": [
    "#Fit Random Forest.\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "rf = RandomForestClassifier(**rf_params)\n",
    "\n",
    "rf.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "pSsBliFuIQPk",
    "outputId": "fdf379e4-cbfc-4a0a-d786-ebffa1831a48"
   },
   "outputs": [],
   "source": [
    "#Predict on the validation set, get predicted probabilities for calibration, and fit calibration function.\n",
    "\n",
    "calib_probs_rf = rf.predict_proba(valid_x)\n",
    "\n",
    "calib_model_rf = LogisticRegression()\n",
    "calib_model_rf.fit(calib_probs_rf, valid_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "o3Ru1zLF-YER",
    "outputId": "4fb07b1f-713d-4e9b-8ecf-0e16a890631f"
   },
   "outputs": [],
   "source": [
    "#Make predictions on the test set based on the trained Random Forest model.\n",
    "\n",
    "preds_rf = rf.predict(x_test)\n",
    "\n",
    "uncalibrated_probs_rf = rf.predict_proba(x_test)\n",
    "\n",
    "probs_rf = calib_model_rf.predict_proba(uncalibrated_probs_rf)\n",
    "probs_rf = probs_rf[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2709,
     "status": "ok",
     "timestamp": 1691684639633,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "haQcIGjoIQPk",
    "outputId": "2c4a0296-36cf-4a8c-e0a3-e3d9d4848024"
   },
   "outputs": [],
   "source": [
    "#Evaluate Random Forest model.\n",
    "\n",
    "rf_tn, rf_fp, rf_fn, rf_tp = confusion_matrix(y_test, preds_rf).ravel()\n",
    "rf_cm = confusion_matrix(y_test, preds_rf)\n",
    "\n",
    "rf_sensitivity = round(rf_tp/(rf_tp+rf_fn), 3)\n",
    "rf_sensitivity_ci_length = z_value * np.sqrt((rf_sensitivity * (1 - rf_sensitivity)) / y_test.shape[0])\n",
    "rf_sensitivity_ci_lower = round((rf_sensitivity - rf_sensitivity_ci_length), 3)\n",
    "rf_sensitivity_ci_upper = round((rf_sensitivity + rf_sensitivity_ci_length), 3)\n",
    "rf_sensitivity_str = str(rf_sensitivity) + ' (' + str(rf_sensitivity_ci_lower) + ' - ' + str(rf_sensitivity_ci_upper) + ')'\n",
    "\n",
    "rf_specificity = round(rf_tn/(rf_tn+rf_fp), 3)\n",
    "rf_specificity_ci_length = z_value * np.sqrt((rf_specificity * (1 - rf_specificity)) / y_test.shape[0])\n",
    "rf_specificity_ci_lower = round((rf_specificity - rf_specificity_ci_length), 3)\n",
    "rf_specificity_ci_upper = round((rf_specificity + rf_specificity_ci_length), 3)\n",
    "rf_specificity_str = str(rf_specificity) + ' (' + str(rf_specificity_ci_lower) + ' - ' + str(rf_specificity_ci_upper) + ')'\n",
    "\n",
    "rf_auprc = round(average_precision_score(y_test, probs_rf), 3)\n",
    "rf_auprc_ci_length = z_value * np.sqrt((rf_auprc * (1 - rf_auprc)) / y_test.shape[0])\n",
    "rf_auprc_ci_lower = round((rf_auprc - rf_auprc_ci_length), 3)\n",
    "rf_auprc_ci_upper = round((rf_auprc + rf_auprc_ci_length), 3)\n",
    "rf_auprc_str = str(rf_auprc) + ' (' + str(rf_auprc_ci_lower) + ' - ' + str(rf_auprc_ci_upper) + ')'\n",
    "\n",
    "rf_accuracy = round(accuracy_score(y_test, preds_rf), 3)\n",
    "rf_accuracy_ci_length = z_value * np.sqrt((rf_accuracy * (1 - rf_accuracy)) / y_test.shape[0])\n",
    "rf_accuracy_ci_lower = round((rf_accuracy - rf_accuracy_ci_length), 3)\n",
    "rf_accuracy_ci_upper = round((rf_accuracy + rf_accuracy_ci_length), 3)\n",
    "rf_accuracy_str = str(rf_accuracy) + ' (' + str(rf_accuracy_ci_lower) + ' - ' + str(rf_accuracy_ci_upper) + ')'\n",
    "\n",
    "rf_auroc, rf_auroc_ci_lower, rf_auroc_ci_upper = auroc_ci(y_test, probs_rf)\n",
    "rf_auroc = round(rf_auroc, 3)\n",
    "rf_auroc_ci_lower = round(rf_auroc_ci_lower, 3)\n",
    "rf_auroc_ci_upper = round(rf_auroc_ci_upper, 3)\n",
    "rf_auroc_str = str(rf_auroc) + ' (' + str(rf_auroc_ci_lower) + ' - ' + str(rf_auroc_ci_upper) + ')'\n",
    "\n",
    "rf_brier = round(brier_score_loss(y_test, probs_rf), 3)\n",
    "rf_brier_ci_length = z_value * np.sqrt((rf_brier * (1 - rf_brier)) / y_test.shape[0])\n",
    "rf_brier_ci_lower = round((rf_brier - rf_brier_ci_length), 3)\n",
    "rf_brier_ci_upper = round((rf_brier + rf_brier_ci_length), 3)\n",
    "rf_brier_str = str(rf_brier) + ' (' + str(rf_brier_ci_lower) + ' - ' + str(rf_brier_ci_upper) + ')'\n",
    "\n",
    "rf_results = [rf_sensitivity_str, rf_specificity_str, rf_auprc_str, rf_accuracy_str, rf_auroc_str, rf_brier_str]\n",
    "\n",
    "print(\"Sensitivity: \", (rf_sensitivity_str))\n",
    "print(\"Specificity: \", (rf_specificity_str))\n",
    "print('AUPRC: ', (rf_auprc_str))\n",
    "print('Accuracy: ', (rf_accuracy_str))\n",
    "print('AUROC: ', (rf_auroc_str))\n",
    "print('Brier Score: ', (rf_brier_str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Drrp3KEQIQPk"
   },
   "outputs": [],
   "source": [
    "#Recalculate precision recall curve for plotting purposes.\n",
    "\n",
    "rf_precision_curve, rf_recall_curve, _ = precision_recall_curve(y_test, probs_rf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "fLvNgketIQPl"
   },
   "source": [
    "# ROC and PR Curves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 942
    },
    "executionInfo": {
     "elapsed": 1716,
     "status": "ok",
     "timestamp": 1691684689258,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "0mMqO-euIQPl",
    "outputId": "81a6d56f-6115-48b3-905d-5425c8d1daeb"
   },
   "outputs": [],
   "source": [
    "#Plot ROC curve.\n",
    "\n",
    "f = pyplot.figure()\n",
    "f.set_figwidth(12)\n",
    "f.set_figheight(12)\n",
    "\n",
    "tabpfn_fpr, tabpfn_tpr, _ = roc_curve(y_test, probs_tabpfn)\n",
    "tabpfn_label = 'TabPFN AUROC: ' + tabpfn_auroc_str\n",
    "pyplot.plot(tabpfn_fpr, tabpfn_tpr, label = tabpfn_label, color = 'b', linewidth = 3.5, alpha = 0.75)\n",
    "\n",
    "tabnet_fpr, tabnet_tpr, _ = roc_curve(y_test, probs_tabnet)\n",
    "tabnet_label = 'TabNet AUROC: ' + tabnet_auroc_str\n",
    "pyplot.plot(tabnet_fpr, tabnet_tpr, label = tabnet_label, color = 'g', linewidth = 3.5, alpha = 0.75)\n",
    "\n",
    "xgb_fpr, xgb_tpr, _ = roc_curve(y_test, probs_xgb)\n",
    "xgb_label = 'XGBoost AUROC: ' + xgb_auroc_str\n",
    "pyplot.plot(xgb_fpr, xgb_tpr, label = xgb_label, color = 'r', linewidth = 3.5, alpha = 0.75)\n",
    "\n",
    "lgb_fpr, lgb_tpr, _ = roc_curve(y_test, probs_lgb)\n",
    "lgb_label = 'LightGBM AUROC: ' + lgb_auroc_str\n",
    "pyplot.plot(lgb_fpr, lgb_tpr, label = lgb_label, color='c', linewidth = 3.5, alpha = 0.75)\n",
    "\n",
    "rf_fpr, rf_tpr, _ = roc_curve(y_test, probs_rf)\n",
    "rf_label = 'Random Forest AUROC: ' + rf_auroc_str\n",
    "pyplot.plot(rf_fpr, rf_tpr, label = rf_label, color = 'm', linewidth = 3.5, alpha = 0.75)\n",
    "\n",
    "pyplot.plot([0, 1], [0, 1], linestyle = '--', linewidth=2)\n",
    "\n",
    "pyplot.title('E', x = -0.075, y = 1.005, fontsize = 75, pad = 20)\n",
    "pyplot.xlabel('False Positive Rate', fontsize = 22, fontweight = 'heavy', labelpad = 16)\n",
    "pyplot.ylabel('True Positive Rate', fontsize = 22, fontweight = 'heavy', labelpad = 16)\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", labelsize = 16)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", labelsize = 16)\n",
    "\n",
    "leg = pyplot.legend(loc = 'lower right', fontsize = 18)\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g3_12m_roc.png', dpi=300)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 938
    },
    "executionInfo": {
     "elapsed": 1685,
     "status": "ok",
     "timestamp": 1691684695732,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "GpotrYFBIQPl",
    "outputId": "18b4be99-6a94-4ca3-dcff-a140e0dd1b40"
   },
   "outputs": [],
   "source": [
    "#Plot PR curve.\n",
    "\n",
    "f = pyplot.figure()\n",
    "f.set_figwidth(12)\n",
    "f.set_figheight(12)\n",
    "\n",
    "tabpfn_label = 'TabPFN AUPRC: ' + tabpfn_auprc_str\n",
    "tabnet_label = 'TabNet AUPRC: ' + tabnet_auprc_str\n",
    "xgb_label = 'XGBoost AUPRC: ' + xgb_auprc_str\n",
    "lgb_label = 'LightGBM AUPRC: ' + lgb_auprc_str\n",
    "rf_label = 'Random Forest AUPRC: ' + rf_auprc_str\n",
    "\n",
    "pyplot.plot(tabpfn_recall_curve, tabpfn_precision_curve, label = tabpfn_label, color = 'b', linewidth = 3.5, alpha = 0.75)\n",
    "pyplot.plot(tabnet_recall_curve, tabnet_precision_curve, label = tabnet_label, color = 'g', linewidth = 3.5, alpha = 0.75)\n",
    "pyplot.plot(xgb_recall_curve, xgb_precision_curve, label = xgb_label, color = 'r', linewidth = 3.5, alpha = 0.75)\n",
    "pyplot.plot(lgb_recall_curve, lgb_precision_curve, label = lgb_label, color = 'c', linewidth = 3.5, alpha = 0.75)\n",
    "pyplot.plot(rf_recall_curve, rf_precision_curve, label = rf_label, color = 'm', linewidth = 3.5, alpha = 0.75)\n",
    "\n",
    "pyplot.title('E', x = -0.075, y = 1.005, fontsize = 75, pad = 20)\n",
    "pyplot.xlabel('Recall', fontsize = 18, fontweight = 'heavy', labelpad = 16)\n",
    "pyplot.ylabel('Precision', fontsize = 22, fontweight = 'heavy', labelpad = 16)\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", labelsize = 16)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", labelsize = 16)\n",
    "\n",
    "leg = pyplot.legend(loc = 'lower left', fontsize = 18)\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g3_12m_prc.png', dpi=300)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rOySRdcDIQPl"
   },
   "source": [
    "# Results Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 350
    },
    "executionInfo": {
     "elapsed": 251,
     "status": "ok",
     "timestamp": 1691684697282,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "kznujF4kIQPl",
    "outputId": "48e461db-cb78-4261-e2ef-8da6ef73478d"
   },
   "outputs": [],
   "source": [
    "results = {'TabPFN':tabpfn_results, 'TabNet':tabnet_results, 'XGBoost':xgb_results, 'LightGBM':lgb_results, 'Random Forest':rf_results}\n",
    "\n",
    "results = pd.DataFrame(results, columns = ['TabPFN', 'TabNet', 'XGBoost', 'LightGBM', 'Random Forest'])\n",
    "results = results.T\n",
    "\n",
    "results.columns = ['Sensitivity (95% CI)', 'Specificity (95% CI)',  'AUPRC (95% CI)', 'Accuracy (95% CI)', 'AUROC (95% CI)', 'Brier Score (95% CI)']\n",
    "\n",
    "results.to_csv('/content/drive/MyDrive/NCDB-G2G3 Glioma/g3_12m_results.csv')\n",
    "\n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uAD5jLGchIp4"
   },
   "source": [
    "# Confusion Matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 716
    },
    "executionInfo": {
     "elapsed": 813,
     "status": "ok",
     "timestamp": 1691684728370,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "BDEMrQ5khLfg",
    "outputId": "0dc155c1-22e9-46c5-9085-c4addcf209ce"
   },
   "outputs": [],
   "source": [
    "tabpfn_cm = confusion_matrix(y_test, preds_tabpfn)\n",
    "\n",
    "f = pyplot.figure(figsize=(8,8))\n",
    "\n",
    "sns.heatmap(tabpfn_cm, annot=True, fmt='d', cmap='Blues', cbar=False, annot_kws={\"size\": 16}, linewidths=1, linecolor='black')\n",
    "\n",
    "labels = ['Survival', 'Mortality']\n",
    "pyplot.xticks([0.5,1.5], labels, fontsize=16, fontweight='heavy')\n",
    "pyplot.yticks([0.5,1.5], labels, fontsize=16, fontweight='heavy', va='center')\n",
    "\n",
    "pyplot.xlabel('Predicted', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "pyplot.ylabel('Truth', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", pad=10)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", pad=10)\n",
    "pyplot.title('A', x = -0.095, y = 1.005, fontsize = 75, pad = 20)\n",
    "\n",
    "pyplot.subplots_adjust(left=0.20, right=0.85, bottom=0.20, top=0.80)\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g3_12m_cm_tabpfn.png', dpi=300)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 753
    },
    "executionInfo": {
     "elapsed": 506,
     "status": "ok",
     "timestamp": 1691684728868,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "MRXGPobHhNEX",
    "outputId": "e0de3c37-6c81-4919-e9bc-3b2030e55a52"
   },
   "outputs": [],
   "source": [
    "tabnet_cm = confusion_matrix(y_test, preds_tabnet)\n",
    "\n",
    "f = pyplot.figure(figsize=(8,8))\n",
    "\n",
    "sns.heatmap(tabnet_cm, annot=True, fmt='d', cmap='Blues', cbar=False, annot_kws={\"size\": 16}, linewidths=1, linecolor='black')\n",
    "\n",
    "labels = ['Survival', 'Mortality']\n",
    "pyplot.xticks([0.5,1.5], labels, fontsize=16, fontweight='heavy')\n",
    "pyplot.yticks([0.5,1.5], labels, fontsize=16, fontweight='heavy', va='center')\n",
    "\n",
    "pyplot.xlabel('Predicted', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "pyplot.ylabel('Truth', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", pad=10)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", pad=10)\n",
    "pyplot.title('B', x = -0.095, y = 1.005, fontsize = 75, pad = 20)\n",
    "\n",
    "pyplot.subplots_adjust(left=0.20, right=0.85, bottom=0.20, top=0.80)\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g3_12m_cm_tabnet.png', dpi=300)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 753
    },
    "executionInfo": {
     "elapsed": 681,
     "status": "ok",
     "timestamp": 1691684729441,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "xLZLslcChShm",
    "outputId": "acd8f493-4e52-4ccc-b3b4-4363401c49e1"
   },
   "outputs": [],
   "source": [
    "xgb_cm = confusion_matrix(y_test, preds_xgb)\n",
    "\n",
    "f = pyplot.figure(figsize=(8,8))\n",
    "\n",
    "sns.heatmap(xgb_cm, annot=True, fmt='d', cmap='Blues', cbar=False, annot_kws={\"size\": 16}, linewidths=1, linecolor='black')\n",
    "\n",
    "labels = ['Survival', 'Mortality']\n",
    "pyplot.xticks([0.5,1.5], labels, fontsize=16, fontweight='heavy')\n",
    "pyplot.yticks([0.5,1.5], labels, fontsize=16, fontweight='heavy', va='center')\n",
    "\n",
    "pyplot.xlabel('Predicted', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "pyplot.ylabel('Truth', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", pad=10)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", pad=10)\n",
    "pyplot.title('C', x = -0.095, y = 1.005, fontsize = 75, pad = 20)\n",
    "\n",
    "pyplot.subplots_adjust(left=0.20, right=0.85, bottom=0.20, top=0.80)\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g3_12m_cm_xgb.png', dpi=300)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 753
    },
    "executionInfo": {
     "elapsed": 542,
     "status": "ok",
     "timestamp": 1691684729978,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "QMNzhoAAhUnc",
    "outputId": "0d19b332-307f-4ec7-a6f0-bd2d54effe33"
   },
   "outputs": [],
   "source": [
    "lgb_cm = confusion_matrix(y_test, preds_lgb)\n",
    "\n",
    "f = pyplot.figure(figsize=(8,8))\n",
    "\n",
    "sns.heatmap(lgb_cm, annot=True, fmt='d', cmap='Blues', cbar=False, annot_kws={\"size\": 16}, linewidths=1, linecolor='black')\n",
    "\n",
    "labels = ['Survival', 'Mortality']\n",
    "pyplot.xticks([0.5,1.5], labels, fontsize=16, fontweight='heavy')\n",
    "pyplot.yticks([0.5,1.5], labels, fontsize=16, fontweight='heavy', va='center')\n",
    "\n",
    "pyplot.xlabel('Predicted', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "pyplot.ylabel('Truth', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", pad=10)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", pad=10)\n",
    "pyplot.title('E', x = -0.095, y = 1.005, fontsize = 75, pad = 20)\n",
    "\n",
    "pyplot.subplots_adjust(left=0.20, right=0.85, bottom=0.20, top=0.80)\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g3_12m_cm_lgb.png', dpi=300)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 753
    },
    "executionInfo": {
     "elapsed": 654,
     "status": "ok",
     "timestamp": 1691684730626,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "-Rn8yc5QhWqo",
    "outputId": "fd059e84-6142-4cbf-f698-45742c086178"
   },
   "outputs": [],
   "source": [
    "rf_cm = confusion_matrix(y_test, preds_rf)\n",
    "\n",
    "f = pyplot.figure(figsize=(8,8))\n",
    "\n",
    "sns.heatmap(rf_cm, annot=True, fmt='d', cmap='Blues', cbar=False, annot_kws={\"size\": 16}, linewidths=1, linecolor='black')\n",
    "\n",
    "labels = ['Survival', 'Mortality']\n",
    "pyplot.xticks([0.5,1.5], labels, fontsize=16, fontweight='heavy')\n",
    "pyplot.yticks([0.5,1.5], labels, fontsize=16, fontweight='heavy', va='center')\n",
    "\n",
    "pyplot.xlabel('Predicted', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "pyplot.ylabel('Truth', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", pad=10)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", pad=10)\n",
    "pyplot.title('D', x = -0.095, y = 1.005, fontsize = 75, pad = 20)\n",
    "\n",
    "pyplot.subplots_adjust(left=0.20, right=0.85, bottom=0.20, top=0.80)\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g3_12m_cm_rf.png', dpi=300)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qYnu4PnCIQPm"
   },
   "source": [
    "# SHAP Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 17,
     "status": "ok",
     "timestamp": 1691684730626,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "BuvljQiYu-Xb",
    "outputId": "bee827a7-1369-47b2-ded0-001c3f45d83d"
   },
   "outputs": [],
   "source": [
    "import textwrap\n",
    "def wrap_labels(ax, width, break_long_words=False):\n",
    "    labels = []\n",
    "    for label in ax.get_yticklabels():\n",
    "        text = label.get_text()\n",
    "        labels.append(textwrap.fill(text, width=width,\n",
    "                                    break_long_words=break_long_words))\n",
    "    ax.set_yticklabels(labels, rotation=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gjL0ZhOZJh-b"
   },
   "outputs": [],
   "source": [
    "feature_names = x_test.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true,
     "base_uri": "https://localhost:8080/"
    },
    "id": "GTKndsLJzH7h"
   },
   "outputs": [],
   "source": [
    "#Calculate SHAP values for TabPFN.\n",
    "\n",
    "tabpfn_explainer = shap.Explainer(tabpfn.predict, x_test)\n",
    "tabpfn_shap_values = tabpfn_explainer(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "Lw5zXxxQzH7p"
   },
   "outputs": [],
   "source": [
    "#Plot SHAP bar plot for TabPFN.\n",
    "\n",
    "shap.plots.bar(tabpfn_shap_values, max_display = 15, show=False)\n",
    "\n",
    "fig = pyplot.gcf()\n",
    "ax = pyplot.gca()\n",
    "fig.set_figheight(12)\n",
    "fig.set_figwidth(5)\n",
    "\n",
    "pyplot.title('A', x = -0.5, y = 1, fontsize = 50, pad = 20)\n",
    "pyplot.xlabel(\"Mean |SHAP Value|\", fontsize =12, fontweight = 'heavy', labelpad = 8)\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", labelsize = 12)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", labelsize = 12)\n",
    "\n",
    "wrap_labels(ax, 30)\n",
    "ax.figure\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g3_12m_shap_tabpfn.png', dpi=300, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "XotWpfehXbCw"
   },
   "outputs": [],
   "source": [
    "#Calculate SHAP values for TabNet.\n",
    "\n",
    "tabnet_explainer = shap.Explainer(tabnet.predict, x_test.values, feature_names=feature_names)\n",
    "tabnet_shap_values = tabnet_explainer(x_test.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "tzHasDCwXoxs"
   },
   "outputs": [],
   "source": [
    "#Plot SHAP bar plot for TabNet.\n",
    "\n",
    "shap.plots.bar(tabnet_shap_values, max_display = 15, show=False)\n",
    "\n",
    "fig = pyplot.gcf()\n",
    "ax = pyplot.gca()\n",
    "fig.set_figheight(12)\n",
    "fig.set_figwidth(5)\n",
    "\n",
    "pyplot.title('B', x = -0.5, y = 1, fontsize = 50, pad = 20)\n",
    "pyplot.xlabel(\"Mean |SHAP Value|\", fontsize =12, fontweight = 'heavy', labelpad = 8)\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", labelsize = 12)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", labelsize = 12)\n",
    "\n",
    "wrap_labels(ax, 30)\n",
    "ax.figure\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g3_12m_shap_tabnet.png', dpi=300, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "C09esVZrIQPm"
   },
   "outputs": [],
   "source": [
    "#Calculate SHAP values for XGBoost.\n",
    "\n",
    "xgb_explainer = shap.Explainer(xgb.predict, x_test)\n",
    "xgb_shap_values = xgb_explainer(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "6O74ff97qW2H"
   },
   "outputs": [],
   "source": [
    "#Plot SHAP bar plot for XGBoost.\n",
    "\n",
    "shap.plots.bar(xgb_shap_values, max_display = 15, show=False)\n",
    "\n",
    "fig = pyplot.gcf()\n",
    "ax = pyplot.gca()\n",
    "fig.set_figheight(12)\n",
    "fig.set_figwidth(5)\n",
    "\n",
    "pyplot.title('C', x = -0.5, y = 1, fontsize = 50, pad = 20)\n",
    "pyplot.xlabel(\"Mean |SHAP Value|\", fontsize =12, fontweight = 'heavy', labelpad = 8)\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", labelsize = 12)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", labelsize = 12)\n",
    "\n",
    "wrap_labels(ax, 30)\n",
    "ax.figure\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g3_12m_shap_xgb.png', dpi=300, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "HBQmzC4-IQPm"
   },
   "outputs": [],
   "source": [
    "#Calculate SHAP values for LightGBM.\n",
    "\n",
    "lgb_explainer = shap.Explainer(lgb.predict, x_test)\n",
    "lgb_shap_values = lgb_explainer(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "4V-NEfx_xysV"
   },
   "outputs": [],
   "source": [
    "#Plot SHAP bar plot for LightGBM.\n",
    "\n",
    "shap.plots.bar(lgb_shap_values, max_display = 15, show=False)\n",
    "\n",
    "fig = pyplot.gcf()\n",
    "ax = pyplot.gca()\n",
    "fig.set_figheight(12)\n",
    "fig.set_figwidth(5)\n",
    "\n",
    "pyplot.title('E', x = -0.5, y = 1, fontsize = 50, pad = 20)\n",
    "pyplot.xlabel(\"Mean |SHAP Value|\", fontsize =12, fontweight = 'heavy', labelpad = 8)\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", labelsize = 12)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", labelsize = 12)\n",
    "\n",
    "wrap_labels(ax, 30)\n",
    "ax.figure\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g3_12m_shap_lgb.png', dpi=300, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "Vj3B1IeWIQPm"
   },
   "outputs": [],
   "source": [
    "#Calculate SHAP values for Random Forest.\n",
    "\n",
    "rf_explainer = shap.Explainer(rf.predict, x_test)\n",
    "rf_shap_values = rf_explainer(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "x2P2or1bx25u"
   },
   "outputs": [],
   "source": [
    "#Plot SHAP bar plot for Random Forest.\n",
    "\n",
    "shap.plots.bar(rf_shap_values, max_display = 15, show=False)\n",
    "\n",
    "fig = pyplot.gcf()\n",
    "ax = pyplot.gca()\n",
    "fig.set_figheight(12)\n",
    "fig.set_figwidth(5)\n",
    "\n",
    "pyplot.title('D', x = -0.5, y = 1, fontsize = 50, pad = 20)\n",
    "pyplot.xlabel(\"Mean |SHAP Value|\", fontsize =12, fontweight = 'heavy', labelpad = 8)\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", labelsize = 12)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", labelsize = 12)\n",
    "\n",
    "wrap_labels(ax, 30)\n",
    "ax.figure\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g3_12m_shap_rf.png', dpi=300, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cwYwpeSZz_-x"
   },
   "source": [
    "#Partial Dependency Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "cCezSsG3XK1c"
   },
   "outputs": [],
   "source": [
    "column_filters = {\n",
    "    'Age': (18, 90),\n",
    "    'Great Circle Distance': (0, 500),\n",
    "}\n",
    "\n",
    "for column, (min_val, max_val) in column_filters.items():\n",
    "    if column in x_test.columns:\n",
    "        x_test = x_test[(x_test[column] > min_val) & (x_test[column] < max_val)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "jLWR5lmo01KM"
   },
   "outputs": [],
   "source": [
    "pyplot.rcParams[\"figure.figsize\"] = (15, 15)\n",
    "pyplot.rcParams[\"figure.dpi\"] = 300\n",
    "pyplot.rcParams['axes.labelweight'] = 'bold'\n",
    "pyplot.rcParams['axes.labelsize'] = 12\n",
    "pyplot.rcParams['axes.labelpad'] = 6\n",
    "pyplot.rcParams['font.weight'] = 'normal'\n",
    "pyplot.rcParams['lines.linewidth'] = 2.5\n",
    "pyplot.rcParams['xtick.labelsize'] = 8\n",
    "pyplot.rcParams['ytick.labelsize'] = 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "4mCUeepoYdNJ"
   },
   "outputs": [],
   "source": [
    "categorical_features = ['Facility Type', 'Facility Location', 'Sex', 'Race', 'Hispanic Ethnicity', 'Histology',\n",
    "                        'Primary Payor', 'Percent No High School Education Quartiles', 'Census Median Income Quartiles',\n",
    "                        'Rurality', 'Charlson-Deyo Score', 'Tumor Localization', 'Laterality', 'Grade',\n",
    "                        'Diagnostic Biopsy', 'Ki-67 Labeling Index', 'Karnofsky Performance Scale',\n",
    "                        'MGMT Methylation', 'Focality', 'Tumor Extension', 'Tumor Size', 'Radiation Treatment-Surgery Sequence',\n",
    "                        'Chemotherapy', 'Immunotherapy', 'Systemic Therapy-Surgery Sequence', 'Extent of Resection',\n",
    "                        'Resective Surgery', 'Radiation Treatment', '1p19q Co-Deletion', 'Histology']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "UNB6VbOg5KKx"
   },
   "outputs": [],
   "source": [
    "#Calculate the mean absolute SHAP values for each feature.\n",
    "tabpfn_mean_abs_shap_values = np.mean(np.abs(tabpfn_shap_values.values), axis=0)\n",
    "\n",
    "#Create a DataFrame to map feature names to their mean absolute SHAP values.\n",
    "tabpfn_shap_summary = pd.DataFrame(list(zip(feature_names, tabpfn_mean_abs_shap_values)), columns=['Feature', 'Mean SHAP'])\n",
    "\n",
    "#Sort the DataFrame by 'Mean SHAP' in descending order.\n",
    "tabpfn_shap_summary_sorted = tabpfn_shap_summary.sort_values('Mean SHAP', ascending=False)\n",
    "\n",
    "#Get the names of the features.\n",
    "tabpfn_features = tabpfn_shap_summary_sorted['Feature'].tolist()\n",
    "tabpfn_features = tabpfn_features[:9]\n",
    "\n",
    "tabpfn_categorical_features = [item for item in tabpfn_features if item in categorical_features]\n",
    "\n",
    "PartialDependenceDisplay.from_estimator(tabpfn, x_test.sample(frac=0.05, random_state=42), tabpfn_features, categorical_features = tabpfn_categorical_features)\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g3_12m_pdp_tabpfn.png', dpi=300, bbox_inches='tight')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "bLDT7TOZ5JFi"
   },
   "outputs": [],
   "source": [
    "#Calculate the mean absolute SHAP values for each feature.\n",
    "tabnet_mean_abs_shap_values = np.mean(np.abs(tabnet_shap_values.values), axis=0)\n",
    "\n",
    "#Create a DataFrame to map feature names to their mean absolute SHAP values.\n",
    "tabnet_shap_summary = pd.DataFrame(list(zip(feature_names, tabnet_mean_abs_shap_values)), columns=['Feature', 'Mean SHAP'])\n",
    "\n",
    "#Sort the DataFrame by 'Mean SHAP' in descending order.\n",
    "tabnet_shap_summary_sorted = tabnet_shap_summary.sort_values('Mean SHAP', ascending=False)\n",
    "\n",
    "#Get the names of the features.\n",
    "tabnet_features = tabnet_shap_summary_sorted['Feature'].tolist()\n",
    "tabnet_features = tabnet_features[:9]\n",
    "\n",
    "tabnet_categorical_features = [item for item in tabnet_features if item in categorical_features]\n",
    "\n",
    "PartialDependenceDisplay.from_estimator(xgb, x_test, tabnet_features, categorical_features = tabnet_categorical_features)\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g3_12m_pdp_tabnet.png', dpi=300, bbox_inches='tight')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "frN2YQUD0IWP"
   },
   "outputs": [],
   "source": [
    "#Calculate the mean absolute SHAP values for each feature.\n",
    "xgb_mean_abs_shap_values = np.mean(np.abs(xgb_shap_values.values), axis=0)\n",
    "\n",
    "#Create a DataFrame to map feature names to their mean absolute SHAP values.\n",
    "xgb_shap_summary = pd.DataFrame(list(zip(feature_names, xgb_mean_abs_shap_values)), columns=['Feature', 'Mean SHAP'])\n",
    "\n",
    "#Sort the DataFrame by 'Mean SHAP' in descending order.\n",
    "xgb_shap_summary_sorted = xgb_shap_summary.sort_values('Mean SHAP', ascending=False)\n",
    "\n",
    "#Get the names of the features.\n",
    "xgb_features = xgb_shap_summary_sorted['Feature'].tolist()\n",
    "xgb_features = xgb_features[:9]\n",
    "\n",
    "xgb_categorical_features = [item for item in xgb_features if item in categorical_features]\n",
    "\n",
    "PartialDependenceDisplay.from_estimator(xgb, x_test, xgb_features, categorical_features = xgb_categorical_features)\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g3_12m_pdp_xgb.png', dpi=300, bbox_inches='tight')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "P9ZSSzpP5MbC"
   },
   "outputs": [],
   "source": [
    "#Calculate the mean absolute SHAP values for each feature.\n",
    "lgb_mean_abs_shap_values = np.mean(np.abs(lgb_shap_values.values), axis=0)\n",
    "\n",
    "#Create a DataFrame to map feature names to their mean absolute SHAP values.\n",
    "lgb_shap_summary = pd.DataFrame(list(zip(feature_names, lgb_mean_abs_shap_values)), columns=['Feature', 'Mean SHAP'])\n",
    "\n",
    "#Sort the DataFrame by 'Mean SHAP' in descending order.\n",
    "lgb_shap_summary_sorted = lgb_shap_summary.sort_values('Mean SHAP', ascending=False)\n",
    "\n",
    "#Get the names of the features.\n",
    "lgb_features = lgb_shap_summary_sorted['Feature'].tolist()\n",
    "lgb_features = lgb_features[:9]\n",
    "\n",
    "lgb_categorical_features = [item for item in lgb_features if item in categorical_features]\n",
    "\n",
    "PartialDependenceDisplay.from_estimator(lgb, x_test, lgb_features, categorical_features = lgb_categorical_features)\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g3_12m_pdp_lgb.png', dpi=300, bbox_inches='tight')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "aq_m3WU_5NKq"
   },
   "outputs": [],
   "source": [
    "#Calculate the mean absolute SHAP values for each feature.\n",
    "rf_mean_abs_shap_values = np.mean(np.abs(rf_shap_values.values), axis=0)\n",
    "\n",
    "#Create a DataFrame to map feature names to their mean absolute SHAP values.\n",
    "rf_shap_summary = pd.DataFrame(list(zip(feature_names, rf_mean_abs_shap_values)), columns=['Feature', 'Mean SHAP'])\n",
    "\n",
    "#Sort the DataFrame by 'Mean SHAP' in descending order.\n",
    "rf_shap_summary_sorted = rf_shap_summary.sort_values('Mean SHAP', ascending=False)\n",
    "\n",
    "#Get the names of the features.\n",
    "rf_features = rf_shap_summary_sorted['Feature'].tolist()\n",
    "rf_features = rf_features[:9]\n",
    "\n",
    "rf_categorical_features = [item for item in rf_features if item in categorical_features]\n",
    "\n",
    "PartialDependenceDisplay.from_estimator(rf, x_test, rf_features, categorical_features = rf_categorical_features)\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-G2G3 Glioma/g3_12m_pdp_rf.png', dpi=300, bbox_inches='tight')\n",
    "pyplot.show()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "machine_shape": "hm",
   "name": "",
   "provenance": [
    {
     "file_id": "1M02d2-HBRrt5xmA-5x863g67zB9zloB3",
     "timestamp": 1691611251612
    },
    {
     "file_id": "14TfLVbewZLcEADsU9RM-JZ4Jrc-obgN-",
     "timestamp": 1691610327686
    },
    {
     "file_id": "1Eo4jPVLNOwgUyqgXVFTcpiRbGrIuXwk5",
     "timestamp": 1690313290635
    },
    {
     "file_id": "1ehzQDERWA7oATVlIze0f5fvBuuSk3Il-",
     "timestamp": 1690235650728
    },
    {
     "file_id": "10kjTiT6ZbSYDMQVY6jE58AJ9bNIvu_Z4",
     "timestamp": 1680804894233
    },
    {
     "file_id": "1SyxRVArbYdS2kIxHOoZLuxtWYNsZL3ei",
     "timestamp": 1675315289522
    },
    {
     "file_id": "1rquc9s2K5gcWtyx-P9n0ie3ooY4kzveD",
     "timestamp": 1673820526724
    },
    {
     "file_id": "1sd9nBAoznz2E976naQ2pJvARRLH-vRK7",
     "timestamp": 1673714326538
    },
    {
     "file_id": "1tydPpyRVpKMFnUtRbvvIdQTTUtRQ1SJK",
     "timestamp": 1672252113955
    },
    {
     "file_id": "1FJ2Tp28HBJo_krRjesYEkutbjOcsodmu",
     "timestamp": 1672251686516
    },
    {
     "file_id": "1vmOc15wf8HLrh9xcXe9IRvqZIrBNUFp_",
     "timestamp": 1671647832509
    },
    {
     "file_id": "1jvG_2brh3p2BhjDqh6LIyEOlVCFKYSX6",
     "timestamp": 1671034103968
    },
    {
     "file_id": "1YKiMGVsPQlDRLgB2d3Svjjsp96BkJxlX",
     "timestamp": 1670597416903
    },
    {
     "file_id": "1gG7ScMqQA2JnmntazVsJaQStNW3qHl9R",
     "timestamp": 1670198888608
    },
    {
     "file_id": "1V4GD1xQ2LBQhBny_OS3iBlGyn9mh7oWM",
     "timestamp": 1669993880849
    },
    {
     "file_id": "1euhY_qXOTy9qT_vn0Q6Kb2eEUOCHom8m",
     "timestamp": 1669310009337
    },
    {
     "file_id": "1ksEkZ_gSIaosRd56LkYx54z5oGNBgq2C",
     "timestamp": 1668779215067
    },
    {
     "file_id": "1o0f_u5loubhXZTvFpGUCA_m2gpTKQ6EL",
     "timestamp": 1668619772114
    }
   ],
   "toc_visible": true,
   "version": ""
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}