NCDB_Meningioma_12M_Survival.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 31494,
     "status": "ok",
     "timestamp": 1691162106717,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "RxtJ9G-aIZ_y",
    "outputId": "0b99c301-ea93-4eb8-ee1a-a766c3b2a8c3"
   },
   "outputs": [],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 40063,
     "status": "ok",
     "timestamp": 1691162146767,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "h_PnaQ4NIuE0",
    "outputId": "cabaab4d-8964-41f8-a9a3-ad342529e341"
   },
   "outputs": [],
   "source": [
    "!pip install optuna\n",
    "!pip install shap\n",
    "!pip install pytorch-tabnet\n",
    "!pip install tabpfn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "JrZMQLTfIQPb"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import sklearn\n",
    "import sklearn.metrics\n",
    "from sklearn.metrics import roc_auc_score, roc_curve, precision_recall_curve, average_precision_score, auc, precision_score, recall_score, accuracy_score, balanced_accuracy_score, brier_score_loss, confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.utils import resample\n",
    "from math import sqrt\n",
    "from scipy import stats as st\n",
    "from random import randrange\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "\n",
    "from tabpfn import TabPFNClassifier\n",
    "from pytorch_tabnet.tab_model import TabNetClassifier\n",
    "import xgboost as xgb\n",
    "import lightgbm as lgb\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "import optuna\n",
    "from optuna.samplers import TPESampler\n",
    "\n",
    "import shap\n",
    "from sklearn.inspection import PartialDependenceDisplay"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UWbpPFMVHS-V"
   },
   "source": [
    "# Preparing Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "sDsi_6tnIQPc"
   },
   "outputs": [],
   "source": [
    "#Open csv file.\n",
    "\n",
    "data = pd.read_csv('/content/drive/MyDrive/NCDB-Meningioma/final_data.csv', index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 32,
     "status": "ok",
     "timestamp": 1691162161735,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "hPmUaNyUN2p8",
    "outputId": "fc411590-0d98-47fd-e3af-6c06727f4898"
   },
   "outputs": [],
   "source": [
    "#Define outcome of interest (3-month mortality).\n",
    "\n",
    "data.loc[(data['Vital Status'] == 'Dead') & (data['Last Contact or Death (Months from Diagnosis)'] < 12), 'OUTCOME'] = 1\n",
    "data.loc[(data['Vital Status'] == 'Dead') & (data['Last Contact or Death (Months from Diagnosis)'] >= 12), 'OUTCOME'] = 0\n",
    "data.loc[(data['Vital Status'] == 'Alive') & (data['Last Contact or Death (Months from Diagnosis)'] >= 12), 'OUTCOME'] = 0\n",
    "data = data.dropna(subset=['OUTCOME'])\n",
    "\n",
    "print(data['OUTCOME'].value_counts(normalize=False, dropna=False))\n",
    "print(data['OUTCOME'].value_counts(normalize=True, dropna=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "LhZeK6d8IQPd"
   },
   "outputs": [],
   "source": [
    "#Define predictor variables (x) and outcome of interest (y).\n",
    "\n",
    "outcomes = ['Vital Status', 'Last Contact or Death (Months from Diagnosis)', 'OUTCOME']\n",
    "\n",
    "x = data.drop(outcomes, axis = 1)\n",
    "y = data['OUTCOME']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 25,
     "status": "ok",
     "timestamp": 1691162161735,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "mt6ks1daIQPd",
    "outputId": "e0ad3317-1fdb-4fbb-9f48-e0e8b986487c"
   },
   "outputs": [],
   "source": [
    "#Check data shapes.\n",
    "\n",
    "print(y.shape)\n",
    "print(x.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 18,
     "status": "ok",
     "timestamp": 1691162161735,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "d0IwUyCsIQPe",
    "outputId": "9dbd2bc5-42ef-4359-dfe6-3d87d5849565"
   },
   "outputs": [],
   "source": [
    "#Split data into initial train set and test set in 80:20 ratio.\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.2, random_state = 0)\n",
    "\n",
    "#Describe initial train set and test set.\n",
    "\n",
    "print(\"Number patients x_train dataset: \", x_train.shape[0])\n",
    "print(\"Number patients y_train dataset: \", y_train.shape[0])\n",
    "print(\"Number patients x_test dataset: \", x_test.shape[0])\n",
    "print(\"Number patients y_test dataset: \", y_test.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 13,
     "status": "ok",
     "timestamp": 1691162161735,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "WyNo7C2Pbzt2",
    "outputId": "c9d0cc4f-1aaa-40ff-ebf6-e66712f6b686"
   },
   "outputs": [],
   "source": [
    "#Split initial train set into final train set and validation set in 75:25 ratio.\n",
    "\n",
    "train_x, valid_x, train_y, valid_y = train_test_split(x_train, y_train, test_size = 0.25, random_state = 0)\n",
    "\n",
    "#Describe train and validation sets.\n",
    "\n",
    "print(\"Number patients train_x dataset: \", train_x.shape[0])\n",
    "print(\"Number patients train_y dataset: \", train_y.shape[0])\n",
    "print(\"Number patients valid_x dataset: \", valid_x.shape[0])\n",
    "print(\"Number patients valid_y dataset: \", valid_y.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 9,
     "status": "ok",
     "timestamp": 1691162161735,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "g2O0He35IQPe",
    "outputId": "fed20120-cf00-400d-b5f8-60d770b333ac"
   },
   "outputs": [],
   "source": [
    "#Describe outcome of interest before resampling.\n",
    "\n",
    "print(\"Before resampling, counts of label '1': {}\".format(sum(train_y == 1)))\n",
    "print(\"Before resampling, counts of label '0': {} \\n\".format(sum(train_y == 0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "QvXczViaIQPe"
   },
   "outputs": [],
   "source": [
    "#Apply SMOTE.\n",
    "\n",
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "resampler = SMOTE(random_state = 0)\n",
    "train_x, train_y = resampler.fit_resample(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 17,
     "status": "ok",
     "timestamp": 1691162162103,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "mYw68tzFIQPe",
    "outputId": "3ddc73e4-6f5a-45e1-cea8-c66bdf3cb165"
   },
   "outputs": [],
   "source": [
    "#Describe outcome of interest after resampling.\n",
    "\n",
    "print(\"After resampling, counts of label '1': {}\".format(sum(train_y == 1)))\n",
    "print(\"After resampling, counts of label '0': {} \\n\".format(sum(train_y == 0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lrb8PN0dEOc1"
   },
   "outputs": [],
   "source": [
    "#Define function for AUROC with 95% confidence intervals.\n",
    "\n",
    "def auroc_ci(y_test, y_probs, positive=1, n_samples=1000, alpha=0.05):\n",
    "    auroc_values = []\n",
    "    for i in range(n_samples):\n",
    "        y_test_bs, y_probs_bs = resample(y_test, y_probs)\n",
    "\n",
    "        auroc = roc_auc_score(y_test_bs, y_probs_bs)\n",
    "        auroc_values.append(auroc)\n",
    "\n",
    "    mean_auroc = np.mean(auroc_values)\n",
    "    std_auroc = np.std(auroc_values)\n",
    "\n",
    "    lower_ci = mean_auroc - 1.96 * std_auroc\n",
    "    upper_ci = mean_auroc + 1.96 * std_auroc\n",
    "\n",
    "    return auroc, lower_ci, upper_ci"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "BLkyBrpf95-0"
   },
   "outputs": [],
   "source": [
    "#Define z-value for other confidence intervals.\n",
    "\n",
    "confidence = 0.95\n",
    "z_value = st.norm.ppf((1 + confidence) / 2.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UWX_CriXYlL_"
   },
   "source": [
    "#TabPFN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "OFQmwskRLrY4"
   },
   "source": [
    "No hyperparameter tuning is performed for TabPFN since the paper that introduced TabPFN claims no hyperparameter tuning is needed for it:\n",
    "\n",
    "\n",
    "*We present TabPFN, a trained Transformer that can do supervised classification for small tabular datasets in less than a second, needs no hyperparameter tuning and is competitive with state-of-the-art classification methods.*\n",
    "\n",
    "https://doi.org/10.48550/arXiv.2207.01848\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 162
    },
    "executionInfo": {
     "elapsed": 14138,
     "status": "ok",
     "timestamp": 1691162176227,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "F3gvvODJ8B2Z",
    "outputId": "2c8b8b48-d0f6-49c8-e37a-790d95c2c1d3"
   },
   "outputs": [],
   "source": [
    "#Fit TabPFN.\n",
    "\n",
    "tabpfn = TabPFNClassifier(device='cpu', N_ensemble_configurations=2)\n",
    "\n",
    "tabpfn.fit(x_train.sample(n=128, random_state=42), y_train.sample(n=128, random_state=42), overwrite_warning = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 74
    },
    "executionInfo": {
     "elapsed": 12370,
     "status": "ok",
     "timestamp": 1691162188489,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "5oLbONin8YGA",
    "outputId": "2cd55625-febe-40eb-ee4a-9e539b1d4c32"
   },
   "outputs": [],
   "source": [
    "#Predict on the validation set, get predicted probabilities for calibration, and fit calibration function.\n",
    "\n",
    "calib_probs_tabpfn = tabpfn.predict_proba(valid_x.values)\n",
    "\n",
    "calib_model_tabpfn = LogisticRegression()\n",
    "calib_model_tabpfn.fit(calib_probs_tabpfn, valid_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 19216,
     "status": "ok",
     "timestamp": 1691162207593,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "UZNInXD39Tux",
    "outputId": "34edbade-9940-4627-addc-a9bffde98fee"
   },
   "outputs": [],
   "source": [
    "#Make predictions on the test set based on the trained TabNet model.\n",
    "\n",
    "preds_tabpfn = tabpfn.predict(x_test.values)\n",
    "\n",
    "uncalibrated_probs_tabpfn = tabpfn.predict_proba(x_test.values)\n",
    "\n",
    "probs_tabpfn = calib_model_tabpfn.predict_proba(uncalibrated_probs_tabpfn)\n",
    "probs_tabpfn = probs_tabpfn[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2506,
     "status": "ok",
     "timestamp": 1691162210084,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "Sm8PNeSm9dtI",
    "outputId": "f3d617ad-d444-4a47-943e-67a91ad73c0c"
   },
   "outputs": [],
   "source": [
    "#Evaluate TabPFN model.\n",
    "\n",
    "tabpfn_tn, tabpfn_fp, tabpfn_fn, tabpfn_tp = confusion_matrix(y_test, preds_tabpfn).ravel()\n",
    "tabpfn_cm = confusion_matrix(y_test, preds_tabpfn)\n",
    "\n",
    "tabpfn_sensitivity = round(tabpfn_tp/(tabpfn_tp+tabpfn_fn), 3)\n",
    "tabpfn_sensitivity_ci_length = z_value * np.sqrt((tabpfn_sensitivity * (1 - tabpfn_sensitivity)) / y_test.shape[0])\n",
    "tabpfn_sensitivity_ci_lower = round((tabpfn_sensitivity - tabpfn_sensitivity_ci_length), 3)\n",
    "tabpfn_sensitivity_ci_upper = round((tabpfn_sensitivity + tabpfn_sensitivity_ci_length), 3)\n",
    "tabpfn_sensitivity_str = str(tabpfn_sensitivity) + ' (' + str(tabpfn_sensitivity_ci_lower) + ' - ' + str(tabpfn_sensitivity_ci_upper) + ')'\n",
    "\n",
    "tabpfn_specificity = round(tabpfn_tn/(tabpfn_tn+tabpfn_fp), 3)\n",
    "tabpfn_specificity_ci_length = z_value * np.sqrt((tabpfn_specificity * (1 - tabpfn_specificity)) / y_test.shape[0])\n",
    "tabpfn_specificity_ci_lower = round((tabpfn_specificity - tabpfn_specificity_ci_length), 3)\n",
    "tabpfn_specificity_ci_upper = round((tabpfn_specificity + tabpfn_specificity_ci_length), 3)\n",
    "tabpfn_specificity_str = str(tabpfn_specificity) + ' (' + str(tabpfn_specificity_ci_lower) + ' - ' + str(tabpfn_specificity_ci_upper) + ')'\n",
    "\n",
    "tabpfn_auprc = round(average_precision_score(y_test, probs_tabpfn), 3)\n",
    "tabpfn_auprc_ci_length = z_value * np.sqrt((tabpfn_auprc * (1 - tabpfn_auprc)) / y_test.shape[0])\n",
    "tabpfn_auprc_ci_lower = round((tabpfn_auprc - tabpfn_auprc_ci_length), 3)\n",
    "tabpfn_auprc_ci_upper = round((tabpfn_auprc + tabpfn_auprc_ci_length), 3)\n",
    "tabpfn_auprc_str = str(tabpfn_auprc) + ' (' + str(tabpfn_auprc_ci_lower) + ' - ' + str(tabpfn_auprc_ci_upper) + ')'\n",
    "\n",
    "tabpfn_accuracy = round(accuracy_score(y_test, preds_tabpfn), 3)\n",
    "tabpfn_accuracy_ci_length = z_value * np.sqrt((tabpfn_accuracy * (1 - tabpfn_accuracy)) / y_test.shape[0])\n",
    "tabpfn_accuracy_ci_lower = round((tabpfn_accuracy - tabpfn_accuracy_ci_length), 3)\n",
    "tabpfn_accuracy_ci_upper = round((tabpfn_accuracy + tabpfn_accuracy_ci_length), 3)\n",
    "tabpfn_accuracy_str = str(tabpfn_accuracy) + ' (' + str(tabpfn_accuracy_ci_lower) + ' - ' + str(tabpfn_accuracy_ci_upper) + ')'\n",
    "\n",
    "tabpfn_auroc, tabpfn_auroc_ci_lower, tabpfn_auroc_ci_upper = auroc_ci(y_test, probs_tabpfn)\n",
    "tabpfn_auroc = round(tabpfn_auroc, 3)\n",
    "tabpfn_auroc_ci_lower = round(tabpfn_auroc_ci_lower, 3)\n",
    "tabpfn_auroc_ci_upper = round(tabpfn_auroc_ci_upper, 3)\n",
    "tabpfn_auroc_str = str(tabpfn_auroc) + ' (' + str(tabpfn_auroc_ci_lower) + ' - ' + str(tabpfn_auroc_ci_upper) + ')'\n",
    "\n",
    "tabpfn_brier = round(brier_score_loss(y_test, probs_tabpfn), 3)\n",
    "tabpfn_brier_ci_length = z_value * np.sqrt((tabpfn_brier * (1 - tabpfn_brier)) / y_test.shape[0])\n",
    "tabpfn_brier_ci_lower = round((tabpfn_brier - tabpfn_brier_ci_length), 3)\n",
    "tabpfn_brier_ci_upper = round((tabpfn_brier + tabpfn_brier_ci_length), 3)\n",
    "tabpfn_brier_str = str(tabpfn_brier) + ' (' + str(tabpfn_brier_ci_lower) + ' - ' + str(tabpfn_brier_ci_upper) + ')'\n",
    "\n",
    "tabpfn_results = [tabpfn_sensitivity_str, tabpfn_specificity_str, tabpfn_auprc_str, tabpfn_accuracy_str, tabpfn_auroc_str, tabpfn_brier_str]\n",
    "\n",
    "print(\"Sensitivity: \", (tabpfn_sensitivity_str))\n",
    "print(\"Specificity: \", (tabpfn_specificity_str))\n",
    "print('AUPRC: ', (tabpfn_auprc_str))\n",
    "print('Accuracy: ', (tabpfn_accuracy_str))\n",
    "print('AUROC: ', (tabpfn_auroc_str))\n",
    "print('Brier Score: ', (tabpfn_brier_str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 22,
     "status": "ok",
     "timestamp": 1691162210085,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "0wp5veHVFWIB",
    "outputId": "6688d69a-4ad8-475b-e1cc-cf978ec5a14a"
   },
   "outputs": [],
   "source": [
    "#Recalculate precision recall curve for plotting purposes.\n",
    "\n",
    "tabpfn_precision_curve, tabpfn_recall_curve, _ = precision_recall_curve(y_test, probs_tabpfn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BY44IPIU7sBi"
   },
   "source": [
    "# TabNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 689947,
     "status": "ok",
     "timestamp": 1691162900017,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "beooviDI7uuS",
    "outputId": "d5d7a5bb-8044-4867-a916-9496417e0112"
   },
   "outputs": [],
   "source": [
    "# Hyperparameter tuning for TabNet.\n",
    "\n",
    "def objective(trial):\n",
    "\n",
    "    param = {\n",
    "        \"n_d\": trial.suggest_int(\"n_d\", 8, 64),\n",
    "        \"n_a\": trial.suggest_int(\"n_a\", 8, 64),\n",
    "        \"n_steps\": trial.suggest_int(\"n_steps\", 1, 10),\n",
    "        \"gamma\": trial.suggest_float(\"gamma\", 1.0, 2.0),\n",
    "        \"n_independent\": trial.suggest_int(\"n_independent\", 1, 5),\n",
    "        \"n_shared\": trial.suggest_int(\"n_shared\", 1, 5),\n",
    "        \"lambda_sparse\": trial.suggest_loguniform(\"lambda_sparse\", 1e-6, 1e-3),\n",
    "        \"optimizer_params\": {\"lr\": trial.suggest_loguniform(\"lr\", 1e-5, 1e-2)},\n",
    "        \"seed\": 31,\n",
    "    }\n",
    "\n",
    "    tabnet_clf = TabNetClassifier(**param)\n",
    "\n",
    "    tabnet_clf.fit(\n",
    "        train_x.values,\n",
    "        train_y.values,\n",
    "        eval_set=[(valid_x.values, valid_y.values)],\n",
    "        eval_metric=[\"auc\"],\n",
    "        max_epochs=20,\n",
    "        patience=20,\n",
    "        batch_size=1024,\n",
    "        virtual_batch_size=128,\n",
    "        num_workers=0,\n",
    "        drop_last=False,\n",
    "    )\n",
    "\n",
    "    preds = tabnet_clf.predict_proba(valid_x.values)[:, 1]\n",
    "    pred_labels = np.rint(preds)\n",
    "    auc = sklearn.metrics.roc_auc_score(valid_y, pred_labels)\n",
    "\n",
    "    return auc\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    study = optuna.create_study(direction='maximize', sampler=TPESampler(seed=31))\n",
    "    study.optimize(objective, n_trials=100, timeout=600)\n",
    "\n",
    "    print(\"Number of finished trials: {}\".format(len(study.trials)))\n",
    "\n",
    "    print(\"Best trial:\")\n",
    "    trial = study.best_trial\n",
    "\n",
    "    print(\"  Value: {}\".format(trial.value))\n",
    "\n",
    "    print(\"  Params: \")\n",
    "    for key, value in trial.params.items():\n",
    "        print(\"    {}: {}\".format(key, value))\n",
    "\n",
    "    tabnet_params = {}\n",
    "\n",
    "    for key, value in trial.params.items():\n",
    "        tabnet_params[key] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 30,
     "status": "ok",
     "timestamp": 1691162900018,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "gXsTkX3n_i2y",
    "outputId": "c36974dc-b8d9-491d-9448-ad080bd20eb4"
   },
   "outputs": [],
   "source": [
    "#See TabNet hyperparameters.\n",
    "\n",
    "tabnet_params.pop('lr')\n",
    "\n",
    "print(tabnet_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 200186,
     "status": "ok",
     "timestamp": 1691163100192,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "pujw5iMQ_i2y",
    "outputId": "1a5d7060-d4e4-46c2-97cb-7ea8b00bf543"
   },
   "outputs": [],
   "source": [
    "#Fit TabNet.\n",
    "\n",
    "tabnet = TabNetClassifier(**tabnet_params)\n",
    "\n",
    "tabnet.fit(train_x.values, train_y.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 111
    },
    "executionInfo": {
     "elapsed": 122,
     "status": "ok",
     "timestamp": 1691163100193,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "xYiPND2v_i2y",
    "outputId": "7a50fb05-3155-465b-9fd2-5f4747ad7d18"
   },
   "outputs": [],
   "source": [
    "#Predict on the validation set, get predicted probabilities for calibration, and fit calibration function.\n",
    "\n",
    "calib_probs_tabnet = tabnet.predict_proba(valid_x.values)\n",
    "\n",
    "calib_model_tabnet = LogisticRegression()\n",
    "calib_model_tabnet.fit(calib_probs_tabnet, valid_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 469,
     "status": "ok",
     "timestamp": 1691163100550,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "y1AJq9eA_i2y",
    "outputId": "9d3546e5-c5a7-4061-ef0b-a2eb5654f590"
   },
   "outputs": [],
   "source": [
    "#Make predictions on the test set based on the trained TabNet model.\n",
    "\n",
    "preds_tabnet = tabnet.predict(x_test.values)\n",
    "\n",
    "uncalibrated_probs_tabnet = tabnet.predict_proba(x_test.values)\n",
    "\n",
    "probs_tabnet = calib_model_tabnet.predict_proba(uncalibrated_probs_tabnet)\n",
    "probs_tabnet = probs_tabnet[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2496,
     "status": "ok",
     "timestamp": 1691163103041,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "AP-y4lEI_i2z",
    "outputId": "79693967-1c85-49e9-c359-d145f2bd399c"
   },
   "outputs": [],
   "source": [
    "#Evaluate TabNet model.\n",
    "\n",
    "tabnet_tn, tabnet_fp, tabnet_fn, tabnet_tp = confusion_matrix(y_test, preds_tabnet).ravel()\n",
    "tabnet_cm = confusion_matrix(y_test, preds_tabnet)\n",
    "\n",
    "tabnet_sensitivity = round(tabnet_tp/(tabnet_tp+tabnet_fn), 3)\n",
    "tabnet_sensitivity_ci_length = z_value * np.sqrt((tabnet_sensitivity * (1 - tabnet_sensitivity)) / y_test.shape[0])\n",
    "tabnet_sensitivity_ci_lower = round((tabnet_sensitivity - tabnet_sensitivity_ci_length), 3)\n",
    "tabnet_sensitivity_ci_upper = round((tabnet_sensitivity + tabnet_sensitivity_ci_length), 3)\n",
    "tabnet_sensitivity_str = str(tabnet_sensitivity) + ' (' + str(tabnet_sensitivity_ci_lower) + ' - ' + str(tabnet_sensitivity_ci_upper) + ')'\n",
    "\n",
    "tabnet_specificity = round(tabnet_tn/(tabnet_tn+tabnet_fp), 3)\n",
    "tabnet_specificity_ci_length = z_value * np.sqrt((tabnet_specificity * (1 - tabnet_specificity)) / y_test.shape[0])\n",
    "tabnet_specificity_ci_lower = round((tabnet_specificity - tabnet_specificity_ci_length), 3)\n",
    "tabnet_specificity_ci_upper = round((tabnet_specificity + tabnet_specificity_ci_length), 3)\n",
    "tabnet_specificity_str = str(tabnet_specificity) + ' (' + str(tabnet_specificity_ci_lower) + ' - ' + str(tabnet_specificity_ci_upper) + ')'\n",
    "\n",
    "tabnet_auprc = round(average_precision_score(y_test, probs_tabnet), 3)\n",
    "tabnet_auprc_ci_length = z_value * np.sqrt((tabnet_auprc * (1 - tabnet_auprc)) / y_test.shape[0])\n",
    "tabnet_auprc_ci_lower = round((tabnet_auprc - tabnet_auprc_ci_length), 3)\n",
    "tabnet_auprc_ci_upper = round((tabnet_auprc + tabnet_auprc_ci_length), 3)\n",
    "tabnet_auprc_str = str(tabnet_auprc) + ' (' + str(tabnet_auprc_ci_lower) + ' - ' + str(tabnet_auprc_ci_upper) + ')'\n",
    "\n",
    "tabnet_accuracy = round(accuracy_score(y_test, preds_tabnet), 3)\n",
    "tabnet_accuracy_ci_length = z_value * np.sqrt((tabnet_accuracy * (1 - tabnet_accuracy)) / y_test.shape[0])\n",
    "tabnet_accuracy_ci_lower = round((tabnet_accuracy - tabnet_accuracy_ci_length), 3)\n",
    "tabnet_accuracy_ci_upper = round((tabnet_accuracy + tabnet_accuracy_ci_length), 3)\n",
    "tabnet_accuracy_str = str(tabnet_accuracy) + ' (' + str(tabnet_accuracy_ci_lower) + ' - ' + str(tabnet_accuracy_ci_upper) + ')'\n",
    "\n",
    "tabnet_auroc, tabnet_auroc_ci_lower, tabnet_auroc_ci_upper = auroc_ci(y_test, probs_tabnet)\n",
    "tabnet_auroc = round(tabnet_auroc, 3)\n",
    "tabnet_auroc_ci_lower = round(tabnet_auroc_ci_lower, 3)\n",
    "tabnet_auroc_ci_upper = round(tabnet_auroc_ci_upper, 3)\n",
    "tabnet_auroc_str = str(tabnet_auroc) + ' (' + str(tabnet_auroc_ci_lower) + ' - ' + str(tabnet_auroc_ci_upper) + ')'\n",
    "\n",
    "tabnet_brier = round(brier_score_loss(y_test, probs_tabnet), 3)\n",
    "tabnet_brier_ci_length = z_value * np.sqrt((tabnet_brier * (1 - tabnet_brier)) / y_test.shape[0])\n",
    "tabnet_brier_ci_lower = round((tabnet_brier - tabnet_brier_ci_length), 3)\n",
    "tabnet_brier_ci_upper = round((tabnet_brier + tabnet_brier_ci_length), 3)\n",
    "tabnet_brier_str = str(tabnet_brier) + ' (' + str(tabnet_brier_ci_lower) + ' - ' + str(tabnet_brier_ci_upper) + ')'\n",
    "\n",
    "tabnet_results = [tabnet_sensitivity_str, tabnet_specificity_str, tabnet_auprc_str, tabnet_accuracy_str, tabnet_auroc_str, tabnet_brier_str]\n",
    "\n",
    "print(\"Sensitivity: \", (tabnet_sensitivity_str))\n",
    "print(\"Specificity: \", (tabnet_specificity_str))\n",
    "print('AUPRC: ', (tabnet_auprc_str))\n",
    "print('Accuracy: ', (tabnet_accuracy_str))\n",
    "print('AUROC: ', (tabnet_auroc_str))\n",
    "print('Brier Score: ', (tabnet_brier_str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 21,
     "status": "ok",
     "timestamp": 1691163103042,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "tsAsKQ1e_i2z",
    "outputId": "cda08134-d768-4819-ca0f-a990e70d2d71"
   },
   "outputs": [],
   "source": [
    "#Recalculate precision recall curve for plotting purposes.\n",
    "\n",
    "tabnet_precision_curve, tabnet_recall_curve, _ = precision_recall_curve(y_test, probs_tabnet)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "esuV5nR8IQPe"
   },
   "source": [
    "# XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 20337,
     "status": "ok",
     "timestamp": 1691163123365,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "dBWNs_MxIQPf",
    "outputId": "6cda940b-9742-4f64-eef3-805b1ae3415d"
   },
   "outputs": [],
   "source": [
    "#Hyperparameter tuning for XGBoost.\n",
    "\n",
    "def objective(trial):\n",
    "\n",
    "    dtrain = xgb.DMatrix(train_x, label=train_y)\n",
    "    dvalid = xgb.DMatrix(valid_x, label=valid_y)\n",
    "\n",
    "    param = {\n",
    "        \"seed\": 31,\n",
    "        \"verbosity\": 0,\n",
    "        \"objective\":  trial.suggest_categorical(\"objective\", [\"binary:logistic\"]),\n",
    "        \"eval_metric\": \"auc\",\n",
    "        \"booster\": trial.suggest_categorical(\"booster\", [\"gbtree\"]),\n",
    "        \"lambda\": trial.suggest_float(\"lambda\", 1e-8, 1.0, log=True),\n",
    "        \"alpha\": trial.suggest_float(\"alpha\", 1e-8, 1.0, log=True),\n",
    "        \"max_depth\" : trial.suggest_int(\"max_depth\", 1, 9),\n",
    "        \"eta\" : trial.suggest_float(\"eta\", 1e-8, 1.0, log=True),\n",
    "        \"gamma\" : trial.suggest_float(\"gamma\", 1e-8, 1.0, log=True),\n",
    "        \"grow_policy\" : trial.suggest_categorical(\"grow_policy\", [\"depthwise\", \"lossguide\"])\n",
    "    }\n",
    "\n",
    "\n",
    "    # Add a callback for pruning.\n",
    "    pruning_callback = optuna.integration.XGBoostPruningCallback(trial, \"validation-auc\")\n",
    "\n",
    "    bst = xgb.train(param, dtrain, evals=[(dvalid, \"validation\")], callbacks=[pruning_callback])\n",
    "    preds = bst.predict(dvalid)\n",
    "    pred_labels = np.rint(preds)\n",
    "    auc = sklearn.metrics.roc_auc_score(valid_y, pred_labels)\n",
    "\n",
    "    return auc\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    study = optuna.create_study(\n",
    "        pruner=optuna.pruners.MedianPruner(n_warmup_steps=5), direction=\"maximize\", sampler = TPESampler(seed=31)\n",
    "    )\n",
    "    study.optimize(objective, n_trials=100)\n",
    "    print(\"Number of finished trials: {}\".format(len(study.trials)))\n",
    "\n",
    "    print(\"Best trial:\")\n",
    "    trial = study.best_trial\n",
    "\n",
    "    print(\"  Value: {}\".format(trial.value))\n",
    "\n",
    "    print(\"  Params: \")\n",
    "    for key, value in trial.params.items():\n",
    "        print(\"    {}: {}\".format(key, value))\n",
    "\n",
    "    xgb_params = {}\n",
    "\n",
    "    for key, value in trial.params.items():\n",
    "        xgb_params[key] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 34,
     "status": "ok",
     "timestamp": 1691163123366,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "algcmJCqkAya",
    "outputId": "fa0c73d6-1fc2-43ee-81ef-6b15aa0371d4"
   },
   "outputs": [],
   "source": [
    "#See XGBoost hyperparameters.\n",
    "\n",
    "xgb_params['eval_metric'] = 'auc'\n",
    "xgb_params['verbosity'] = 0\n",
    "xgb_params['seed'] = 31\n",
    "\n",
    "print(xgb_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 251
    },
    "executionInfo": {
     "elapsed": 1021,
     "status": "ok",
     "timestamp": 1691163124381,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "uZk8hbTUIQPg",
    "outputId": "aa326d7e-c459-4842-e30e-3cbcd1d001d6"
   },
   "outputs": [],
   "source": [
    "#Fit XGBoost.\n",
    "\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "xgb = XGBClassifier(**xgb_params)\n",
    "\n",
    "xgb.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 74
    },
    "executionInfo": {
     "elapsed": 130,
     "status": "ok",
     "timestamp": 1691163124382,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "rUf-OtCoIQPg",
    "outputId": "feb800bb-8739-4592-a6a8-991cfb1e9624"
   },
   "outputs": [],
   "source": [
    "#Predict on the validation set, get predicted probabilities for calibration, and fit calibration function.\n",
    "\n",
    "calib_probs_xgb = xgb.predict_proba(valid_x)\n",
    "\n",
    "calib_model_xgb = LogisticRegression()\n",
    "calib_model_xgb.fit(calib_probs_xgb, valid_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 127,
     "status": "ok",
     "timestamp": 1691163124382,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "wq-OU3c29-4L",
    "outputId": "4b2165f1-7870-4b78-8c95-a75f2e5f2102"
   },
   "outputs": [],
   "source": [
    "#Make predictions on the test set based on the trained and calibrated XGBoost model.\n",
    "\n",
    "preds_xgb = xgb.predict(x_test)\n",
    "\n",
    "uncalibrated_probs_xgb = xgb.predict_proba(x_test)\n",
    "\n",
    "probs_xgb = calib_model_xgb.predict_proba(uncalibrated_probs_xgb)\n",
    "probs_xgb = probs_xgb[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 3092,
     "status": "ok",
     "timestamp": 1691163127354,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "5m3AkfQJIQPg",
    "outputId": "e77a062e-28f6-4444-a7ad-e80a3df2a62a"
   },
   "outputs": [],
   "source": [
    "#Evaluate XGBoost model.\n",
    "\n",
    "xgb_tn, xgb_fp, xgb_fn, xgb_tp = confusion_matrix(y_test, preds_xgb).ravel()\n",
    "xgb_cm = confusion_matrix(y_test, preds_xgb)\n",
    "\n",
    "xgb_sensitivity = round(xgb_tp/(xgb_tp+xgb_fn), 3)\n",
    "xgb_sensitivity_ci_length = z_value * np.sqrt((xgb_sensitivity * (1 - xgb_sensitivity)) / y_test.shape[0])\n",
    "xgb_sensitivity_ci_lower = round((xgb_sensitivity - xgb_sensitivity_ci_length), 3)\n",
    "xgb_sensitivity_ci_upper = round((xgb_sensitivity + xgb_sensitivity_ci_length), 3)\n",
    "xgb_sensitivity_str = str(xgb_sensitivity) + ' (' + str(xgb_sensitivity_ci_lower) + ' - ' + str(xgb_sensitivity_ci_upper) + ')'\n",
    "\n",
    "xgb_specificity = round(xgb_tn/(xgb_tn+xgb_fp), 3)\n",
    "xgb_specificity_ci_length = z_value * np.sqrt((xgb_specificity * (1 - xgb_specificity)) / y_test.shape[0])\n",
    "xgb_specificity_ci_lower = round((xgb_specificity - xgb_specificity_ci_length), 3)\n",
    "xgb_specificity_ci_upper = round((xgb_specificity + xgb_specificity_ci_length), 3)\n",
    "xgb_specificity_str = str(xgb_specificity) + ' (' + str(xgb_specificity_ci_lower) + ' - ' + str(xgb_specificity_ci_upper) + ')'\n",
    "\n",
    "xgb_auprc = round(average_precision_score(y_test, probs_xgb), 3)\n",
    "xgb_auprc_ci_length = z_value * np.sqrt((xgb_auprc * (1 - xgb_auprc)) / y_test.shape[0])\n",
    "xgb_auprc_ci_lower = round((xgb_auprc - xgb_auprc_ci_length), 3)\n",
    "xgb_auprc_ci_upper = round((xgb_auprc + xgb_auprc_ci_length), 3)\n",
    "xgb_auprc_str = str(xgb_auprc) + ' (' + str(xgb_auprc_ci_lower) + ' - ' + str(xgb_auprc_ci_upper) + ')'\n",
    "\n",
    "xgb_accuracy = round(accuracy_score(y_test, preds_xgb), 3)\n",
    "xgb_accuracy_ci_length = z_value * np.sqrt((xgb_accuracy * (1 - xgb_accuracy)) / y_test.shape[0])\n",
    "xgb_accuracy_ci_lower = round((xgb_accuracy - xgb_accuracy_ci_length), 3)\n",
    "xgb_accuracy_ci_upper = round((xgb_accuracy + xgb_accuracy_ci_length), 3)\n",
    "xgb_accuracy_str = str(xgb_accuracy) + ' (' + str(xgb_accuracy_ci_lower) + ' - ' + str(xgb_accuracy_ci_upper) + ')'\n",
    "\n",
    "xgb_auroc, xgb_auroc_ci_lower, xgb_auroc_ci_upper = auroc_ci(y_test, probs_xgb)\n",
    "xgb_auroc = round(xgb_auroc, 3)\n",
    "xgb_auroc_ci_lower = round(xgb_auroc_ci_lower, 3)\n",
    "xgb_auroc_ci_upper = round(xgb_auroc_ci_upper, 3)\n",
    "xgb_auroc_str = str(xgb_auroc) + ' (' + str(xgb_auroc_ci_lower) + ' - ' + str(xgb_auroc_ci_upper) + ')'\n",
    "\n",
    "xgb_brier = round(brier_score_loss(y_test, probs_xgb), 3)\n",
    "xgb_brier_ci_length = z_value * np.sqrt((xgb_brier * (1 - xgb_brier)) / y_test.shape[0])\n",
    "xgb_brier_ci_lower = round((xgb_brier - xgb_brier_ci_length), 3)\n",
    "xgb_brier_ci_upper = round((xgb_brier + xgb_brier_ci_length), 3)\n",
    "xgb_brier_str = str(xgb_brier) + ' (' + str(xgb_brier_ci_lower) + ' - ' + str(xgb_brier_ci_upper) + ')'\n",
    "\n",
    "xgb_results = [xgb_sensitivity_str, xgb_specificity_str, xgb_auprc_str, xgb_accuracy_str, xgb_auroc_str, xgb_brier_str]\n",
    "\n",
    "print(\"Sensitivity: \", (xgb_sensitivity_str))\n",
    "print(\"Specificity: \", (xgb_specificity_str))\n",
    "print('AUPRC: ', (xgb_auprc_str))\n",
    "print('Accuracy: ', (xgb_accuracy_str))\n",
    "print('AUROC: ', (xgb_auroc_str))\n",
    "print('Brier Score: ', (xgb_brier_str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 23,
     "status": "ok",
     "timestamp": 1691163127355,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "Ch-wpxcQIQPg",
    "outputId": "e0c84625-4052-4722-cbcb-a06e64bedc3e"
   },
   "outputs": [],
   "source": [
    "#Recalculate precision recall curve for plotting purposes.\n",
    "\n",
    "xgb_precision_curve, xgb_recall_curve, _ = precision_recall_curve(y_test, probs_xgb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9VszU3k5IQPh"
   },
   "source": [
    "# LightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 43420,
     "status": "ok",
     "timestamp": 1691163170759,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "oR81p9HpIQPh",
    "outputId": "ab0d2d73-d321-45ad-ae58-8a9a0b54b545"
   },
   "outputs": [],
   "source": [
    "#Hyperparameter tuning for LightGBM.\n",
    "\n",
    "def objective(trial):\n",
    "    dtrain = lgb.Dataset(train_x, label=train_y)\n",
    "\n",
    "    param = {\n",
    "        \"objective\":  trial.suggest_categorical(\"objective\", [\"binary\"]),\n",
    "        \"metric\": \"binary_logloss\",\n",
    "        \"verbosity\": -1,\n",
    "        \"random_state\": 31,\n",
    "        \"boosting_type\":  trial.suggest_categorical(\"boosting_type\", [\"gbdt\"]),\n",
    "        \"lambda_l1\": trial.suggest_float(\"lambda_l1\", 1e-8, 10.0, log=True),\n",
    "        \"lambda_l2\": trial.suggest_float(\"lambda_l2\", 1e-8, 10.0, log=True),\n",
    "        \"num_leaves\": trial.suggest_int(\"num_leaves\", 2, 256),\n",
    "        \"feature_fraction\": trial.suggest_float(\"feature_fraction\", 0.4, 1.0),\n",
    "        \"bagging_fraction\": trial.suggest_float(\"bagging_fraction\", 0.4, 1.0),\n",
    "        \"bagging_freq\": trial.suggest_int(\"bagging_freq\", 1, 7),\n",
    "        \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 5, 100),\n",
    "    }\n",
    "\n",
    "    gbm = lgb.train(param, dtrain)\n",
    "    preds = gbm.predict(valid_x)\n",
    "    pred_labels = np.rint(preds)\n",
    "    auc = sklearn.metrics.roc_auc_score(valid_y, pred_labels)\n",
    "    return auc\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    study = optuna.create_study(direction=\"maximize\", sampler = TPESampler(seed=31))\n",
    "    study.optimize(objective, n_trials=100)\n",
    "\n",
    "    print(\"Number of finished trials: {}\".format(len(study.trials)))\n",
    "\n",
    "    print(\"Best trial:\")\n",
    "    trial = study.best_trial\n",
    "\n",
    "    print(\"  Value: {}\".format(trial.value))\n",
    "\n",
    "    print(\"  Params: \")\n",
    "    for key, value in trial.params.items():\n",
    "        print(\"    {}: {}\".format(key, value))\n",
    "\n",
    "    lgb_params = {}\n",
    "\n",
    "    for key, value in trial.params.items():\n",
    "        lgb_params[key] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 214,
     "status": "ok",
     "timestamp": 1691163170760,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "h7P7Mw7UkC-y",
    "outputId": "df1e98c8-4f65-4841-9dce-f36ddccab569"
   },
   "outputs": [],
   "source": [
    "#See LightGBM hyperparameters.\n",
    "\n",
    "lgb_params['metric'] = 'binary_logloss'\n",
    "lgb_params['verbosity'] = -1\n",
    "lgb_params['random_state'] = 31\n",
    "\n",
    "print(lgb_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 247
    },
    "executionInfo": {
     "elapsed": 195,
     "status": "ok",
     "timestamp": 1691163170760,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "UAidUSs1IQPh",
    "outputId": "354738b7-465a-42f3-da2c-4dd1e7d2e22e"
   },
   "outputs": [],
   "source": [
    "#Fit LightGBM.\n",
    "\n",
    "import lightgbm as lgb\n",
    "\n",
    "lgb = lgb.LGBMClassifier(**lgb_params)\n",
    "\n",
    "lgb.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 111
    },
    "executionInfo": {
     "elapsed": 187,
     "status": "ok",
     "timestamp": 1691163170760,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "JlKe1z0RIQPh",
    "outputId": "0f3319e5-2edd-4e89-ab64-303de72d1f59"
   },
   "outputs": [],
   "source": [
    "#Predict on the validation set, get predicted probabilities for calibration, and fit calibration function.\n",
    "\n",
    "calib_probs_lgb = lgb.predict_proba(valid_x)\n",
    "\n",
    "calib_model_lgb = LogisticRegression()\n",
    "calib_model_lgb.fit(calib_probs_lgb, valid_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 182,
     "status": "ok",
     "timestamp": 1691163170760,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "mPyYlwGX-HkZ",
    "outputId": "c2274b8e-121c-4a83-dd31-972eb9ba5f6e"
   },
   "outputs": [],
   "source": [
    "#Make predictions on the test set based on the trained and calibrated LightGBM model.\n",
    "\n",
    "preds_lgb = lgb.predict(x_test)\n",
    "\n",
    "uncalibrated_probs_lgb = lgb.predict_proba(x_test)\n",
    "uncalibrated_probs_lgb = uncalibrated_probs_lgb\n",
    "\n",
    "probs_lgb = calib_model_lgb.predict_proba(uncalibrated_probs_lgb)\n",
    "probs_lgb = probs_lgb[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2716,
     "status": "ok",
     "timestamp": 1691163173298,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "r259oDdIIQPi",
    "outputId": "41f71f70-e4a5-4e1d-87e1-d2ac61d8befd"
   },
   "outputs": [],
   "source": [
    "#Evaluate LightGBM model.\n",
    "\n",
    "lgb_tn, lgb_fp, lgb_fn, lgb_tp = confusion_matrix(y_test, preds_lgb).ravel()\n",
    "lgb_cm = confusion_matrix(y_test, preds_lgb)\n",
    "\n",
    "lgb_sensitivity = round(lgb_tp/(lgb_tp+lgb_fn), 3)\n",
    "lgb_sensitivity_ci_length = z_value * np.sqrt((lgb_sensitivity * (1 - lgb_sensitivity)) / y_test.shape[0])\n",
    "lgb_sensitivity_ci_lower = round((lgb_sensitivity - lgb_sensitivity_ci_length), 3)\n",
    "lgb_sensitivity_ci_upper = round((lgb_sensitivity + lgb_sensitivity_ci_length), 3)\n",
    "lgb_sensitivity_str = str(lgb_sensitivity) + ' (' + str(lgb_sensitivity_ci_lower) + ' - ' + str(lgb_sensitivity_ci_upper) + ')'\n",
    "\n",
    "lgb_specificity = round(lgb_tn/(lgb_tn+lgb_fp), 3)\n",
    "lgb_specificity_ci_length = z_value * np.sqrt((lgb_specificity * (1 - lgb_specificity)) / y_test.shape[0])\n",
    "lgb_specificity_ci_lower = round((lgb_specificity - lgb_specificity_ci_length), 3)\n",
    "lgb_specificity_ci_upper = round((lgb_specificity + lgb_specificity_ci_length), 3)\n",
    "lgb_specificity_str = str(lgb_specificity) + ' (' + str(lgb_specificity_ci_lower) + ' - ' + str(lgb_specificity_ci_upper) + ')'\n",
    "\n",
    "lgb_auprc = round(average_precision_score(y_test, probs_lgb), 3)\n",
    "lgb_auprc_ci_length = z_value * np.sqrt((lgb_auprc * (1 - lgb_auprc)) / y_test.shape[0])\n",
    "lgb_auprc_ci_lower = round((lgb_auprc - lgb_auprc_ci_length), 3)\n",
    "lgb_auprc_ci_upper = round((lgb_auprc + lgb_auprc_ci_length), 3)\n",
    "lgb_auprc_str = str(lgb_auprc) + ' (' + str(lgb_auprc_ci_lower) + ' - ' + str(lgb_auprc_ci_upper) + ')'\n",
    "\n",
    "lgb_accuracy = round(accuracy_score(y_test, preds_lgb), 3)\n",
    "lgb_accuracy_ci_length = z_value * np.sqrt((lgb_accuracy * (1 - lgb_accuracy)) / y_test.shape[0])\n",
    "lgb_accuracy_ci_lower = round((lgb_accuracy - lgb_accuracy_ci_length), 3)\n",
    "lgb_accuracy_ci_upper = round((lgb_accuracy + lgb_accuracy_ci_length), 3)\n",
    "lgb_accuracy_str = str(lgb_accuracy) + ' (' + str(lgb_accuracy_ci_lower) + ' - ' + str(lgb_accuracy_ci_upper) + ')'\n",
    "\n",
    "lgb_auroc, lgb_auroc_ci_lower, lgb_auroc_ci_upper = auroc_ci(y_test, probs_lgb)\n",
    "lgb_auroc = round(lgb_auroc, 3)\n",
    "lgb_auroc_ci_lower = round(lgb_auroc_ci_lower, 3)\n",
    "lgb_auroc_ci_upper = round(lgb_auroc_ci_upper, 3)\n",
    "lgb_auroc_str = str(lgb_auroc) + ' (' + str(lgb_auroc_ci_lower) + ' - ' + str(lgb_auroc_ci_upper) + ')'\n",
    "\n",
    "lgb_brier = round(brier_score_loss(y_test, probs_lgb), 3)\n",
    "lgb_brier_ci_length = z_value * np.sqrt((lgb_brier * (1 - lgb_brier)) / y_test.shape[0])\n",
    "lgb_brier_ci_lower = round((lgb_brier - lgb_brier_ci_length), 3)\n",
    "lgb_brier_ci_upper = round((lgb_brier + lgb_brier_ci_length), 3)\n",
    "lgb_brier_str = str(lgb_brier) + ' (' + str(lgb_brier_ci_lower) + ' - ' + str(lgb_brier_ci_upper) + ')'\n",
    "\n",
    "lgb_results = [lgb_sensitivity_str, lgb_specificity_str, lgb_auprc_str, lgb_accuracy_str, lgb_auroc_str, lgb_brier_str]\n",
    "\n",
    "print(\"Sensitivity: \", (lgb_sensitivity_str))\n",
    "print(\"Specificity: \", (lgb_specificity_str))\n",
    "print('AUPRC: ', (lgb_auprc_str))\n",
    "print('Accuracy: ', (lgb_accuracy_str))\n",
    "print('AUROC: ', (lgb_auroc_str))\n",
    "print('Brier Score: ', (lgb_brier_str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 22,
     "status": "ok",
     "timestamp": 1691163173299,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "PXzXI9O-IQPi",
    "outputId": "2fe13617-d2c6-4112-c6ae-484bae173f08"
   },
   "outputs": [],
   "source": [
    "#Recalculate precision recall curve for plotting purposes.\n",
    "\n",
    "lgb_precision_curve, lgb_recall_curve, _ = precision_recall_curve(y_test, probs_lgb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mQRRrBp-IQPk"
   },
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true,
     "base_uri": "https://localhost:8080/"
    },
    "id": "kFTmQMQxIQPk",
    "outputId": "1f53674c-307f-48c0-f334-8b4d8dee3849"
   },
   "outputs": [],
   "source": [
    "#Hyperparameter tuning for Random Forest.\n",
    "\n",
    "def objective(trial):\n",
    "\n",
    "    param = {\n",
    "        \"criterion\": trial.suggest_categorical(\"criterion\", [\"gini\", \"entropy\"]),\n",
    "        \"random_state\": 31,\n",
    "        \"max_features\": trial.suggest_categorical(\"max_features\", [\"auto\", \"sqrt\",\"log2\", None]),\n",
    "        \"max_depth\": trial.suggest_int(\"max_depth\", 1, 100),\n",
    "        \"n_estimators\": trial.suggest_int(\"n_estimators\", 100, 2000, 100),\n",
    "        \"min_samples_leaf\": trial.suggest_int(\"min_samples_leaf\", 1, 4, 1),\n",
    "        \"min_samples_split\": trial.suggest_int(\"min_samples_split\", 2, 10, 1),\n",
    "    }\n",
    "\n",
    "    rf = RandomForestClassifier(**param)\n",
    "\n",
    "    rf.fit(\n",
    "        train_x,\n",
    "        train_y,\n",
    "    )\n",
    "\n",
    "    preds = rf.predict(valid_x)\n",
    "    pred_labels = np.rint(preds)\n",
    "    auc = sklearn.metrics.roc_auc_score(valid_y, pred_labels)\n",
    "\n",
    "    return auc\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    study = optuna.create_study(direction='maximize', sampler = TPESampler(seed=31))\n",
    "    study.optimize(objective, n_trials=100, timeout=600)\n",
    "\n",
    "    print(\"Number of finished trials: {}\".format(len(study.trials)))\n",
    "\n",
    "    print(\"Best trial:\")\n",
    "    trial = study.best_trial\n",
    "\n",
    "    print(\"  Value: {}\".format(trial.value))\n",
    "\n",
    "    print(\"  Params: \")\n",
    "    for key, value in trial.params.items():\n",
    "        print(\"    {}: {}\".format(key, value))\n",
    "\n",
    "    rf_params = {}\n",
    "\n",
    "    for key, value in trial.params.items():\n",
    "        rf_params[key] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "yTqnQTzgkG3R",
    "outputId": "ce90159d-dba0-49b0-f958-2bb71bc8f625"
   },
   "outputs": [],
   "source": [
    "#See Random Forest hyperparameters.\n",
    "\n",
    "rf_params['random_state'] = 31\n",
    "\n",
    "print(rf_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "o2nnbDAsIQPk",
    "outputId": "f5ce4113-1d52-4452-8832-5d7eaf14d1e3"
   },
   "outputs": [],
   "source": [
    "#Fit Random Forest.\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "rf = RandomForestClassifier(**rf_params)\n",
    "\n",
    "rf.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "pSsBliFuIQPk",
    "outputId": "0b5d5be7-7717-4dfd-d0bc-1cd93fc2d4c0"
   },
   "outputs": [],
   "source": [
    "#Predict on the validation set, get predicted probabilities for calibration, and fit calibration function.\n",
    "\n",
    "calib_probs_rf = rf.predict_proba(valid_x)\n",
    "\n",
    "calib_model_rf = LogisticRegression()\n",
    "calib_model_rf.fit(calib_probs_rf, valid_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "o3Ru1zLF-YER",
    "outputId": "131cda56-394e-4f86-a53e-ae9a03718c0e"
   },
   "outputs": [],
   "source": [
    "#Make predictions on the test set based on the trained Random Forest model.\n",
    "\n",
    "preds_rf = rf.predict(x_test)\n",
    "\n",
    "uncalibrated_probs_rf = rf.predict_proba(x_test)\n",
    "\n",
    "probs_rf = calib_model_rf.predict_proba(uncalibrated_probs_rf)\n",
    "probs_rf = probs_rf[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "haQcIGjoIQPk",
    "outputId": "93fe703b-ea9c-4324-91be-cd5cdea9935c"
   },
   "outputs": [],
   "source": [
    "#Evaluate Random Forest model.\n",
    "\n",
    "rf_tn, rf_fp, rf_fn, rf_tp = confusion_matrix(y_test, preds_rf).ravel()\n",
    "rf_cm = confusion_matrix(y_test, preds_rf)\n",
    "\n",
    "rf_sensitivity = round(rf_tp/(rf_tp+rf_fn), 3)\n",
    "rf_sensitivity_ci_length = z_value * np.sqrt((rf_sensitivity * (1 - rf_sensitivity)) / y_test.shape[0])\n",
    "rf_sensitivity_ci_lower = round((rf_sensitivity - rf_sensitivity_ci_length), 3)\n",
    "rf_sensitivity_ci_upper = round((rf_sensitivity + rf_sensitivity_ci_length), 3)\n",
    "rf_sensitivity_str = str(rf_sensitivity) + ' (' + str(rf_sensitivity_ci_lower) + ' - ' + str(rf_sensitivity_ci_upper) + ')'\n",
    "\n",
    "rf_specificity = round(rf_tn/(rf_tn+rf_fp), 3)\n",
    "rf_specificity_ci_length = z_value * np.sqrt((rf_specificity * (1 - rf_specificity)) / y_test.shape[0])\n",
    "rf_specificity_ci_lower = round((rf_specificity - rf_specificity_ci_length), 3)\n",
    "rf_specificity_ci_upper = round((rf_specificity + rf_specificity_ci_length), 3)\n",
    "rf_specificity_str = str(rf_specificity) + ' (' + str(rf_specificity_ci_lower) + ' - ' + str(rf_specificity_ci_upper) + ')'\n",
    "\n",
    "rf_auprc = round(average_precision_score(y_test, probs_rf), 3)\n",
    "rf_auprc_ci_length = z_value * np.sqrt((rf_auprc * (1 - rf_auprc)) / y_test.shape[0])\n",
    "rf_auprc_ci_lower = round((rf_auprc - rf_auprc_ci_length), 3)\n",
    "rf_auprc_ci_upper = round((rf_auprc + rf_auprc_ci_length), 3)\n",
    "rf_auprc_str = str(rf_auprc) + ' (' + str(rf_auprc_ci_lower) + ' - ' + str(rf_auprc_ci_upper) + ')'\n",
    "\n",
    "rf_accuracy = round(accuracy_score(y_test, preds_rf), 3)\n",
    "rf_accuracy_ci_length = z_value * np.sqrt((rf_accuracy * (1 - rf_accuracy)) / y_test.shape[0])\n",
    "rf_accuracy_ci_lower = round((rf_accuracy - rf_accuracy_ci_length), 3)\n",
    "rf_accuracy_ci_upper = round((rf_accuracy + rf_accuracy_ci_length), 3)\n",
    "rf_accuracy_str = str(rf_accuracy) + ' (' + str(rf_accuracy_ci_lower) + ' - ' + str(rf_accuracy_ci_upper) + ')'\n",
    "\n",
    "rf_auroc, rf_auroc_ci_lower, rf_auroc_ci_upper = auroc_ci(y_test, probs_rf)\n",
    "rf_auroc = round(rf_auroc, 3)\n",
    "rf_auroc_ci_lower = round(rf_auroc_ci_lower, 3)\n",
    "rf_auroc_ci_upper = round(rf_auroc_ci_upper, 3)\n",
    "rf_auroc_str = str(rf_auroc) + ' (' + str(rf_auroc_ci_lower) + ' - ' + str(rf_auroc_ci_upper) + ')'\n",
    "\n",
    "rf_brier = round(brier_score_loss(y_test, probs_rf), 3)\n",
    "rf_brier_ci_length = z_value * np.sqrt((rf_brier * (1 - rf_brier)) / y_test.shape[0])\n",
    "rf_brier_ci_lower = round((rf_brier - rf_brier_ci_length), 3)\n",
    "rf_brier_ci_upper = round((rf_brier + rf_brier_ci_length), 3)\n",
    "rf_brier_str = str(rf_brier) + ' (' + str(rf_brier_ci_lower) + ' - ' + str(rf_brier_ci_upper) + ')'\n",
    "\n",
    "rf_results = [rf_sensitivity_str, rf_specificity_str, rf_auprc_str, rf_accuracy_str, rf_auroc_str, rf_brier_str]\n",
    "\n",
    "print(\"Sensitivity: \", (rf_sensitivity_str))\n",
    "print(\"Specificity: \", (rf_specificity_str))\n",
    "print('AUPRC: ', (rf_auprc_str))\n",
    "print('Accuracy: ', (rf_accuracy_str))\n",
    "print('AUROC: ', (rf_auroc_str))\n",
    "print('Brier Score: ', (rf_brier_str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "Drrp3KEQIQPk",
    "outputId": "19f8405e-0e2f-4eca-f596-0822ae64209d"
   },
   "outputs": [],
   "source": [
    "#Recalculate precision recall curve for plotting purposes.\n",
    "\n",
    "rf_precision_curve, rf_recall_curve, _ = precision_recall_curve(y_test, probs_rf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "fLvNgketIQPl"
   },
   "source": [
    "# ROC and PR Curves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "0mMqO-euIQPl",
    "outputId": "a61278e3-fa7c-47d1-b27f-280b95e6ebd0"
   },
   "outputs": [],
   "source": [
    "#Plot ROC curve.\n",
    "\n",
    "f = pyplot.figure()\n",
    "f.set_figwidth(12)\n",
    "f.set_figheight(12)\n",
    "\n",
    "tabpfn_fpr, tabpfn_tpr, _ = roc_curve(y_test, probs_tabpfn)\n",
    "tabpfn_label = 'TabPFN AUROC: ' + tabpfn_auroc_str\n",
    "pyplot.plot(tabpfn_fpr, tabpfn_tpr, label = tabpfn_label, color = 'b', linewidth = 3.5, alpha = 0.75)\n",
    "\n",
    "tabnet_fpr, tabnet_tpr, _ = roc_curve(y_test, probs_tabnet)\n",
    "tabnet_label = 'TabNet AUROC: ' + tabnet_auroc_str\n",
    "pyplot.plot(tabnet_fpr, tabnet_tpr, label = tabnet_label, color = 'g', linewidth = 3.5, alpha = 0.75)\n",
    "\n",
    "xgb_fpr, xgb_tpr, _ = roc_curve(y_test, probs_xgb)\n",
    "xgb_label = 'XGBoost AUROC: ' + xgb_auroc_str\n",
    "pyplot.plot(xgb_fpr, xgb_tpr, label = xgb_label, color = 'r', linewidth = 3.5, alpha = 0.75)\n",
    "\n",
    "lgb_fpr, lgb_tpr, _ = roc_curve(y_test, probs_lgb)\n",
    "lgb_label = 'LightGBM AUROC: ' + lgb_auroc_str\n",
    "pyplot.plot(lgb_fpr, lgb_tpr, label = lgb_label, color='c', linewidth = 3.5, alpha = 0.75)\n",
    "\n",
    "rf_fpr, rf_tpr, _ = roc_curve(y_test, probs_rf)\n",
    "rf_label = 'Random Forest AUROC: ' + rf_auroc_str\n",
    "pyplot.plot(rf_fpr, rf_tpr, label = rf_label, color = 'm', linewidth = 3.5, alpha = 0.75)\n",
    "\n",
    "pyplot.plot([0, 1], [0, 1], linestyle = '--', linewidth=2)\n",
    "\n",
    "pyplot.title('A', x = -0.075, y = 1.005, fontsize = 75, pad = 20)\n",
    "pyplot.xlabel('False Positive Rate', fontsize = 22, fontweight = 'heavy', labelpad = 16)\n",
    "pyplot.ylabel('True Positive Rate', fontsize = 22, fontweight = 'heavy', labelpad = 16)\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", labelsize = 16)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", labelsize = 16)\n",
    "\n",
    "leg = pyplot.legend(loc = 'lower right', fontsize = 18)\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-Meningioma/12m_roc.png', dpi=300)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "GpotrYFBIQPl",
    "outputId": "bbba8bd0-a262-42c6-8322-fd6c476cc53c"
   },
   "outputs": [],
   "source": [
    "#Plot PR curve.\n",
    "\n",
    "f = pyplot.figure()\n",
    "f.set_figwidth(12)\n",
    "f.set_figheight(12)\n",
    "\n",
    "tabpfn_label = 'TabPFN AUPRC: ' + tabpfn_auprc_str\n",
    "tabnet_label = 'TabNet AUPRC: ' + tabnet_auprc_str\n",
    "xgb_label = 'XGBoost AUPRC: ' + xgb_auprc_str\n",
    "lgb_label = 'LightGBM AUPRC: ' + lgb_auprc_str\n",
    "rf_label = 'Random Forest AUPRC: ' + rf_auprc_str\n",
    "\n",
    "pyplot.plot(tabpfn_recall_curve, tabpfn_precision_curve, label = tabpfn_label, color = 'b', linewidth = 3.5, alpha = 0.75)\n",
    "pyplot.plot(tabnet_recall_curve, tabnet_precision_curve, label = tabnet_label, color = 'g', linewidth = 3.5, alpha = 0.75)\n",
    "pyplot.plot(xgb_recall_curve, xgb_precision_curve, label = xgb_label, color = 'r', linewidth = 3.5, alpha = 0.75)\n",
    "pyplot.plot(lgb_recall_curve, lgb_precision_curve, label = lgb_label, color = 'c', linewidth = 3.5, alpha = 0.75)\n",
    "pyplot.plot(rf_recall_curve, rf_precision_curve, label = rf_label, color = 'm', linewidth = 3.5, alpha = 0.75)\n",
    "\n",
    "pyplot.title('A', x = -0.075, y = 1.005, fontsize = 75, pad = 20)\n",
    "pyplot.xlabel('Recall', fontsize = 18, fontweight = 'heavy', labelpad = 16)\n",
    "pyplot.ylabel('Precision', fontsize = 22, fontweight = 'heavy', labelpad = 16)\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", labelsize = 16)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", labelsize = 16)\n",
    "\n",
    "leg = pyplot.legend(loc = 'upper right', fontsize = 18)\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-Meningioma/12m_prc.png', dpi=300)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rOySRdcDIQPl"
   },
   "source": [
    "# Results Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "kznujF4kIQPl",
    "outputId": "e87ad4c8-51d1-415b-f677-a173f9f12748"
   },
   "outputs": [],
   "source": [
    "results = {'TabPFN':tabpfn_results, 'TabNet':tabnet_results, 'XGBoost':xgb_results, 'LightGBM':lgb_results, 'Random Forest':rf_results}\n",
    "\n",
    "results = pd.DataFrame(results, columns = ['TabPFN', 'TabNet', 'XGBoost', 'LightGBM', 'Random Forest'])\n",
    "results = results.T\n",
    "\n",
    "results.columns = ['Sensitivity (95% CI)', 'Specificity (95% CI)',  'AUPRC (95% CI)', 'Accuracy (95% CI)', 'AUROC (95% CI)', 'Brier Score (95% CI)']\n",
    "\n",
    "results.to_csv('/content/drive/MyDrive/NCDB-Meningioma/12m_results.csv')\n",
    "\n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uAD5jLGchIp4"
   },
   "source": [
    "# Confusion Matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 716
    },
    "executionInfo": {
     "elapsed": 2154,
     "status": "ok",
     "timestamp": 1691164692565,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "BDEMrQ5khLfg",
    "outputId": "ccf74271-fe30-4c50-a218-8904d0c8b37b"
   },
   "outputs": [],
   "source": [
    "tabpfn_cm = confusion_matrix(y_test, preds_tabpfn)\n",
    "\n",
    "f = pyplot.figure(figsize=(8,8))\n",
    "\n",
    "sns.heatmap(tabpfn_cm, annot=True, fmt='d', cmap='Blues', cbar=False, annot_kws={\"size\": 16}, linewidths=1, linecolor='black')\n",
    "\n",
    "labels = ['Survival', 'Mortality']\n",
    "pyplot.xticks([0.5,1.5], labels, fontsize=16, fontweight='heavy')\n",
    "pyplot.yticks([0.5,1.5], labels, fontsize=16, fontweight='heavy', va='center')\n",
    "\n",
    "pyplot.xlabel('Predicted', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "pyplot.ylabel('Truth', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", pad=10)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", pad=10)\n",
    "pyplot.title('A', x = -0.095, y = 1.005, fontsize = 75, pad = 20)\n",
    "\n",
    "pyplot.subplots_adjust(left=0.20, right=0.85, bottom=0.20, top=0.80)\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-Meningioma/12m_cm_tabpfn.png', dpi=300)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 753
    },
    "executionInfo": {
     "elapsed": 1190,
     "status": "ok",
     "timestamp": 1691164693744,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "MRXGPobHhNEX",
    "outputId": "876c7d36-202d-484d-80d2-532f796257e0"
   },
   "outputs": [],
   "source": [
    "tabnet_cm = confusion_matrix(y_test, preds_tabnet)\n",
    "\n",
    "f = pyplot.figure(figsize=(8,8))\n",
    "\n",
    "sns.heatmap(tabnet_cm, annot=True, fmt='d', cmap='Blues', cbar=False, annot_kws={\"size\": 16}, linewidths=1, linecolor='black')\n",
    "\n",
    "labels = ['Survival', 'Mortality']\n",
    "pyplot.xticks([0.5,1.5], labels, fontsize=16, fontweight='heavy')\n",
    "pyplot.yticks([0.5,1.5], labels, fontsize=16, fontweight='heavy', va='center')\n",
    "\n",
    "pyplot.xlabel('Predicted', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "pyplot.ylabel('Truth', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", pad=10)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", pad=10)\n",
    "pyplot.title('B', x = -0.095, y = 1.005, fontsize = 75, pad = 20)\n",
    "\n",
    "pyplot.subplots_adjust(left=0.20, right=0.85, bottom=0.20, top=0.80)\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-Meningioma/12m_cm_tabnet.png', dpi=300)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 753
    },
    "executionInfo": {
     "elapsed": 1079,
     "status": "ok",
     "timestamp": 1691164694814,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "xLZLslcChShm",
    "outputId": "330675e9-a43b-42b4-b0b1-8b2f383d807d"
   },
   "outputs": [],
   "source": [
    "xgb_cm = confusion_matrix(y_test, preds_xgb)\n",
    "\n",
    "f = pyplot.figure(figsize=(8,8))\n",
    "\n",
    "sns.heatmap(xgb_cm, annot=True, fmt='d', cmap='Blues', cbar=False, annot_kws={\"size\": 16}, linewidths=1, linecolor='black')\n",
    "\n",
    "labels = ['Survival', 'Mortality']\n",
    "pyplot.xticks([0.5,1.5], labels, fontsize=16, fontweight='heavy')\n",
    "pyplot.yticks([0.5,1.5], labels, fontsize=16, fontweight='heavy', va='center')\n",
    "\n",
    "pyplot.xlabel('Predicted', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "pyplot.ylabel('Truth', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", pad=10)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", pad=10)\n",
    "pyplot.title('C', x = -0.095, y = 1.005, fontsize = 75, pad = 20)\n",
    "\n",
    "pyplot.subplots_adjust(left=0.20, right=0.85, bottom=0.20, top=0.80)\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-Meningioma/12m_cm_xgb.png', dpi=300)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 753
    },
    "executionInfo": {
     "elapsed": 1413,
     "status": "ok",
     "timestamp": 1691164696222,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "QMNzhoAAhUnc",
    "outputId": "a7139480-7aeb-44e2-8f49-342cce4b489e"
   },
   "outputs": [],
   "source": [
    "lgb_cm = confusion_matrix(y_test, preds_lgb)\n",
    "\n",
    "f = pyplot.figure(figsize=(8,8))\n",
    "\n",
    "sns.heatmap(lgb_cm, annot=True, fmt='d', cmap='Blues', cbar=False, annot_kws={\"size\": 16}, linewidths=1, linecolor='black')\n",
    "\n",
    "labels = ['Survival', 'Mortality']\n",
    "pyplot.xticks([0.5,1.5], labels, fontsize=16, fontweight='heavy')\n",
    "pyplot.yticks([0.5,1.5], labels, fontsize=16, fontweight='heavy', va='center')\n",
    "\n",
    "pyplot.xlabel('Predicted', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "pyplot.ylabel('Truth', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", pad=10)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", pad=10)\n",
    "pyplot.title('D', x = -0.095, y = 1.005, fontsize = 75, pad = 20)\n",
    "\n",
    "pyplot.subplots_adjust(left=0.20, right=0.85, bottom=0.20, top=0.80)\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-Meningioma/12m_cm_lgb.png', dpi=300)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 753
    },
    "executionInfo": {
     "elapsed": 2079,
     "status": "ok",
     "timestamp": 1691164698292,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "-Rn8yc5QhWqo",
    "outputId": "655427f8-ad62-4489-f3af-3c0b6bf94b01"
   },
   "outputs": [],
   "source": [
    "rf_cm = confusion_matrix(y_test, preds_rf)\n",
    "\n",
    "f = pyplot.figure(figsize=(8,8))\n",
    "\n",
    "sns.heatmap(rf_cm, annot=True, fmt='d', cmap='Blues', cbar=False, annot_kws={\"size\": 16}, linewidths=1, linecolor='black')\n",
    "\n",
    "labels = ['Survival', 'Mortality']\n",
    "pyplot.xticks([0.5,1.5], labels, fontsize=16, fontweight='heavy')\n",
    "pyplot.yticks([0.5,1.5], labels, fontsize=16, fontweight='heavy', va='center')\n",
    "\n",
    "pyplot.xlabel('Predicted', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "pyplot.ylabel('Truth', fontsize=22, fontweight='heavy', labelpad=16)\n",
    "\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", pad=10)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", pad=10)\n",
    "pyplot.title('C', x = -0.095, y = 1.005, fontsize = 75, pad = 20)\n",
    "\n",
    "pyplot.subplots_adjust(left=0.20, right=0.85, bottom=0.20, top=0.80)\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-Meningioma/12m_cm_rf.png', dpi=300)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qYnu4PnCIQPm"
   },
   "source": [
    "# SHAP Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 17,
     "status": "ok",
     "timestamp": 1691164698292,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "BuvljQiYu-Xb",
    "outputId": "e1df210e-6292-4b42-811b-b4600307b2ac"
   },
   "outputs": [],
   "source": [
    "import textwrap\n",
    "def wrap_labels(ax, width, break_long_words=False):\n",
    "    labels = []\n",
    "    for label in ax.get_yticklabels():\n",
    "        text = label.get_text()\n",
    "        labels.append(textwrap.fill(text, width=width,\n",
    "                                    break_long_words=break_long_words))\n",
    "    ax.set_yticklabels(labels, rotation=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 13,
     "status": "ok",
     "timestamp": 1691164698292,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "gjL0ZhOZJh-b"
   },
   "outputs": [],
   "source": [
    "feature_names = x_test.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 676538,
     "status": "ok",
     "timestamp": 1691165374818,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "GTKndsLJzH7h",
    "outputId": "44c15f20-dd5f-4208-e4ad-cefc453a43c0"
   },
   "outputs": [],
   "source": [
    "#Calculate SHAP values for TabPFN.\n",
    "\n",
    "tabpfn_explainer = shap.Explainer(tabpfn.predict, x_test)\n",
    "tabpfn_shap_values = tabpfn_explainer(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 2169,
     "status": "ok",
     "timestamp": 1691165376968,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "Lw5zXxxQzH7p",
    "outputId": "3f4b62e5-a0eb-440d-eafe-d2503ae7ec26"
   },
   "outputs": [],
   "source": [
    "#Plot SHAP bar plot for TabPFN.\n",
    "\n",
    "shap.plots.bar(tabpfn_shap_values, max_display = 15, show=False)\n",
    "\n",
    "fig = pyplot.gcf()\n",
    "ax = pyplot.gca()\n",
    "fig.set_figheight(12)\n",
    "fig.set_figwidth(5)\n",
    "\n",
    "pyplot.title('A', x = -0.5, y = 1, fontsize = 50, pad = 20)\n",
    "pyplot.xlabel(\"Mean |SHAP Value|\", fontsize =12, fontweight = 'heavy', labelpad = 8)\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", labelsize = 12)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", labelsize = 12)\n",
    "\n",
    "wrap_labels(ax, 30)\n",
    "ax.figure\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-Meningioma/12m_shap_tabpfn.png', dpi=300, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 19835,
     "status": "ok",
     "timestamp": 1691165396796,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "XotWpfehXbCw",
    "outputId": "91626b34-d132-48e0-c9c9-74d100293294"
   },
   "outputs": [],
   "source": [
    "#Calculate SHAP values for TabNet.\n",
    "\n",
    "tabnet_explainer = shap.Explainer(tabnet.predict, x_test.values, feature_names=feature_names)\n",
    "tabnet_shap_values = tabnet_explainer(x_test.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 2066,
     "status": "ok",
     "timestamp": 1691165398850,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "tzHasDCwXoxs",
    "outputId": "b70c6f1c-9a3b-4b5f-9dc3-419b7c8c689f"
   },
   "outputs": [],
   "source": [
    "#Plot SHAP bar plot for TabNet.\n",
    "\n",
    "shap.plots.bar(tabnet_shap_values, max_display = 15, show=False)\n",
    "\n",
    "fig = pyplot.gcf()\n",
    "ax = pyplot.gca()\n",
    "fig.set_figheight(12)\n",
    "fig.set_figwidth(5)\n",
    "\n",
    "pyplot.title('B', x = -0.5, y = 1, fontsize = 50, pad = 20)\n",
    "pyplot.xlabel(\"Mean |SHAP Value|\", fontsize =12, fontweight = 'heavy', labelpad = 8)\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", labelsize = 12)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", labelsize = 12)\n",
    "\n",
    "wrap_labels(ax, 30)\n",
    "ax.figure\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-Meningioma/12m_shap_tabnet.png', dpi=300, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 45961,
     "status": "ok",
     "timestamp": 1691165444803,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "C09esVZrIQPm",
    "outputId": "7d3831ae-1c03-4f8a-84df-1182c881c278"
   },
   "outputs": [],
   "source": [
    "#Calculate SHAP values for XGBoost.\n",
    "\n",
    "xgb_explainer = shap.Explainer(xgb.predict, x_test)\n",
    "xgb_shap_values = xgb_explainer(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 2037,
     "status": "ok",
     "timestamp": 1691165446828,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "6O74ff97qW2H",
    "outputId": "8bcf21f0-d1e9-4b68-ab03-584e11dc32c8"
   },
   "outputs": [],
   "source": [
    "#Plot SHAP bar plot for XGBoost.\n",
    "\n",
    "shap.plots.bar(xgb_shap_values, max_display = 15, show=False)\n",
    "\n",
    "fig = pyplot.gcf()\n",
    "ax = pyplot.gca()\n",
    "fig.set_figheight(12)\n",
    "fig.set_figwidth(5)\n",
    "\n",
    "pyplot.title('C', x = -0.5, y = 1, fontsize = 50, pad = 20)\n",
    "pyplot.xlabel(\"Mean |SHAP Value|\", fontsize =12, fontweight = 'heavy', labelpad = 8)\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", labelsize = 12)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", labelsize = 12)\n",
    "\n",
    "wrap_labels(ax, 30)\n",
    "ax.figure\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-Meningioma/12m_shap_xgb.png', dpi=300, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 56919,
     "status": "ok",
     "timestamp": 1691165503740,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "HBQmzC4-IQPm",
    "outputId": "3c28a4b3-90bc-4e69-b305-60614e15562f"
   },
   "outputs": [],
   "source": [
    "#Calculate SHAP values for LightGBM.\n",
    "\n",
    "lgb_explainer = shap.Explainer(lgb.predict, x_test)\n",
    "lgb_shap_values = lgb_explainer(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 2261,
     "status": "ok",
     "timestamp": 1691165505981,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "4V-NEfx_xysV",
    "outputId": "9df523df-d188-4964-edb1-e4b61ffd81da"
   },
   "outputs": [],
   "source": [
    "#Plot SHAP bar plot for LightGBM.\n",
    "\n",
    "shap.plots.bar(lgb_shap_values, max_display = 15, show=False)\n",
    "\n",
    "fig = pyplot.gcf()\n",
    "ax = pyplot.gca()\n",
    "fig.set_figheight(12)\n",
    "fig.set_figwidth(5)\n",
    "\n",
    "pyplot.title('D', x = -0.5, y = 1, fontsize = 50, pad = 20)\n",
    "pyplot.xlabel(\"Mean |SHAP Value|\", fontsize =12, fontweight = 'heavy', labelpad = 8)\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", labelsize = 12)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", labelsize = 12)\n",
    "\n",
    "wrap_labels(ax, 30)\n",
    "ax.figure\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-Meningioma/12m_shap_lgb.png', dpi=300, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1079540,
     "status": "ok",
     "timestamp": 1691166585513,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "Vj3B1IeWIQPm",
    "outputId": "61a05833-79ad-44ff-a2a3-fb056ccef4ab"
   },
   "outputs": [],
   "source": [
    "#Calculate SHAP values for Random Forest.\n",
    "\n",
    "rf_explainer = shap.Explainer(rf.predict, x_test)\n",
    "rf_shap_values = rf_explainer(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 1857,
     "status": "ok",
     "timestamp": 1691166587343,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "x2P2or1bx25u",
    "outputId": "766431e0-8759-40ec-bf2f-1ea933728399"
   },
   "outputs": [],
   "source": [
    "#Plot SHAP bar plot for Random Forest.\n",
    "\n",
    "shap.plots.bar(rf_shap_values, max_display = 15, show=False)\n",
    "\n",
    "fig = pyplot.gcf()\n",
    "ax = pyplot.gca()\n",
    "fig.set_figheight(12)\n",
    "fig.set_figwidth(5)\n",
    "\n",
    "pyplot.title('C', x = -0.5, y = 1, fontsize = 50, pad = 20)\n",
    "pyplot.xlabel(\"Mean |SHAP Value|\", fontsize =12, fontweight = 'heavy', labelpad = 8)\n",
    "pyplot.tick_params(axis=\"y\",direction=\"out\", labelsize = 12)\n",
    "pyplot.tick_params(axis=\"x\",direction=\"out\", labelsize = 12)\n",
    "\n",
    "wrap_labels(ax, 30)\n",
    "ax.figure\n",
    "\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-Meningioma/12m_shap_rf.png', dpi=300, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cwYwpeSZz_-x"
   },
   "source": [
    "#Partial Dependency Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 7,
     "status": "ok",
     "timestamp": 1691166587343,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "cCezSsG3XK1c"
   },
   "outputs": [],
   "source": [
    "column_filters = {\n",
    "    'Age': (18, 90),\n",
    "}\n",
    "\n",
    "for column, (min_val, max_val) in column_filters.items():\n",
    "    if column in x_test.columns:\n",
    "        x_test = x_test[(x_test[column] > min_val) & (x_test[column] < max_val)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 7,
     "status": "ok",
     "timestamp": 1691166587344,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "jLWR5lmo01KM"
   },
   "outputs": [],
   "source": [
    "pyplot.rcParams[\"figure.figsize\"] = (15, 15)\n",
    "pyplot.rcParams[\"figure.dpi\"] = 300\n",
    "pyplot.rcParams['axes.labelweight'] = 'bold'\n",
    "pyplot.rcParams['axes.labelsize'] = 12\n",
    "pyplot.rcParams['axes.labelpad'] = 6\n",
    "pyplot.rcParams['font.weight'] = 'normal'\n",
    "pyplot.rcParams['lines.linewidth'] = 2.5\n",
    "pyplot.rcParams['xtick.labelsize'] = 8\n",
    "pyplot.rcParams['ytick.labelsize'] = 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 7,
     "status": "ok",
     "timestamp": 1691166587344,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "4mCUeepoYdNJ"
   },
   "outputs": [],
   "source": [
    "categorical_features = ['Histology', 'Grade', 'Facility Type', 'Facility Location', 'Sex', 'Race', 'Hispanic Ethnicity',\n",
    "                        'Primary Payor', 'Percent No High School Education Quartiles', 'Census Median Income Quartiles',\n",
    "                        'Rurality', 'Charlson-Deyo Score', 'Laterality', 'Diagnostic Biopsy',\n",
    "                        'Ki-67 Labeling Index', 'Karnofsky Performance Scale', 'Focality', 'Tumor Size',\n",
    "                        'Extent of Resection', 'Resective Surgery', 'Radiation Treatment']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 20893,
     "status": "ok",
     "timestamp": 1691166608231,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "UNB6VbOg5KKx",
    "outputId": "88054555-0f14-4a5d-9e79-8b9a66451ed9"
   },
   "outputs": [],
   "source": [
    "#Calculate the mean absolute SHAP values for each feature.\n",
    "tabpfn_mean_abs_shap_values = np.mean(np.abs(tabpfn_shap_values.values), axis=0)\n",
    "\n",
    "#Create a DataFrame to map feature names to their mean absolute SHAP values.\n",
    "tabpfn_shap_summary = pd.DataFrame(list(zip(feature_names, tabpfn_mean_abs_shap_values)), columns=['Feature', 'Mean SHAP'])\n",
    "\n",
    "#Sort the DataFrame by 'Mean SHAP' in descending order.\n",
    "tabpfn_shap_summary_sorted = tabpfn_shap_summary.sort_values('Mean SHAP', ascending=False)\n",
    "\n",
    "#Get the names of the features.\n",
    "tabpfn_features = tabpfn_shap_summary_sorted['Feature'].tolist()\n",
    "tabpfn_features = tabpfn_features[:9]\n",
    "\n",
    "tabpfn_categorical_features = [item for item in tabpfn_features if item in categorical_features]\n",
    "\n",
    "PartialDependenceDisplay.from_estimator(tabpfn, x_test.sample(frac=0.05, random_state=42), tabpfn_features, categorical_features = tabpfn_categorical_features)\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-Meningioma/12m_pdp_tabpfn.png', dpi=300, bbox_inches='tight')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 8247,
     "status": "ok",
     "timestamp": 1691166616364,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "bLDT7TOZ5JFi",
    "outputId": "3ec0a309-5f64-47f4-f8e6-5073bb97b802"
   },
   "outputs": [],
   "source": [
    "#Calculate the mean absolute SHAP values for each feature.\n",
    "tabnet_mean_abs_shap_values = np.mean(np.abs(tabnet_shap_values.values), axis=0)\n",
    "\n",
    "#Create a DataFrame to map feature names to their mean absolute SHAP values.\n",
    "tabnet_shap_summary = pd.DataFrame(list(zip(feature_names, tabnet_mean_abs_shap_values)), columns=['Feature', 'Mean SHAP'])\n",
    "\n",
    "#Sort the DataFrame by 'Mean SHAP' in descending order.\n",
    "tabnet_shap_summary_sorted = tabnet_shap_summary.sort_values('Mean SHAP', ascending=False)\n",
    "\n",
    "#Get the names of the features.\n",
    "tabnet_features = tabnet_shap_summary_sorted['Feature'].tolist()\n",
    "tabnet_features = tabnet_features[:9]\n",
    "\n",
    "tabnet_categorical_features = [item for item in tabnet_features if item in categorical_features]\n",
    "\n",
    "PartialDependenceDisplay.from_estimator(xgb, x_test, tabnet_features, categorical_features = tabnet_categorical_features)\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-Meningioma/12m_pdp_tabnet.png', dpi=300, bbox_inches='tight')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 5754,
     "status": "ok",
     "timestamp": 1691166622111,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "frN2YQUD0IWP",
    "outputId": "59def893-d196-419f-cd35-361027a356e1"
   },
   "outputs": [],
   "source": [
    "#Calculate the mean absolute SHAP values for each feature.\n",
    "xgb_mean_abs_shap_values = np.mean(np.abs(xgb_shap_values.values), axis=0)\n",
    "\n",
    "#Create a DataFrame to map feature names to their mean absolute SHAP values.\n",
    "xgb_shap_summary = pd.DataFrame(list(zip(feature_names, xgb_mean_abs_shap_values)), columns=['Feature', 'Mean SHAP'])\n",
    "\n",
    "#Sort the DataFrame by 'Mean SHAP' in descending order.\n",
    "xgb_shap_summary_sorted = xgb_shap_summary.sort_values('Mean SHAP', ascending=False)\n",
    "\n",
    "#Get the names of the features.\n",
    "xgb_features = xgb_shap_summary_sorted['Feature'].tolist()\n",
    "xgb_features = xgb_features[:9]\n",
    "\n",
    "xgb_categorical_features = [item for item in xgb_features if item in categorical_features]\n",
    "\n",
    "PartialDependenceDisplay.from_estimator(xgb, x_test, xgb_features, categorical_features = xgb_categorical_features)\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-Meningioma/12m_pdp_xgb.png', dpi=300, bbox_inches='tight')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 6640,
     "status": "ok",
     "timestamp": 1691166628636,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "P9ZSSzpP5MbC",
    "outputId": "e996f239-9845-42cb-ce47-31093fcb9a0d"
   },
   "outputs": [],
   "source": [
    "#Calculate the mean absolute SHAP values for each feature.\n",
    "lgb_mean_abs_shap_values = np.mean(np.abs(lgb_shap_values.values), axis=0)\n",
    "\n",
    "#Create a DataFrame to map feature names to their mean absolute SHAP values.\n",
    "lgb_shap_summary = pd.DataFrame(list(zip(feature_names, lgb_mean_abs_shap_values)), columns=['Feature', 'Mean SHAP'])\n",
    "\n",
    "#Sort the DataFrame by 'Mean SHAP' in descending order.\n",
    "lgb_shap_summary_sorted = lgb_shap_summary.sort_values('Mean SHAP', ascending=False)\n",
    "\n",
    "#Get the names of the features.\n",
    "lgb_features = lgb_shap_summary_sorted['Feature'].tolist()\n",
    "lgb_features = lgb_features[:9]\n",
    "\n",
    "lgb_categorical_features = [item for item in lgb_features if item in categorical_features]\n",
    "\n",
    "PartialDependenceDisplay.from_estimator(lgb, x_test, lgb_features, categorical_features = lgb_categorical_features)\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-Meningioma/12m_pdp_lgb.png', dpi=300, bbox_inches='tight')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 12924,
     "status": "ok",
     "timestamp": 1691166641444,
     "user": {
      "displayName": "Mert Karabacak",
      "userId": "14308977132914475412"
     },
     "user_tz": 240
    },
    "id": "aq_m3WU_5NKq",
    "outputId": "72b3702f-328b-4e6f-fa79-d40cbf003ebc"
   },
   "outputs": [],
   "source": [
    "#Calculate the mean absolute SHAP values for each feature.\n",
    "rf_mean_abs_shap_values = np.mean(np.abs(rf_shap_values.values), axis=0)\n",
    "\n",
    "#Create a DataFrame to map feature names to their mean absolute SHAP values.\n",
    "rf_shap_summary = pd.DataFrame(list(zip(feature_names, rf_mean_abs_shap_values)), columns=['Feature', 'Mean SHAP'])\n",
    "\n",
    "#Sort the DataFrame by 'Mean SHAP' in descending order.\n",
    "rf_shap_summary_sorted = rf_shap_summary.sort_values('Mean SHAP', ascending=False)\n",
    "\n",
    "#Get the names of the features.\n",
    "rf_features = rf_shap_summary_sorted['Feature'].tolist()\n",
    "rf_features = rf_features[:9]\n",
    "\n",
    "rf_categorical_features = [item for item in rf_features if item in categorical_features]\n",
    "\n",
    "PartialDependenceDisplay.from_estimator(rf, x_test, rf_features, categorical_features = rf_categorical_features)\n",
    "pyplot.savefig('/content/drive/MyDrive/NCDB-Meningioma/12m_pdp_rf.png', dpi=300, bbox_inches='tight')\n",
    "pyplot.show()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "machine_shape": "hm",
   "provenance": [
    {
     "file_id": "14TfLVbewZLcEADsU9RM-JZ4Jrc-obgN-",
     "timestamp": 1690322786628
    },
    {
     "file_id": "1Eo4jPVLNOwgUyqgXVFTcpiRbGrIuXwk5",
     "timestamp": 1690313290635
    },
    {
     "file_id": "1ehzQDERWA7oATVlIze0f5fvBuuSk3Il-",
     "timestamp": 1690235650728
    },
    {
     "file_id": "10kjTiT6ZbSYDMQVY6jE58AJ9bNIvu_Z4",
     "timestamp": 1680804894233
    },
    {
     "file_id": "1SyxRVArbYdS2kIxHOoZLuxtWYNsZL3ei",
     "timestamp": 1675315289522
    },
    {
     "file_id": "1rquc9s2K5gcWtyx-P9n0ie3ooY4kzveD",
     "timestamp": 1673820526724
    },
    {
     "file_id": "1sd9nBAoznz2E976naQ2pJvARRLH-vRK7",
     "timestamp": 1673714326538
    },
    {
     "file_id": "1tydPpyRVpKMFnUtRbvvIdQTTUtRQ1SJK",
     "timestamp": 1672252113955
    },
    {
     "file_id": "1FJ2Tp28HBJo_krRjesYEkutbjOcsodmu",
     "timestamp": 1672251686516
    },
    {
     "file_id": "1vmOc15wf8HLrh9xcXe9IRvqZIrBNUFp_",
     "timestamp": 1671647832509
    },
    {
     "file_id": "1jvG_2brh3p2BhjDqh6LIyEOlVCFKYSX6",
     "timestamp": 1671034103968
    },
    {
     "file_id": "1YKiMGVsPQlDRLgB2d3Svjjsp96BkJxlX",
     "timestamp": 1670597416903
    },
    {
     "file_id": "1gG7ScMqQA2JnmntazVsJaQStNW3qHl9R",
     "timestamp": 1670198888608
    },
    {
     "file_id": "1V4GD1xQ2LBQhBny_OS3iBlGyn9mh7oWM",
     "timestamp": 1669993880849
    },
    {
     "file_id": "1euhY_qXOTy9qT_vn0Q6Kb2eEUOCHom8m",
     "timestamp": 1669310009337
    },
    {
     "file_id": "1ksEkZ_gSIaosRd56LkYx54z5oGNBgq2C",
     "timestamp": 1668779215067
    },
    {
     "file_id": "1o0f_u5loubhXZTvFpGUCA_m2gpTKQ6EL",
     "timestamp": 1668619772114
    }
   ],
   "toc_visible": true
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}