Run_Example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from Geo_cal_utils import GeometricCalibrator,ECE_calc\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.metrics import accuracy_score\n",
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = make_classification(\n",
    "    n_samples=1000,  # row number\n",
    "    n_features=900, # because our calibrator is working on images the number of features need to simulate image, thus the number we choose should hav square root.\n",
    "    n_informative=6, # The number of informative features\n",
    "    n_classes = 2, # The number of classes\n",
    "    random_state = 42 # random seed \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.25, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "model = RandomForestClassifier().fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "0.83"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_test = model.predict(X_test)\n",
    "y_test_probs = model.predict_proba(X_test)\n",
    "accuracy_score(y_test, y_pred_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:00<00:00, 571.49it/s]\n",
      "100%|██████████| 200/200 [00:00<00:00, 627.09it/s]\n"
     ]
    }
   ],
   "source": [
    "# Fast separation calibration\n",
    "GeoCalibrator = GeometricCalibrator(model, X_train, y_train)\n",
    "GeoCalibrator.fit(X_val, y_val)\n",
    "calibrated_prob_Geo = GeoCalibrator.calibrate(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:00<00:00, 1019.53it/s]\n",
      "100%|██████████| 200/200 [00:00<00:00, 1035.58it/s]\n"
     ]
    }
   ],
   "source": [
    "# Fast separation calibration -compressed\n",
    "GeoCalibrator_compressed = GeometricCalibrator(model, X_train, y_train, comprasion_mode='Maxpool',\n",
    "                                               compression_param=2)\n",
    "GeoCalibrator_compressed.fit(X_val, y_val)\n",
    "calibrated_prob_GeoCompressed = GeoCalibrator_compressed.calibrate(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Geometric Calibration Fast separation ECE: \t0.0369\n",
      "Geometric Calibration Fast separation ECE: \t0.0548\n",
      "No Calibration ECE: \t0.2369\n"
     ]
    }
   ],
   "source": [
    "# After Calibration\n",
    "print(f'Geometric Calibration Fast separation ECE: \\t{(ECE_calc(calibrated_prob_Geo, y_pred_test, y_test)):.4f}')\n",
    "print(f'Geometric Calibration Fast separation ECE: \\t{(ECE_calc(calibrated_prob_GeoCompressed, y_pred_test, y_test)):.4f}')\n",
    "print(f'No Calibration ECE: \\t{(ECE_calc(y_test_probs, y_pred_test, y_test)):.4f}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}