Introdução-Prática-Previsão-Conforme.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Introdução Prática à Previsão Conforme\n",
    "\n",
    "Previsão Conforme (em inglês Conformal Prediction) é um método que transforma as previsões pontuais em conjuntos de previsões. Dessa forma, fornece previsões associadas à um nível de confiança estatística, permitindo que sejam medidos e controlados os riscos de uma previsão estar errada.\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importar Libs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install -U numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install mapie -use-deprecated=legacy-resolver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import os\n",
    "import wget\n",
    "import zipfile\n",
    "from os.path import exists\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "pd.set_option('display.float_format', lambda x: '%.3f' % x)\n",
    "pd.options.display.max_rows=50\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import style\n",
    "import seaborn as sns\n",
    "style.use(\"fivethirtyeight\")\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "from mapie.classification import MapieClassifier"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "bean_data_file = \"./DryBeanDataset/Dry_Bean_Dataset.xlsx\"\n",
    "base = \"https://archive.ics.uci.edu/ml/machine-learning-databases/\"\n",
    "dataset_number = \"00602\"\n",
    "filename = \"DryBeanDataset.zip\"\n",
    "url = base + dataset_number + \"/\" + filename\n",
    "wget.download(url)\n",
    "zipfile.ZipFile(filename, 'r') .extractall('./')\n",
    "os.remove(filename)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importar Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "bean_data_file = \"./DryBeanDataset/Dry_Bean_Dataset.xlsx\"\n",
    "beans = pd.read_excel(bean_data_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Area</th>\n",
       "      <th>Perimeter</th>\n",
       "      <th>MajorAxisLength</th>\n",
       "      <th>MinorAxisLength</th>\n",
       "      <th>AspectRation</th>\n",
       "      <th>Eccentricity</th>\n",
       "      <th>ConvexArea</th>\n",
       "      <th>EquivDiameter</th>\n",
       "      <th>Extent</th>\n",
       "      <th>Solidity</th>\n",
       "      <th>roundness</th>\n",
       "      <th>Compactness</th>\n",
       "      <th>ShapeFactor1</th>\n",
       "      <th>ShapeFactor2</th>\n",
       "      <th>ShapeFactor3</th>\n",
       "      <th>ShapeFactor4</th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>28395</td>\n",
       "      <td>610.291</td>\n",
       "      <td>208.178</td>\n",
       "      <td>173.889</td>\n",
       "      <td>1.197</td>\n",
       "      <td>0.550</td>\n",
       "      <td>28715</td>\n",
       "      <td>190.141</td>\n",
       "      <td>0.764</td>\n",
       "      <td>0.989</td>\n",
       "      <td>0.958</td>\n",
       "      <td>0.913</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.834</td>\n",
       "      <td>0.999</td>\n",
       "      <td>SEKER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>28734</td>\n",
       "      <td>638.018</td>\n",
       "      <td>200.525</td>\n",
       "      <td>182.734</td>\n",
       "      <td>1.097</td>\n",
       "      <td>0.412</td>\n",
       "      <td>29172</td>\n",
       "      <td>191.273</td>\n",
       "      <td>0.784</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.887</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.004</td>\n",
       "      <td>0.910</td>\n",
       "      <td>0.998</td>\n",
       "      <td>SEKER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>29380</td>\n",
       "      <td>624.110</td>\n",
       "      <td>212.826</td>\n",
       "      <td>175.931</td>\n",
       "      <td>1.210</td>\n",
       "      <td>0.563</td>\n",
       "      <td>29690</td>\n",
       "      <td>193.411</td>\n",
       "      <td>0.778</td>\n",
       "      <td>0.990</td>\n",
       "      <td>0.948</td>\n",
       "      <td>0.909</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.826</td>\n",
       "      <td>0.999</td>\n",
       "      <td>SEKER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>30008</td>\n",
       "      <td>645.884</td>\n",
       "      <td>210.558</td>\n",
       "      <td>182.517</td>\n",
       "      <td>1.154</td>\n",
       "      <td>0.499</td>\n",
       "      <td>30724</td>\n",
       "      <td>195.467</td>\n",
       "      <td>0.783</td>\n",
       "      <td>0.977</td>\n",
       "      <td>0.904</td>\n",
       "      <td>0.928</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.862</td>\n",
       "      <td>0.994</td>\n",
       "      <td>SEKER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>30140</td>\n",
       "      <td>620.134</td>\n",
       "      <td>201.848</td>\n",
       "      <td>190.279</td>\n",
       "      <td>1.061</td>\n",
       "      <td>0.334</td>\n",
       "      <td>30417</td>\n",
       "      <td>195.897</td>\n",
       "      <td>0.773</td>\n",
       "      <td>0.991</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.004</td>\n",
       "      <td>0.942</td>\n",
       "      <td>0.999</td>\n",
       "      <td>SEKER</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Area  Perimeter  MajorAxisLength  MinorAxisLength  AspectRation  \\\n",
       "0  28395    610.291          208.178          173.889         1.197   \n",
       "1  28734    638.018          200.525          182.734         1.097   \n",
       "2  29380    624.110          212.826          175.931         1.210   \n",
       "3  30008    645.884          210.558          182.517         1.154   \n",
       "4  30140    620.134          201.848          190.279         1.061   \n",
       "\n",
       "   Eccentricity  ConvexArea  EquivDiameter  Extent  Solidity  roundness  \\\n",
       "0         0.550       28715        190.141   0.764     0.989      0.958   \n",
       "1         0.412       29172        191.273   0.784     0.985      0.887   \n",
       "2         0.563       29690        193.411   0.778     0.990      0.948   \n",
       "3         0.499       30724        195.467   0.783     0.977      0.904   \n",
       "4         0.334       30417        195.897   0.773     0.991      0.985   \n",
       "\n",
       "   Compactness  ShapeFactor1  ShapeFactor2  ShapeFactor3  ShapeFactor4  Class  \n",
       "0        0.913         0.007         0.003         0.834         0.999  SEKER  \n",
       "1        0.954         0.007         0.004         0.910         0.998  SEKER  \n",
       "2        0.909         0.007         0.003         0.826         0.999  SEKER  \n",
       "3        0.928         0.007         0.003         0.862         0.994  SEKER  \n",
       "4        0.971         0.007         0.004         0.942         0.999  SEKER  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "beans.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pre-Process and Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Labels are characters but should be integers for sklearn\n",
    "le = LabelEncoder()\n",
    "beans[\"Class\"] = le.fit_transform(beans[\"Class\"])\n",
    "\n",
    "# Split data into classification target and features\n",
    "y = beans[\"Class\"]\n",
    "X = beans.drop(\"Class\", axis = 1)\n",
    "\n",
    "# Split of training data\n",
    "X_train, X_rest1, y_train, y_rest1 = train_test_split(X, y, train_size=10000, random_state=2)\n",
    "\n",
    "# From the remaining data, split of test data\n",
    "X_test, X_rest2, y_test, y_rest2 = train_test_split(X_rest1, y_rest1, train_size=1000, random_state=42)\n",
    "\n",
    "# Split remaining into calibration and \"new\" data\n",
    "X_calib, X_new, y_calib, y_new = train_test_split(X_rest2, y_rest2, train_size=1000, random_state=42)\n",
    "\n",
    "# Fit the model\n",
    "model = GaussianNB().fit(X_train, y_train)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.758\n",
      "          BARBUNYA  BOMBAY  CALI  DERMASON  HOROZ  SEKER  SIRA\n",
      "BARBUNYA        46       0    47         0      6      0     4\n",
      "BOMBAY           0      33     0         0      0      0     0\n",
      "CALI            20       0    81         0      3      0     0\n",
      "DERMASON         0       0     0       223      0     32     9\n",
      "HOROZ            0       0     4         3    104      0    22\n",
      "SEKER            2       0     0        26      1    127    22\n",
      "SIRA             0       0     0        10     10     21   144\n"
     ]
    }
   ],
   "source": [
    "# Check accuracy\n",
    "y_pred = model.predict(X_test)\n",
    "print(\"Accuracy:\", (y_pred == y_test).mean())\n",
    "\n",
    "# Create the confusion matrix\n",
    "cm = confusion_matrix(y_test, y_pred)\n",
    "print(pd.DataFrame(cm, index=le.classes_, columns=le.classes_))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Calibration - Naive Approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.896\n"
     ]
    }
   ],
   "source": [
    "#Get the \"probabilities\" from the model\n",
    "predictions = model.predict_proba(X_calib)\n",
    "\n",
    "# Get for each instance the highest probability\n",
    "high_prob_predictions = np.amax(predictions, axis=1)\n",
    "\n",
    "# Select the predictions where probability over 99%\n",
    "high_p_beans = np.where(high_prob_predictions >= 0.95)\n",
    "\n",
    "# Let's count how often we hit the right label\n",
    "its_a_match = (model.predict(X_calib) == y_calib)\n",
    "coverage = np.mean(its_a_match.values[high_p_beans])\n",
    "print(round(coverage, 3))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Calibration - Conformal Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA58AAAHtCAYAAACeZMqfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAAsTAAALEwEAmpwYAAAkrElEQVR4nO3de7hlV1kn6t9HAoQAJkAknU6iQSnAAIJpArTQEgmXBGmCttJwREIMYj+H44FDewSVbrFtD+AFGk4DRgkQaJHriUQMCgSQjnIJBIwQxF1ykRSBACaFEBIEvvPHmpVelnVZVbXHWrt2ve/zrGfPOebtW6tGqvLbY86xqrsDAAAAI91s1QUAAACw+QmfAAAADCd8AgAAMJzwCQAAwHDCJwAAAMMJnwAAAAwnfALAAaiqE6vqkqr6WlWt5PvLquqVVfWOVVwbABYlfAKwLqYA1FX1Gzu1nzC1n7bEWu5eVa+uqm1VdWNVfaaqLqyqHx5wuV9Kcsck905y3IDzL+KpSX5ix0pVvayq3r0eJ66qO1TVi6rqU9Nn+cWq+p9V9bj1OD8Ah47DV10AAJvKDUn+z6p6cXd/ZhUFVNXDk/xhkr9I8qQkf5Pk1knOTHJekrus8yW3JPlAd68dyEmq6ubd/Y/7c2x3bz+Qa+/Fm5IcneRnk3wiyTFJ7pfkDgOvmaq6RXd/Y+Q1AFguI58ArKe/SPKXSf6fPe1UVXetqj+uqq9Orz+qqjvPbX9iVX2zqh5QVZdX1fVV9aGqOnUv5z0yyauSvLu7T+/ut3b333b3Fd39vMxC0459j6uq11bVdVX19ap6d1XdZ277adOI7UOr6j1TDVdW1Zlz+3SS05P89LTvK/fx3D9SVZdW1Q1JnrTj9tmq+rmqumr6bF5WVTevqv8wjeBeW1W/W1W3mDvfTbfdVtWzk5yb5EHTNXr6PF9ZVW/bxWf2zqo6fzef59FJHpTkWd39tu7+THd/qLtf0t3/fad9nzJ9PjdW1TVV9aa5bbetqvOmUdMbq+qDVfWwue0nTXX+ZFVdXFVfS/Jr07bHVtVHquqGqvp0VT2/qm69+14AwEYlfAKwnjrJzyd53HzYmldVt0rytiRHZBZsHpTkNkn+ZD5QZfZv1HMyu6X0lCTXJHl9Ve3prp2HZXYL7K/vsrjua6caKrPR0bsleWSS+yb5QpK3V9UxOx32W5mF6XsleX+S11XV7aZtxyV5b5LXTMtP3cdz/3aS5yX5viR/NLXdN8l9kjw0yeOSPD7JRUl+MMkZ0/pPZRYwd+W3pnreO9V0XJLXZTbq+5CqutOOHafAf1qS393Nub6a5B+SnLWnwFdVvzq9j5ckuedU5+Vzu7w8ycOn2u+d5M+TvKWq7rbTqZ6X5PeT3CPJ71TVE5O8NLPP6eQkT0jykCS/s7taANjAutvLy8vLy+uAX0lemeQd0/KFmY0+JskJmYXS06b1c5Ncn+SYuWOPTfL1JE+Y1p84HXPK3D73m9ruuocafmHa5/Z7qfX0ab+T59pumeTqJP95Wj9t2ufHdqqzkzx8ru3dSV62n+f+qV18htckucVc2x8n+VKSW861vTnJG3f12U/rL9vx+e90/iuS/Ne59eck+cu9fFY/Ol3/G0k+mOSFSR48t/3W05/dz+/m+DtP7/URO7VfnuTl0/JJ0z7/aad9Pp3kP+zU9kPTvrdbdZ/38vLy8tq3l5FPAEZ4RpIHVNWjdrHt7kmu7O4v7Wjo7i9k9jzh3ef268xu4d3hc9PPY5Okqt46d9vuV6dttWB9d0/y5e6+cq6GGzMb2bz7Tvt+ZKc6v7WjhnU49wd2cfzH+58+6/j5JJ+YzjHfdsc91LA75yU5p6oOm0aQn5jk9/Z0QHdfmOT4zEYz35TZCOQlVfXiaZe7ZzaK/c9u6Z2cPP18z07t78kePo+q+s4k353k+Tv9Ob912uXOAeCgYsIhANZdd/9NVZ2X2W2UZ+5t/934dnd/a/60088dvzh9UpJb7XTMJ6afJye5dD+vu7NdTXqzXr+8/dou2naedKh307Y/Nbw6sz+TH5mOPyrJ/9jbQVPwfef0ek5VPSvJr1XVb+5HDXsy/3nseH9PTfKuXex71TpfG4DBjHwCMMqvJvmXSZ68U/vHkpw8//xjVR2b5K5JPrroybt7W3dv3fGamt+W2W2rv7yrY+ae1fxYkjtU1clz226Z2a29C9ewGyPPvahvJDls58bu/kqS1yb5men1hu6+bj/O//Hp53cmuTKzWY4ftpt9Pzb9/KGd2n8oe/g8plHmz2Z2m/XWXbxu2I+6AVgh4ROAIbr7i0mem+RpO216TZIvZjZxzylV9a8yC0TbMpsY50CueX1mt5L+8DRr7JlV9T1Vdc+q+vkk75t2fWdmt3i+ZppR9x6ZzZJ7RGYT3ByIkede1KeS3K1m33d6zBR+dzgvs9Hoh2f3Ew0luek7Pt9dVWdX1b2nWWkfmdmzop9K8pHu/mpmEwI9e5rx9i5Vda+q+sUk6e6/TfKGJC+pqodX1d2q6oWZTSq0t5HTX87sq3t+uaruUbNZkh89jaoDcJARPgEY6QWZTVZzk+7+emajZDdm9tzfn2V2u+UZvQ7f69jdb01yamYzzJ6f5K8zm7TnBzMb7Ut3d5JHz227LMm/SPLQ+WdR9/P6w869D86frvsXmQX9x83Vd1mSv8rsOdI/38t5vjqd4ymZheqPJ3nRtPyg/l/fS/qfMgXFzEYz35bZDMU7PCnJn2Z2i+9fJnlAkkd291/v6eLd/eokj8ls1uAPTO/p2Zn9ogKAg0zN/o0EAA4FVXXzzGaR/Y3ufuGKywHgEGLCIQA4BFTVzZIck+RnM/t6lFestiIADjXCJwAcGr4rs+c0r07y09PkQwCwNG67BQAAYLiDauRz+/btkjIAAMBB4Kijjqr5dbPdAgAAMJzwCQAAwHDC5zpaW1tbdQkcwvQ/VkXfY5X0P1ZF32OVDtb+J3wCAAAwnPAJAADAcMInAAAAwwmfAAAADCd8AgAAMJzwCQAAwHDCJwAAAMMJnwAAAAwnfAIAADCc8AkAAMBwwicAAADDCZ8AAAAMJ3wCAAAwnPAJAADAcMInAAAAwwmfAAAADCd8AgAAMJzwCQAAwHCHr7qAzeTUS49MLt22lGtdd87xS7kOAADAejDyCQAAwHDCJwAAAMMJnwAAAAwnfAIAADCc8AkAAMBwwicAAADDCZ8AAAAMJ3wCAAAwnPAJAADAcMInAAAAwwmfAAAADCd8AgAAMJzwCQAAwHDCJwAAAMMJnwAAAAwnfAIAADCc8AkAAMBwwicAAADDCZ8AAAAMNzR8VtWnq+qvquojVfXBqe32VfX2qlqbft5uaq+qelFVba2qK6rqlJG1AQAAsDzLGPn84e6+d3ffZ1p/ZpJLuntLkkum9SQ5M8mW6fXkJC9dQm0AAAAswSpuuz0ryQXT8gVJHj3X/qqeeV+So6vquBXUBwAAwDqr7h538qpPJbk2SSc5r7t/t6qu6+6jp+2V5NruPrqq3pLkud196bTtkiTP6O4P7jjf9u3bbyp2bW1tWN3769RLj1zatS574PVLuxYAAMAitmzZctPyUUcdVfPbDh987Qd297aqumOSt1fVX89v7O6uqv1Kv/NvasO4dNvSLrUh3z8rtba2pl+wEvoeq6T/sSr6Hqt0sPa/obfddve26ec1SS5Mct8kX9hxO+3085pp921JTpw7/ISpDQAAgIPcsPBZVbeuqtvuWE7ysCQfTXJRkrOn3c5O8uZp+aIkT5hmvb1/ku3dffWo+gAAAFiekbfdHpvkwtljnTk8yWu6+0+q6rIkr6+qc5N8Jsljpv0vTvKIJFuTXJ/knIG1AQAAsETDwmd3fzLJvXbR/uUkp++ivZM8ZVQ9AAAArM4qvmoFAACAQ4zwCQAAwHDCJwAAAMMJnwAAAAwnfAIAADCc8AkAAMBwwicAAADDCZ8AAAAMJ3wCAAAwnPAJAADAcMInAAAAwwmfAAAADCd8AgAAMJzwCQAAwHDCJwAAAMMJnwAAAAwnfAIAADCc8AkAAMBwwicAAADDCZ8AAAAMJ3wCAAAwnPAJAADAcMInAAAAwwmfAAAADCd8AgAAMJzwCQAAwHDCJwAAAMMJnwAAAAwnfAIAADCc8AkAAMBwwicAAADDCZ8AAAAMJ3wCAAAwnPAJAADAcMInAAAAwwmfAAAADCd8AgAAMJzwCQAAwHDCJwAAAMMJnwAAAAwnfAIAADCc8AkAAMBwwicAAADDCZ8AAAAMJ3wCAAAwnPAJAADAcMInAAAAwwmfAAAADCd8AgAAMJzwCQAAwHDCJwAAAMMJnwAAAAwnfAIAADCc8AkAAMBwwicAAADDCZ8AAAAMJ3wCAAAwnPAJAADAcMInAAAAwwmfAAAADCd8AgAAMJzwCQAAwHDCJwAAAMMJnwAAAAwnfAIAADCc8AkAAMBwwicAAADDCZ8AAAAMJ3wCAAAw3PDwWVWHVdWHq+ot0/qdqur9VbW1ql5XVbeY2m85rW+dtp80ujYAAACWYxkjn09N8vG59ecleUF33znJtUnOndrPTXLt1P6CaT8AAAA2gaHhs6pOSPIjSV42rVeSByd547TLBUkePS2fNa1n2n76tD8AAAAHucMHn/+/JfmFJLed1u+Q5Lru/ua0flWS46fl45N8Nkm6+5tVtX3a/0u7OvHa2tqgkg/EkUu70sZ8/6yafsGq6Huskv7Hquh7rNJG7X9btmzZ7bZh4bOqHpnkmu7+UFWdtt7n39ObWplLty3tUhvy/bNSa2tr+gUroe+xSvofq6LvsUoHa/8bOfL5gCSPqqpHJDkiyXckeWGSo6vq8Gn084QkOxLbtiQnJrmqqg5PclSSLw+sDwAAgCUZ9sxnd/9id5/Q3ScleWySd3b3TyZ5V5Ifn3Y7O8mbp+WLpvVM29/Z3T2qPgAAAJZnFd/z+YwkT6+qrZk903n+1H5+kjtM7U9P8swV1AYAAMAAoyccSpJ097uTvHta/mSS++5inxuS/MQy6gEAAGC5VjHyCQAAwCFG+AQAAGA44RMAAIDhhE8AAACGEz4BAAAYTvgEAABgOOETAACA4YRPAAAAhhM+AQAAGE74BAAAYDjhEwAAgOGETwAAAIYTPgEAABhO+AQAAGA44RMAAIDhhE8AAACGEz4BAAAYTvgEAABgOOETAACA4YRPAAAAhhM+AQAAGE74BAAAYDjhEwAAgOGETwAAAIYTPgEAABhO+AQAAGA44RMAAIDhhE8AAACGEz4BAAAYTvgEAABgOOETAACA4YRPAAAAhhM+AQAAGE74BAAAYDjhEwAAgOGETwAAAIYTPgEAABhO+AQAAGA44RMAAIDhhE8AAACGEz4BAAAYTvgEAABgOOETAACA4YRPAAAAhhM+AQAAGE74BAAAYDjhEwAAgOGETwAAAIYTPgEAABhO+AQAAGA44RMAAIDhhE8AAACGEz4BAAAYTvgEAABgOOETAACA4YRPAAAAhhM+AQAAGE74BAAAYDjhEwAAgOGETwAAAIYTPgEAABhO+AQAAGA44RMAAIDhhE8AAACGEz4BAAAYTvgEAABguIXCZ1Xdc3QhAAAAbF6Ljny+pKo+UFX/e1UdNbQiAAAANp2Fwmd3/5skP5nkxCQfqqrXVNVDh1YGAADAprHwM5/dvZbkWUmekeRBSV5UVX9dVT+2q/2r6ohptPQvq+pjVfWrU/udqur9VbW1ql5XVbeY2m85rW+dtp90wO8OAACADWHRZz6/v6pekOTjSR6c5N929/dNyy/YzWE3Jnlwd98ryb2TnFFV90/yvCQv6O47J7k2ybnT/ucmuXZqf8G0HwAAAJvAoiOf/2+Sy5Pcq7uf0t2XJ0l3fy6z0dB/pme+Oq3efHp1ZoH1jVP7BUkePS2fNa1n2n56VdXibwUAAICNqrp77ztV3SbJ17v7W9P6zZIc0d3X7+W4w5J8KMmdk7w4yW8med80upmqOjHJW7v7HlX10SRndPdV07a/TXK/7v7SjvNt3779pmLX1tb26Y0uw6mXHrm0a132wD1+9AAAAEu3ZcuWm5aPOuqofzKYePiC53hHkock2TGSeWSStyX5wT0dNIXVe1fV0UkuTHK3Ba+3V/NvasO4dNvSLrUh3z8rtba2pl+wEvoeq6T/sSr6Hqt0sPa/RW+7PWLuFtpMywsP83X3dUneleRfJzm6qnaE3hOS7Ehs2zKbTTfT9qOSfHnRawAAALBxLRo+v1ZVp+xYqap/leTrezqgqr5zGvFMVd0qyUMzm7DoXUl+fNrt7CRvnpYvmtYzbX9nL3JPMAAAABveorfdPi3JG6rqc0kqyb9I8u/3csxxSS6Ynvu8WZLXd/dbqurKJK+tqv+a5MNJzp/2Pz/Jq6tqa5K/T/LYfXonAAAAbFgLhc/uvqyq7pbkrlPTJ7r7H/dyzBVJfmAX7Z9Mct9dtN+Q5CcWqQcAAICDy6Ijn0lyapKTpmNOqap096uGVAUAAMCmslD4rKpXJ/neJB9J8q2puZMInwAAAOzVoiOf90lysgmAAAAA2B+Lznb70cwmGQIAAIB9tujI5zFJrqyqDyS5cUdjdz9qSFUAAABsKouGz2ePLAIAAIDNbdGvWvmzqvruJFu6+x1VdWSSw8aWBgAAwGax0DOfVfUzSd6Y5Lyp6fgkfzioJgAAADaZRSccekqSByT5SpJ091qSO44qCgAAgM1l0fB5Y3d/Y8dKVR2e2fd8AgAAwF4tGj7/rKp+KcmtquqhSd6Q5I/GlQUAAMBmsmj4fGaSLyb5qyQ/m+TiJM8aVRQAAACby6Kz3X47ye9NLwAAANgnC4XPqvpUdvGMZ3d/z7pXBAAAwKazUPhMcp+55SOS/ESS269/OQAAAGxGCz3z2d1fnntt6+7/luRHxpYGAADAZrHobbenzK3eLLOR0EVHTQEAADjELRogf3tu+ZtJPp3kMeteDQAAAJvSorPd/vDoQgAAANi8Fr3t9ul72t7dz1+fcgAAANiM9mW221OTXDSt/9skH0iyNqIoAAAANpdFw+cJSU7p7n9Ikqp6dpI/7u7HjyoMAACAzWOhr1pJcmySb8ytf2NqAwAAgL1adOTzVUk+UFUXTuuPTnLBkIoAAADYdBad7fbXq+qtSf7N1HROd394XFkAAABsJovedpskRyb5Sne/MMlVVXWnQTUBAACwySwUPqvqV5I8I8kvTk03T/I/RhUFAADA5rLoyOePJnlUkq8lSXd/LsltRxUFAADA5rJo+PxGd3eSTpKquvW4kgAAANhsFg2fr6+q85IcXVU/k+QdSX5vXFkAAABsJnud7baqKsnrktwtyVeS3DXJf+7utw+uDQAAgE1ir+Gzu7uqLu7ueyYROAEAANhni952e3lVnTq0EgAAADatvY58Tu6X5PFV9enMZrytzAZFv39UYQAAAGweewyfVfVd3f13SR6+pHoAAADYhPY28vmHSU7p7s9U1Zu6+98toSYAAAA2mb0981lzy98zshAAAAA2r72Fz97NMgAAACxsb7fd3quqvpLZCOitpuXkf0049B1DqwMAAGBT2GP47O7DllUIAAAAm9ei3/MJAAAA+034BAAAYDjhEwAAgOGETwAAAIYTPgEAABhO+AQAAGA44RMAAIDhhE8AAACGEz4BAAAYTvgEAABgOOETAACA4YRPAAAAhhM+AQAAGE74BAAAYDjhEwAAgOGETwAAAIYTPgEAABhO+AQAAGA44RMAAIDhhE8AAACGEz4BAAAYTvgEAABgOOETAACA4YRPAAAAhhM+AQAAGE74BAAAYDjhEwAAgOGETwAAAIYTPgEAABhuWPisqhOr6l1VdWVVfayqnjq1376q3l5Va9PP203tVVUvqqqtVXVFVZ0yqjYAAACWa+TI5zeT/MfuPjnJ/ZM8papOTvLMJJd095Ykl0zrSXJmki3T68lJXjqwNgAAAJZoWPjs7qu7+/Jp+R+SfDzJ8UnOSnLBtNsFSR49LZ+V5FU9874kR1fVcaPqAwAAYHmW8sxnVZ2U5AeSvD/Jsd199bTp80mOnZaPT/LZucOumtoAAAA4yB0++gJVdZskb0rytO7+SlXdtK27u6p6f867tra2ThWupyOXdqWN+f5ZNf2CVdH3WCX9j1XR91iljdr/tmzZstttQ8NnVd08s+D5+939/03NX6iq47r76um22mum9m1JTpw7/ISpbZf29KZW5tLdlrvuNuT7Z6XW1tb0C1ZC32OV9D9WRd9jlQ7W/jdytttKcn6Sj3f38+c2XZTk7Gn57CRvnmt/wjTr7f2TbJ+7PRcAAICD2MiRzwck+akkf1VVH5nafinJc5O8vqrOTfKZJI+Ztl2c5BFJtia5Psk5A2sDAABgiYaFz+6+NEntZvPpu9i/kzxlVD0AAACszlJmuwUAAODQJnwCAAAwnPAJAADAcMInAAAAwwmfAAAADCd8AgAAMJzwCQAAwHDCJwAAAMMJnwAAAAwnfAIAADCc8AkAAMBwwicAAADDCZ8AAAAMJ3wCAAAwnPAJAADAcMInAAAAwwmfAAAADCd8AgAAMJzwCQAAwHDCJwAAAMMJnwAAAAwnfAIAADCc8AkAAMBwwicAAADDCZ8AAAAMJ3wCAAAwnPAJAADAcMInAAAAwwmfAAAADCd8AgAAMJzwCQAAwHDCJwAAAMMJnwAAAAwnfAIAADCc8AkAAMBwwicAAADDCZ8AAAAMJ3wCAAAwnPAJAADAcMInAAAAwwmfAAAADCd8AgAAMJzwCQAAwHDCJwAAAMMJnwAAAAwnfAIAADCc8AkAAMBwwicAAADDCZ8AAAAMJ3wCAAAwnPAJAADAcMInAAAAwwmfAAAADCd8AgAAMJzwCQAAwHDCJwAAAMMJnwAAAAwnfAIAADCc8AkAAMBwwicAAADDCZ8AAAAMJ3wCAAAwnPAJAADAcMInAAAAwwmfAAAADCd8AgAAMJzwCQAAwHDCJwAAAMMJnwAAAAx3+KoLAAAAONgd/YptS7vWZQ9c2qXW1bCRz6p6eVVdU1UfnWu7fVW9varWpp+3m9qrql5UVVur6oqqOmVUXQAAACzfyNtuX5nkjJ3anpnkku7ekuSSaT1JzkyyZXo9OclLB9YFAADAkg0Ln939niR/v1PzWUkumJYvSPLoufZX9cz7khxdVceNqg0AAIDlWvYzn8d299XT8ueTHDstH5/ks3P7XTW1XZ3dWFtbG1LggTlyaVfamO+fVdMvWBV9j1XS/1gVfY9/anlZINm4/W/Lli273bayCYe6u6uq9/f4Pb2plbl0eQ8Zb8j3z0qtra3pF6yEvscq6X+sir7HP7PELJAcnHlg2V+18oUdt9NOP6+Z2rclOXFuvxOmNgAAADaBZYfPi5KcPS2fneTNc+1PmGa9vX+S7XO35wIAAHCQG3bbbVX9QZLTkhxTVVcl+ZUkz03y+qo6N8lnkjxm2v3iJI9IsjXJ9UnOGVUXAAAAyzcsfHb343az6fRd7NtJnjKqFgAAAFZr2bfdAgAAcAgSPgEAABhO+AQAAGA44RMAAIDhhE8AAACGEz4BAAAYTvgEAABgOOETAACA4YRPAAAAhhM+AQAAGE74BAAAYDjhEwAAgOGETwAAAIYTPgEAABhO+AQAAGA44RMAAIDhhE8AAACGEz4BAAAYTvgEAABgOOETAACA4YRPAAAAhhM+AQAAGE74BAAAYDjhEwAAgOGETwAAAIYTPgEAABhO+AQAAGA44RMAAIDhhE8AAACGEz4BAAAYTvgEAABgOOETAACA4YRPAAAAhhM+AQAAGO7wVRfA/jn6FduWdq3rzjl+adcCAAA2JyOfAAAADCd8AgAAMJzwCQAAwHDCJwAAAMMJnwAAAAwnfAIAADCc8AkAAMBwwicAAADDCZ8AAAAMJ3wCAAAwnPAJAADAcMInAAAAwwmfAAAADCd8AgAAMJzwCQAAwHCHr7oANr6jX7FtKde57pzjl3IdAABg+Yx8AgAAMJyRTzaMZY2wJkZZAQBg2Yx8AgAAMJzwCQAAwHDCJwAAAMMJnwAAAAxnwiFgn5gYCgCA/SF8ckhaZoBalsseuOoKWITwDgDLtRn/v+9g5bZbAAAAhjPyCZvEqZcemVzqN3sAAGxMwicArJBbsQE4VLjtFgAAgOGMfAIAsGks626CZU70t6z35O4IRhM+ATapzfg/YAAbhbkWDowZaA9NwiewYXkW7uCwzP8B8+d0YDbj6Mmy+p++B3DghE8A2InfyAPA+hM+AQAOUZtxNBzYuIRPAIC9MBp+YHx+Bwd/Toy2ocJnVZ2R5IVJDkvysu5+7opLAg4R/sE9OPhzAoCD14YJn1V1WJIXJ3lokquSXFZVF3X3lautDADYF35JAMCubJjwmeS+SbZ29yeTpKpem+SsJAdN+PQ8AwAAwK7dbNUFzDk+yWfn1q+a2gAAADjIbaTwCQAAwCa1kW673ZbkxLn1E6a2mxx11FG11IoAAABYFxtp5POyJFuq6k5VdYskj01y0YprAgAAYB1smJHP7v5mVf0fSf40s69aeXl3f2zFZQEAALAONtLIZ7r74u6+S3d/b3f/+qrr2Z2qOqOqPlFVW6vqmbvYfsuqet20/f1VddIKymSTWqD/Pb2qrqyqK6rqkqr67lXUyeazt743t9+/q6quqvsssz42r0X6XlU9Zvq772NV9Zpl18jmtcC/u99VVe+qqg9P//Y+YhV1svlU1cur6pqq+uhutldVvWjqm1dU1SnLrnFfbajweTCY+z7SM5OcnORxVXXyTrudm+Ta7r5zkhcked5yq2SzWrD/fTjJfbr7+5O8MclvLLdKNqMF+16q6rZJnprk/cutkM1qkb5XVVuS/GKSB3T33ZM8bdl1sjkt+Hffs5K8vrt/ILPHxl6y3CrZxF6Z5Iw9bD8zyZbp9eQkL11CTQdE+Nx3N30faXd/I8mO7yOdd1aSC6blNyY5vapMlsR62Gv/6+53dff10+r7Mpu8Cw7UIn/3JcmvZfYLtxuWWRyb2iJ972eSvLi7r02S7r5myTWyeS3S/zrJd0zLRyX53BLrYxPr7vck+fs97HJWklf1zPuSHF1Vxy2nuv0jfO67Rb6P9KZ9uvubSbYnucNSqmOz29fvwz03yVuHVsShYq99b7rd58Tu/uNlFsamt8jfe3dJcpeq+vOqel9V7WmkAPbFIv3v2UkeX1VXJbk4yc8tpzTY5/8vXLkNM+EQsL6q6vFJ7pPkQauuhc2vqm6W5PlJnrjiUjg0HZ7ZbWenZXa3x3uq6p7dfd0qi+KQ8bgkr+zu366qf53k1VV1j+7+9qoLg43GyOe+2+v3kc7vU1WHZ3YLxpeXUh2b3SL9L1X1kCS/nORR3X3jkmpjc9tb37ttknskeXdVfTrJ/ZNcZNIh1sEif+9dleSi7v7H7v5Ukr/JLIzCgVqk/52b5PVJ0t3vTXJEkmOWUh2HuoX+v3AjET733SLfR3pRkrOn5R9P8s7u7iXWyOa11/5XVT+Q5LzMgqfnnlgve+x73b29u4/p7pO6+6TMnjd+VHd/cDXlsoks8u/uH2Y26pmqOiaz23A/ucQa2bwW6X9/l+T0JKmq78ssfH5xqVVyqLooyROmWW/vn2R7d1+96qL2xG23+2h330daVf8lyQe7+6Ik52d2y8XWzB4SfuzqKmYzWbD//WaS2yR5wzTP1d9196NWVjSbwoJ9D9bdgn3vT5M8rKquTPKtJP93d7vjiAO2YP/7j0l+r6r+r8wmH3qiQQfWQ1X9QWa/WDtmeqb4V5LcPEm6+3cye8b4EUm2Jrk+yTmrqXRx5b8NAAAARnPbLQAAAMMJnwAAAAwnfAIAADCc8AkAAMBwwicAAADDCZ8AsKCqenlVXVNVH13n8z6tqp6wDud5bVVtWY+aAGC9CZ8AsLhXJjljPU9YVYcn+ekkr1mH0700yS+sw3kAYN0JnwCwoO5+T5K/39/jq+q5VXVlVV1RVb81NT84yeXTl9l/b1VdPrf/lvn1qe3wqrqsqk6b1p9TVb8+bf6fSR4yBVoA2FD84wQAS1BVd0jyo0nu1t1dVUdPmx6Q5ENJ0t1/W1Xbq+re3f2RJOckecX8eaaQ+sQkb6yqn8tsJPZ+07ZvV9XWJPfacU4A2CiMfALAcmxPckOS86vqx5JcP7Ufl+SLc/u9LMk5VXVYkn+fXdyO290fS/LqJG9J8tPd/Y25zdck+ZfrXz4AHBjhEwDWSVUdVlUfmV7/ZX5bd38zyX2TvDHJI5P8ybTp60mOmNv1TUnOnPb5UHd/eTeXu2eS65Lccaf2I6ZzAsCG4rZbAFgn3f2tJPfe1baquk2SI7v74qr68ySfnDZ9PMmd585xQ1X9aWaTB507d/xzknyguy+cRk5vn+SHkrylqu7b3ddNu94lybrOxgsA68HIJwAsqKr+IMl7k9y1qq6qqnP3dsyc22YWFK9IcmmSp0/tb80sRM77/STfTvK2ubZ7Jvl8VR2T5LlJntTdf5Pkvyd54VTfsUm+3t2f37d3BgDjGfkEgAV19+MO4NirM7vtduf2z1TVl6tqS3evTc0PTPKKaSR1h5t393un5bvMHf+iuX3+tyTn7W+NADBSdfeqawCAQ1pV3TXJsd39nqq6MMn3Jnlwd39pH89zTpJXT8+XAsCGInwCAAAwnGc+AQAAGE74BAAAYDjhEwAAgOGETwAAAIYTPgEAABju/weTU2nayraCSgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 1008x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Size of calibration data\n",
    "n = len(X_calib)\n",
    "\n",
    "# Get the probability predictions\n",
    "predictions = model.predict_proba(X_calib)\n",
    "\n",
    "# We only need the probability for the true class\n",
    "prob_true_class = predictions[np.arange(n),y_calib]\n",
    "\n",
    "# Turn into uncertainty score (larger means more uncertain)\n",
    "scores = 1 - prob_true_class\n",
    "\n",
    "# Setting the alpha so that we get 95% prediction sets\n",
    "alpha = 0.05\n",
    "\n",
    "# define quantile\n",
    "q_level = np.ceil((n+1)*(1-alpha))/n\n",
    "qhat = np.quantile(scores, q_level)\n",
    "\n",
    "# Get the \"probabilities\" from the model\n",
    "predictions = model.predict_proba(X_calib)\n",
    "\n",
    "# Get for each instance the actual probability of ground truth\n",
    "prob_for_true_class = predictions[np.arange(len(y_calib)),y_calib]\n",
    "\n",
    "# Create a histogram\n",
    "fig, axs = plt.subplots(figsize=(14, 8))\n",
    "plt.hist(1 - prob_for_true_class, bins=30, range=(0, 1))\n",
    "plt.xlabel(\"1 - s(y,x)\")\n",
    "plt.ylabel(\"Frequency\")\n",
    "plt.title(\"Non-Conformity Score\")\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Calibration - Conformal Classification - Prediction Sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['DERMASON']\n",
      "['DERMASON']\n",
      "['DERMASON' 'SEKER']\n"
     ]
    }
   ],
   "source": [
    "prediction_sets = (1 - model.predict_proba(X_new) <= qhat)\n",
    "for i in range(3):\n",
    "    print(le.classes_[prediction_sets[i]])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Calibration - Conformal Classification - MAPIE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "cp = MapieClassifier(estimator=model, cv=\"prefit\", method=\"score\")\n",
    "cp.fit(X_calib, y_calib)\n",
    "y_pred, y_set = cp.predict(X_new, alpha=0.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['DERMASON']\n",
      "['DERMASON']\n",
      "['DERMASON' 'SEKER']\n",
      "['DERMASON']\n",
      "['DERMASON' 'SEKER']\n"
     ]
    }
   ],
   "source": [
    "y_set = np.squeeze(y_set)\n",
    "\n",
    "for i in range(5):\n",
    "    print(le.classes_[y_set[i]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2    871\n",
      "1    506\n",
      "3    233\n",
      "4      1\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# first count number of classes per bean\n",
    "set_sizes = y_set.sum(axis=1)\n",
    "\n",
    "# use pandas to compute how often each size occurs\n",
    "print(pd.Series(set_sizes).value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3    780\n",
      "2    372\n",
      "4    236\n",
      "1    222\n",
      "5      1\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "y_pred, y_set = cp.predict(X_new, alpha=0.01)\n",
    "\n",
    "# remove the 1-dim dimension\n",
    "y_set = np.squeeze(y_set)\n",
    "\n",
    "# first count number of classes per bean\n",
    "set_sizes = y_set.sum(axis=1)\n",
    "\n",
    "# use pandas to compute how often each size occurs\n",
    "print(pd.Series(set_sizes).value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['HOROZ' 'SIRA']\n"
     ]
    }
   ],
   "source": [
    "y_pred, y_set = cp.predict(X_new, alpha=[0.1, 0.05])\n",
    "\n",
    "# get prediction sets for 10th observation and second alpha (0.05)\n",
    "print(le.classes_[y_set[10,:,1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>set</th>\n",
       "      <th>setsize</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[DERMASON]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[DERMASON]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[DERMASON, SEKER]</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[DERMASON]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[DERMASON, SEKER]</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 set  setsize\n",
       "0         [DERMASON]        1\n",
       "1         [DERMASON]        1\n",
       "2  [DERMASON, SEKER]        2\n",
       "3         [DERMASON]        1\n",
       "4  [DERMASON, SEKER]        2"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred, y_set = cp.predict(X_new, alpha=0.05)\n",
    "y_set = np.squeeze(y_set)\n",
    "df = pd.DataFrame()\n",
    "\n",
    "for i in range(len(y_pred)):\n",
    "    predset = le.classes_[y_set[i]]\n",
    "    \n",
    "    # Create a new dataframe with the calculated values\n",
    "    temp_df = pd.DataFrame({\n",
    "        \"set\": [predset],\n",
    "        \"setsize\": [len(predset)]\n",
    "        },\n",
    "        index=[i])\n",
    "    \n",
    "    # Concatenate the new dataframe with the existing one\n",
    "    df = pd.concat([df, temp_df])\n",
    "    \n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a63dbfce567fe45cf0ad99c7fc02df65ff5753f7718eaa046a5ef4c1827daabc"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}