Previsão-Conforme-Classificação.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Introdução Prática à Previsão Conforme\n",
    "\n",
    "Previsão Conforme (em inglês Conformal Prediction) é um método que transforma as previsões pontuais em conjuntos de previsões. Dessa forma, fornece previsões associadas à um nível de confiança estatística, permitindo que sejam medidos e controlados os riscos de uma previsão estar errada.\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importar Libs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install -U numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install mapie -use-deprecated=legacy-resolver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import os\n",
    "import wget\n",
    "import zipfile\n",
    "from os.path import exists\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "pd.set_option('display.float_format', lambda x: '%.3f' % x)\n",
    "pd.options.display.max_rows=50\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import style\n",
    "import seaborn as sns\n",
    "style.use(\"fivethirtyeight\")\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "from mapie.classification import MapieClassifier\n",
    "from mapie.metrics import classification_coverage_score\n",
    "from mapie.metrics import classification_mean_width_score"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "bean_data_file = \"./DryBeanDataset/Dry_Bean_Dataset.xlsx\"\n",
    "base = \"https://archive.ics.uci.edu/ml/machine-learning-databases/\"\n",
    "dataset_number = \"00602\"\n",
    "filename = \"DryBeanDataset.zip\"\n",
    "url = base + dataset_number + \"/\" + filename\n",
    "wget.download(url)\n",
    "zipfile.ZipFile(filename, 'r') .extractall('./')\n",
    "os.remove(filename)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importar Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "bean_data_file = \"./DryBeanDataset/Dry_Bean_Dataset.xlsx\"\n",
    "beans = pd.read_excel(bean_data_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Area</th>\n",
       "      <th>Perimeter</th>\n",
       "      <th>MajorAxisLength</th>\n",
       "      <th>MinorAxisLength</th>\n",
       "      <th>AspectRation</th>\n",
       "      <th>Eccentricity</th>\n",
       "      <th>ConvexArea</th>\n",
       "      <th>EquivDiameter</th>\n",
       "      <th>Extent</th>\n",
       "      <th>Solidity</th>\n",
       "      <th>roundness</th>\n",
       "      <th>Compactness</th>\n",
       "      <th>ShapeFactor1</th>\n",
       "      <th>ShapeFactor2</th>\n",
       "      <th>ShapeFactor3</th>\n",
       "      <th>ShapeFactor4</th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>28395</td>\n",
       "      <td>610.291</td>\n",
       "      <td>208.178</td>\n",
       "      <td>173.889</td>\n",
       "      <td>1.197</td>\n",
       "      <td>0.550</td>\n",
       "      <td>28715</td>\n",
       "      <td>190.141</td>\n",
       "      <td>0.764</td>\n",
       "      <td>0.989</td>\n",
       "      <td>0.958</td>\n",
       "      <td>0.913</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.834</td>\n",
       "      <td>0.999</td>\n",
       "      <td>SEKER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>28734</td>\n",
       "      <td>638.018</td>\n",
       "      <td>200.525</td>\n",
       "      <td>182.734</td>\n",
       "      <td>1.097</td>\n",
       "      <td>0.412</td>\n",
       "      <td>29172</td>\n",
       "      <td>191.273</td>\n",
       "      <td>0.784</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.887</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.004</td>\n",
       "      <td>0.910</td>\n",
       "      <td>0.998</td>\n",
       "      <td>SEKER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>29380</td>\n",
       "      <td>624.110</td>\n",
       "      <td>212.826</td>\n",
       "      <td>175.931</td>\n",
       "      <td>1.210</td>\n",
       "      <td>0.563</td>\n",
       "      <td>29690</td>\n",
       "      <td>193.411</td>\n",
       "      <td>0.778</td>\n",
       "      <td>0.990</td>\n",
       "      <td>0.948</td>\n",
       "      <td>0.909</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.826</td>\n",
       "      <td>0.999</td>\n",
       "      <td>SEKER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>30008</td>\n",
       "      <td>645.884</td>\n",
       "      <td>210.558</td>\n",
       "      <td>182.517</td>\n",
       "      <td>1.154</td>\n",
       "      <td>0.499</td>\n",
       "      <td>30724</td>\n",
       "      <td>195.467</td>\n",
       "      <td>0.783</td>\n",
       "      <td>0.977</td>\n",
       "      <td>0.904</td>\n",
       "      <td>0.928</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.862</td>\n",
       "      <td>0.994</td>\n",
       "      <td>SEKER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>30140</td>\n",
       "      <td>620.134</td>\n",
       "      <td>201.848</td>\n",
       "      <td>190.279</td>\n",
       "      <td>1.061</td>\n",
       "      <td>0.334</td>\n",
       "      <td>30417</td>\n",
       "      <td>195.897</td>\n",
       "      <td>0.773</td>\n",
       "      <td>0.991</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.004</td>\n",
       "      <td>0.942</td>\n",
       "      <td>0.999</td>\n",
       "      <td>SEKER</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Area  Perimeter  MajorAxisLength  MinorAxisLength  AspectRation  \\\n",
       "0  28395    610.291          208.178          173.889         1.197   \n",
       "1  28734    638.018          200.525          182.734         1.097   \n",
       "2  29380    624.110          212.826          175.931         1.210   \n",
       "3  30008    645.884          210.558          182.517         1.154   \n",
       "4  30140    620.134          201.848          190.279         1.061   \n",
       "\n",
       "   Eccentricity  ConvexArea  EquivDiameter  Extent  Solidity  roundness  \\\n",
       "0         0.550       28715        190.141   0.764     0.989      0.958   \n",
       "1         0.412       29172        191.273   0.784     0.985      0.887   \n",
       "2         0.563       29690        193.411   0.778     0.990      0.948   \n",
       "3         0.499       30724        195.467   0.783     0.977      0.904   \n",
       "4         0.334       30417        195.897   0.773     0.991      0.985   \n",
       "\n",
       "   Compactness  ShapeFactor1  ShapeFactor2  ShapeFactor3  ShapeFactor4  Class  \n",
       "0        0.913         0.007         0.003         0.834         0.999  SEKER  \n",
       "1        0.954         0.007         0.004         0.910         0.998  SEKER  \n",
       "2        0.909         0.007         0.003         0.826         0.999  SEKER  \n",
       "3        0.928         0.007         0.003         0.862         0.994  SEKER  \n",
       "4        0.971         0.007         0.004         0.942         0.999  SEKER  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "beans.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pre-Process and Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Labels are characters but should be integers for sklearn\n",
    "le = LabelEncoder()\n",
    "beans[\"Class\"] = le.fit_transform(beans[\"Class\"])\n",
    "\n",
    "# Split data into classification target and features\n",
    "y = beans[\"Class\"]\n",
    "X = beans.drop(\"Class\", axis = 1)\n",
    "\n",
    "# Split of training data\n",
    "X_train, X_rest1, y_train, y_rest1 = train_test_split(X, y, train_size=10000, random_state=2)\n",
    "\n",
    "# From the remaining data, split of test data\n",
    "X_test, X_rest2, y_test, y_rest2 = train_test_split(X_rest1, y_rest1, train_size=1000, random_state=42)\n",
    "\n",
    "# Split remaining into calibration and \"new\" data\n",
    "X_calib, X_new, y_calib, y_new = train_test_split(X_rest2, y_rest2, train_size=1000, random_state=42)\n",
    "\n",
    "# Fit the model\n",
    "model = GaussianNB().fit(X_train, y_train)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.758\n",
      "          BARBUNYA  BOMBAY  CALI  DERMASON  HOROZ  SEKER  SIRA\n",
      "BARBUNYA        46       0    47         0      6      0     4\n",
      "BOMBAY           0      33     0         0      0      0     0\n",
      "CALI            20       0    81         0      3      0     0\n",
      "DERMASON         0       0     0       223      0     32     9\n",
      "HOROZ            0       0     4         3    104      0    22\n",
      "SEKER            2       0     0        26      1    127    22\n",
      "SIRA             0       0     0        10     10     21   144\n"
     ]
    }
   ],
   "source": [
    "# Check accuracy\n",
    "y_pred = model.predict(X_test)\n",
    "print(\"Accuracy:\", (y_pred == y_test).mean())\n",
    "\n",
    "# Create the confusion matrix\n",
    "cm = confusion_matrix(y_test, y_pred)\n",
    "print(pd.DataFrame(cm, index=le.classes_, columns=le.classes_))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Calibration - MAPIE - Naive Approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize the MapieClassifier\n",
    "mapie_score = MapieClassifier(model, cv=\"prefit\", method=\"naive\")\n",
    "\n",
    "# Calibration step\n",
    "mapie_score.fit(X_train, y_train)\n",
    "\n",
    "# Prediction step\n",
    "y_pred, y_set = mapie_score.predict(X_new, alpha=0.05)\n",
    "\n",
    "# Removing the alpha-dimension\n",
    "y_set = np.squeeze(y_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coverage: 90.32%\n",
      "Avg. set size: 1.41\n"
     ]
    }
   ],
   "source": [
    "cov = classification_coverage_score(y_new, y_set)\n",
    "setsize = classification_mean_width_score(y_set)\n",
    "print('Coverage: {:.2%}'.format(cov))\n",
    "print('Avg. set size: {:.2f}'.format(setsize))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Calibration - MAPIE - Score Method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapie_score = MapieClassifier(model, cv='prefit', method='score')\n",
    "mapie_score.fit(X_calib, y_calib)\n",
    "y_pred, y_set = mapie_score.predict(X_new, alpha=0.05)\n",
    "y_set = np.squeeze(y_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coverage: 96.28%\n",
      "Avg. set size: 1.83\n"
     ]
    }
   ],
   "source": [
    "cov = classification_coverage_score(y_new, y_set)\n",
    "setsize = classification_mean_width_score(y_set)\n",
    "print('Coverage: {:.2%}'.format(cov))\n",
    "print(\"Avg. set size: {:.2f}\".format(setsize))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Calibration - MAPIE - Marginal Coverage x Conditional Coverage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def class_wise_performance(y_new, y_set, classes):\n",
    "    df = pd.DataFrame()\n",
    "    # Loop through the classes\n",
    "    for i in range(len(classes)):  \n",
    "        # Calculate the coverage and set size for the current class\n",
    "        ynew = y_new.values[y_new.values == i]\n",
    "        yscore = y_set[y_new.values == i]\n",
    "        cov = classification_coverage_score(ynew, yscore)\n",
    "        size = classification_mean_width_score(yscore)\n",
    "        # Create a new dataframe with the calculated values\n",
    "        temp_df = pd.DataFrame({\n",
    "            \"class\": [classes[i]],\n",
    "            \"coverage\": [cov],\n",
    "            \"avg. set size\": [size]\n",
    "            }, index = [i])\n",
    "        # Concatenate the new dataframe with the existing one\n",
    "        df = pd.concat([df, temp_df])\n",
    "    return(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      class  coverage  avg. set size\n",
      "0  BARBUNYA     0.925          2.138\n",
      "1    BOMBAY     1.000          1.000\n",
      "2      CALI     0.966          2.131\n",
      "3  DERMASON     0.973          1.463\n",
      "4     HOROZ     0.939          1.873\n",
      "5     SEKER     0.970          2.017\n",
      "6      SIRA     0.974          1.974\n"
     ]
    }
   ],
   "source": [
    "# We can apply this function to our new data and the corresponding prediction sets\n",
    "print(class_wise_performance(y_new, y_set, le.classes_))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Calibration - MAPIE - Adaptative Approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-3 {color: black;background-color: white;}#sk-container-id-3 pre{padding: 0;}#sk-container-id-3 div.sk-toggleable {background-color: white;}#sk-container-id-3 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-3 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-3 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-3 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-3 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-3 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-3 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-3 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-3 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-3 div.sk-item {position: relative;z-index: 1;}#sk-container-id-3 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-3 div.sk-item::before, #sk-container-id-3 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-3 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-3 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-3 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-3 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-3 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-3 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-3 div.sk-label-container {text-align: center;}#sk-container-id-3 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-3 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-3\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>MapieClassifier(cv=&#x27;prefit&#x27;, estimator=GaussianNB(), method=&#x27;cumulated_score&#x27;,\n",
       "                random_state=1)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-7\" type=\"checkbox\" ><label for=\"sk-estimator-id-7\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MapieClassifier</label><div class=\"sk-toggleable__content\"><pre>MapieClassifier(cv=&#x27;prefit&#x27;, estimator=GaussianNB(), method=&#x27;cumulated_score&#x27;,\n",
       "                random_state=1)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-8\" type=\"checkbox\" ><label for=\"sk-estimator-id-8\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: GaussianNB</label><div class=\"sk-toggleable__content\"><pre>GaussianNB()</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-9\" type=\"checkbox\" ><label for=\"sk-estimator-id-9\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GaussianNB</label><div class=\"sk-toggleable__content\"><pre>GaussianNB()</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "MapieClassifier(cv='prefit', estimator=GaussianNB(), method='cumulated_score',\n",
       "                random_state=1)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mapie_score = MapieClassifier(model, cv=\"prefit\",\n",
    "                              method=\"cumulated_score\",\n",
    "                              random_state=1)\n",
    "\n",
    "mapie_score.fit(X_calib, y_calib)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coverage: 96.96%\n",
      "Avg. set size: 1.91\n",
      "      class  coverage  avg. set size\n",
      "0  BARBUNYA     0.931          2.207\n",
      "1    BOMBAY     1.000          1.000\n",
      "2      CALI     0.972          2.153\n",
      "3  DERMASON     0.983          1.512\n",
      "4     HOROZ     0.947          1.980\n",
      "5     SEKER     0.983          2.115\n",
      "6      SIRA     0.974          2.132\n"
     ]
    }
   ],
   "source": [
    "y_pred, y_set = mapie_score.predict(X_new, alpha=0.05, include_last_label=True)\n",
    "y_set = np.squeeze(y_set)\n",
    "\n",
    "cov = classification_coverage_score(y_new, y_set)\n",
    "setsize = classification_mean_width_score(y_set)\n",
    "print('Coverage: {:.2%}'.format(cov))\n",
    "print(\"Avg. set size: {:.2f}\".format(setsize))\n",
    "print(class_wise_performance(y_new, y_set, le.classes_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coverage: 81.50%\n",
      "Avg. set size: 1.18\n",
      "      class  coverage  avg. set size\n",
      "0  BARBUNYA     0.534          1.218\n",
      "1    BOMBAY     1.000          1.000\n",
      "2      CALI     0.847          1.153\n",
      "3  DERMASON     0.921          1.163\n",
      "4     HOROZ     0.803          1.275\n",
      "5     SEKER     0.732          1.132\n",
      "6      SIRA     0.849          1.170\n"
     ]
    }
   ],
   "source": [
    "y_pred, y_set = mapie_score.predict(X_new, alpha=0.05, include_last_label=False)\n",
    "y_set = np.squeeze(y_set)\n",
    "cov = classification_coverage_score(y_new, y_set)\n",
    "setsize = classification_mean_width_score(y_set)\n",
    "\n",
    "print('Coverage: {:.2%}'.format(cov))\n",
    "print(\"Avg. set size: {:.2f}\".format(setsize))\n",
    "print(class_wise_performance(y_new, y_set, le.classes_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coverage: 95.97%\n",
      "Avg. set size: 1.89\n",
      "      class  coverage  avg. set size\n",
      "0  BARBUNYA     0.931          2.213\n",
      "1    BOMBAY     1.000          2.000\n",
      "2      CALI     0.960          2.165\n",
      "3  DERMASON     0.973          1.443\n",
      "4     HOROZ     0.934          1.980\n",
      "5     SEKER     0.962          1.979\n",
      "6      SIRA     0.968          1.981\n"
     ]
    }
   ],
   "source": [
    "y_pred, y_set = mapie_score.predict(X_new, alpha=0.05, include_last_label=\"randomized\")\n",
    "y_set = np.squeeze(y_set)\n",
    "cov = classification_coverage_score(y_new, y_set)\n",
    "setsize = classification_mean_width_score(y_set)\n",
    "\n",
    "print('Coverage: {:.2%}'.format(cov))\n",
    "print(\"Avg. set size: {:.2f}\".format(setsize))\n",
    "print(class_wise_performance(y_new, y_set, le.classes_))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Calibration - MAPIE - Top-k method for fixed size sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coverage: 98.82%\n",
      "Avg. set size: 3.61\n",
      "      class  coverage  avg. set size\n",
      "0  BARBUNYA     0.971          3.414\n",
      "1    BOMBAY     1.000          7.000\n",
      "2      CALI     0.983          3.750\n",
      "3  DERMASON     0.998          4.261\n",
      "4     HOROZ     0.971          3.000\n",
      "5     SEKER     0.987          3.000\n",
      "6      SIRA     1.000          3.000\n"
     ]
    }
   ],
   "source": [
    "mapie_score = MapieClassifier(model, cv=\"prefit\", method=\"top_k\")\n",
    "mapie_score.fit(X_calib, y_calib)\n",
    "\n",
    "y_pred, y_set = mapie_score.predict(X_new, alpha=0.05)\n",
    "y_set = np.squeeze(y_set)\n",
    "cov = classification_coverage_score(y_new, y_set)\n",
    "setsize = classification_mean_width_score(y_set)\n",
    "\n",
    "print('Coverage: {:.2%}'.format(cov))\n",
    "print(\"Avg. set size: {:.2f}\".format(setsize))\n",
    "print(class_wise_performance(y_new, y_set, le.classes_))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Calibration - MAPIE - Regularized APS (RAPS) for small sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coverage: 95.59%\n",
      "Avg. set size: 1.87\n",
      "      class  coverage  avg. set size\n",
      "0  BARBUNYA     0.925          2.167\n",
      "1    BOMBAY     1.000          2.000\n",
      "2      CALI     0.955          2.114\n",
      "3  DERMASON     0.970          1.461\n",
      "4     HOROZ     0.926          1.852\n",
      "5     SEKER     0.966          2.026\n",
      "6      SIRA     0.961          1.971\n"
     ]
    }
   ],
   "source": [
    "mapie_score = MapieClassifier(model, cv=\"prefit\", method=\"raps\")\n",
    "mapie_score.fit(X_calib, y_calib, size_raps=0.2)\n",
    "\n",
    "y_pred, y_set = mapie_score.predict(X_new, alpha=0.05, include_last_label=\"randomized\")\n",
    "y_set = np.squeeze(y_set)\n",
    "cov = classification_coverage_score(y_new, y_set)\n",
    "setsize = classification_mean_width_score(y_set)\n",
    "\n",
    "print('Coverage: {:.2%}'.format(cov))\n",
    "print(\"Avg. set size: {:.2f}\".format(setsize))\n",
    "print(class_wise_performance(y_new, y_set, le.classes_))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Calibration - MAPIE - Group-balanced conformal prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa8AAAEJCAYAAAAjLqjyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAAsTAAALEwEAmpwYAAAfTElEQVR4nO3dfVSUdf7/8df8zDoEFi0Mg6ajpxxBPZplAbk3lZJEbhmJd2fP1lIEQm7KpqhtW7ttexByESpiM6yts3q2mtwNt46ePNAmhk611bhpSMcjRz00IDnlkHSj1/cPf17beNNgDc5c9HycM+fIdb3nms+8u+LF57quucbm9/sNAQBgIf8v0gMAAOBMEV4AAMshvAAAlkN4AQAsh/ACAFgO4QUAsBzCCwBgOYQXAMByCC+LaG1tjfQQoh49Co0ehUaPQouGHhFeAADLIbwAAJZDeAEALIfwAgBYDuEFALAcwgsAYDmEFwDAcggvAIDlnBPpAfRn8c/sN//tz7s4giMBgP6FmRcAwHJ6FV5btmzRnDlzNHr0aMXHx2vNmjVB6w3DUFlZmVJTU5WcnKxp06Zp586dQTV+v18FBQVyOp1yOp0qKCiQ3+8Pqvnggw904403Kjk5WaNHj1Z5ebkMw/h+7xAA0O/0Kry6u7s1ZswYLV++XDExMSetr66uVk1NjcrLy9XQ0CC73a6cnBwdOnTIrMnPz5fX65Xb7Zbb7ZbX61VhYaG5/rPPPlNOTo6SkpLU0NCg5cuX67HHHtPjjz8ehrcJAOhPenXOa+rUqZo6daokqbi4OGidYRiqra3VwoULNX36dElSbW2tXC6X3G638vLy1NLSok2bNmnDhg1KS0uTJK1cuVLZ2dlqbW2Vy+XSiy++qMOHD6u2tlYxMTEaM2aMdu3apSeeeELz58+XzWYL5/sGAFjY9z7n1dbWJp/Pp8mTJ5vLYmJiNGnSJG3btk2S5PF4FBcXp/T0dLMmIyNDsbGxQTVXX3110MxuypQpam9vV1tb2/cdZsTFP7M/6AEA+O6+99WGPp9PkmS324OW2+12tbe3S5I6OjqUkJAQNHuy2WxKTExUR0eHWTNkyJCTtnF83YgRI075+tFwa/7TO/+0a77LuKP7vUYHehQaPQqNHoXWVz1yuVy9qrP8pfK9faMR0XT6GdaZjvv44VWcHj0KjR6FRo9Ci4Yefe/Dhg6HQ5LU2dkZtLyzs1NJSUmSpKSkJHV1dQVdOWgYhg4cOBBUc6ptHF8HAMBx3zu8hg8fLofDocbGRnNZT0+PmpubzXNcaWlpCgQC8ng8Zo3H41F3d3dQTXNzs3p6esyaxsZGDR48WMOHD/++wwQA9CO9Cq9AICCv1yuv16ujR49q37598nq92rt3r2w2m4qKilRdXa36+nrt2LFDxcXFio2NVW5uriQpJSVFmZmZKikpkcfjkcfjUUlJibKyssypZ25urmJiYlRcXKwdO3aovr5eVVVVKi4u5kpDAECQXp3zevfdd3XTTTeZP5eVlamsrExz585VbW2tFixYoMOHD2vx4sXy+/2aOHGi1q1bp0GDBpnPqaurU2lpqWbMmCFJys7OVkVFhbn+wgsv1D/+8Q8tWrRI1113neLj43X33Xdr/vz54XqvAIB+wub3+7mFRR/5tkviz/Reh9FwgjTa0aPQ6FFo9Ci0aOgR9zYEAFgO4QUAsBzCCwBgOYQXAMByCC8AgOUQXgAAyyG8AACWY/kb80YTvuoEAM4OZl4AAMshvAAAlkN4AQAsh/ACAFgO4QUAsBzCCwBgOYQXAMBy+JxXhHzzM2Fn+t1eAPBDx8wLAGA5hBcAwHIILwCA5RBeAADLIbwAAJZDeAEALIfwAgBYDuEFALAcwgsAYDmEFwDAcggvAIDlEF4AAMshvAAAlkN4AQAsh/ACAFgO3+cVBfhuLwA4M2GZeR05ckQPP/ywxo8fL4fDofHjx+vhhx/W119/bdYYhqGysjKlpqYqOTlZ06ZN086dO4O24/f7VVBQIKfTKafTqYKCAvn9/nAMEQDQj4QlvKqqqlRXV6fy8nJ5PB4tX75cTz31lCorK82a6upq1dTUqLy8XA0NDbLb7crJydGhQ4fMmvz8fHm9Xrndbrndbnm9XhUWFoZjiACAfiQshw09Ho9uuOEGZWdnS5KGDx+u7OxsvfPOO5KOzbpqa2u1cOFCTZ8+XZJUW1srl8slt9utvLw8tbS0aNOmTdqwYYPS0tIkSStXrlR2drZaW1vlcrnCMVQAQD8QlplXRkaGmpqatGvXLknShx9+qM2bN+v666+XJLW1tcnn82ny5Mnmc2JiYjRp0iRt27ZN0rEAjIuLU3p6etB2Y2NjzRoAAKQwzbwWLlyoQCCg9PR0DRgwQF9//bUWLVqk/Px8SZLP55Mk2e32oOfZ7Xa1t7dLkjo6OpSQkCCbzWaut9lsSkxMVEdHRziGCQDoJ8ISXuvWrdPf//531dXVKTU1Vdu3b9fSpUvldDp12223heMlTqu1tbVPt39mzv/eW/i29xNd7zU60aPQ6FFo9Ci0vupRb08RhSW8HnjgAc2fP18zZsyQJI0dO1Z79+7VypUrddttt8nhcEiSOjs7NWzYMPN5nZ2dSkpKkiQlJSWpq6tLhmGYsy/DMHTgwAGz5lSi6lxY0/7QNSGc7v1w3i80ehQaPQqNHoUWDT0Kyzmvzz//XAMGDAhaNmDAAB09elTSsQs4HA6HGhsbzfU9PT1qbm42z3GlpaUpEAjI4/GYNR6PR93d3UHnwQAACMvM64YbblBVVZWGDx+u1NRUeb1e1dTUaM6cOZKOnbsqKipSZWWlXC6XRo4cqRUrVig2Nla5ubmSpJSUFGVmZqqkpERVVVWSpJKSEmVlZUU84QEA0SUs4VVRUaE//elPuvfee3XgwAE5HA7dfvvtKi0tNWsWLFigw4cPa/HixfL7/Zo4caLWrVunQYMGmTV1dXUqLS01Dz9mZ2eroqIiHEMEAPQjNr/fb0R6EP3FN2/z9F2d7vZQ0XCMOdrRo9DoUWj0KLRo6BE35gUAWA7hBQCwHO4q/z2F41AhAODMMPMCAFgO4QUAsBzCCwBgOYQXAMByCC8AgOUQXgAAyyG8AACWQ3gBACyH8AIAWA7hBQCwHMILAGA5hBcAwHIILwCA5RBeAADL4StRvgO+BgUAIovwijInBqM/7+IIjQQAoheHDQEAlkN4AQAsh/ACAFgO4QUAsBzCCwBgOYQXAMByCC8AgOUQXgAAyyG8AACWQ3gBACyH8AIAWA7hBQCwHMILAGA5YQuvjz/+WPPmzdOll14qh8Oh9PR0NTU1mesNw1BZWZlSU1OVnJysadOmaefOnUHb8Pv9KigokNPplNPpVEFBgfx+f7iGCADoJ8ISXn6/X1lZWTIMQy+88IK2bdumiooK2e12s6a6ulo1NTUqLy9XQ0OD7Ha7cnJydOjQIbMmPz9fXq9XbrdbbrdbXq9XhYWF4RgiAKAfCcv3eT366KNKTk7Wk08+aS4bMWKE+W/DMFRbW6uFCxdq+vTpkqTa2lq5XC653W7l5eWppaVFmzZt0oYNG5SWliZJWrlypbKzs9Xa2iqXyxWOoQIA+oGwzLxeeeUVTZw4UXl5eRo5cqR+8pOfaNWqVTIMQ5LU1tYmn8+nyZMnm8+JiYnRpEmTtG3bNkmSx+NRXFyc0tPTzZqMjAzFxsaaNQAASGEKrz179mj16tUaMWKEXnrpJc2bN09/+MMf9NRTT0mSfD6fJAUdRjz+c0dHhySpo6NDCQkJstls5nqbzabExESzBgAAKUyHDY8eParLL79cDz74oCTpsssu0+7du1VXV6eCgoJwvMRptba29un2T+38s/ZK33x/kXmv1kKPQqNHodGj0PqqR709RRSW8HI4HEpJSQlaNmrUKO3bt89cL0mdnZ0aNmyYWdPZ2amkpCRJUlJSkrq6umQYhjn7MgxDBw4cMGtOJSLnwpr2n7WXOv7+OO8XGj0KjR6FRo9Ci4YeheWwYUZGhj766KOgZR999JEZVMOHD5fD4VBjY6O5vqenR83NzeY5rrS0NAUCAXk8HrPG4/Gou7s76DwYAABhmXkVFxdr6tSpWrFihW699VZ5vV6tWrVKv/vd7yQdO3dVVFSkyspKuVwujRw5UitWrFBsbKxyc3MlSSkpKcrMzFRJSYmqqqokSSUlJcrKyop4wkdS/DPHZ3nny//DbQMABAlLeF1xxRVas2aNHnroIT3yyCMaOnSo7rvvPuXn55s1CxYs0OHDh7V48WL5/X5NnDhR69at06BBg8yauro6lZaWasaMGZKk7OxsVVRUhGOIAIB+xOb3+41ID8Jq/jcbOrv8eRdH5HWtIhqOw0c7ehQaPQotGnrEvQ0BAJYTlsOG/V2kZloAgFNj5gUAsBzCCwBgOYQXAMByCC8AgOUQXgAAyyG8AACWQ3gBACyH8AIAWA7hBQCwHMILAGA5hBcAwHIILwCA5RBeAADLIbwAAJZDeAEALIfwAgBYDuEFALAcwgsAYDmEFwDAcggvAIDlEF4AAMshvAAAlkN4AQAs55xIDwC9F//M/qCf/XkXR2gkABBZzLwAAJZDeAEALIfwAgBYDuEFALAcwgsAYDmEFwDAcggvAIDl9El4VVZWKj4+XosXLzaXGYahsrIypaamKjk5WdOmTdPOnTuDnuf3+1VQUCCn0ymn06mCggL5/f6+GCIAwMLCHl5vvfWW/vrXv2rs2LFBy6urq1VTU6Py8nI1NDTIbrcrJydHhw4dMmvy8/Pl9Xrldrvldrvl9XpVWFgY7iH2Svwz+80HACC6hDW8Pv30U9111116/PHHFR8fby43DEO1tbVauHChpk+frjFjxqi2tlaBQEBut1uS1NLSok2bNqmqqkppaWlKS0vTypUrtXHjRrW2toZzmAAAiwtreB0Pp5/97GdBy9va2uTz+TR58mRzWUxMjCZNmqRt27ZJkjwej+Li4pSenm7WZGRkKDY21qwBAEAK470Nn332We3evVurVq06aZ3P55Mk2e32oOV2u13t7e2SpI6ODiUkJMhms5nrbTabEhMT1dHRcdrX7btZ2fl9tN3wYUZ6MnoSGj0KjR6F1lc9crlcvaoLS3i1trbqoYce0oYNGzRw4MBwbLLXevtGz1hT9J/ruqrpfwHLTXqP7Yd9tj/0E/QoNHoUWjT0KCyHDT0ej7q6upSRkaGEhAQlJCRoy5YtqqurU0JCgn70ox9Jkjo7O4Oe19nZqaSkJElSUlKSurq6ZBiGud4wDB04cMCsAQBAClN4TZs2TW+++aY2b95sPi6//HLNmDFDmzdv1siRI+VwONTY2Gg+p6enR83NzeY5rrS0NAUCAXk8HrPG4/Gou7s76DwYAABhOWwYHx8fdHWhJJ1//vm66KKLNGbMGElSUVGRKisr5XK5NHLkSK1YsUKxsbHKzc2VJKWkpCgzM1MlJSWqqqqSJJWUlCgrKyvi01MAQHQ5a19GuWDBAh0+fFiLFy+W3+/XxIkTtW7dOg0aNMisqaurU2lpqWbMmCFJys7OVkVFxdkaIgDAImx+v98IXfbDY7UPJ3PBRnScRI529Cg0ehRaNPSIexsCACyH8AIAWA7hBQCwHMILAGA5hBcAwHIILwCA5RBeAADLIbwAAJZDeAEALOes3R4K0e/Eu4pw1w4A0YqZFwDAcph59RN9MWv65jaZhQGIJsy8AACWQ3gBACyH8AIAWA7hBQCwHC7YwBnjknoAkUZ49VNcKQigPyO8fgDCEWQnzrYAIJI45wUAsBzCCwBgORw2/IHhYgsA/QEzLwCA5RBeAADL4bDhDxxXEQKwImZeAADLIbwAAJZDeAEALIfwAgBYDuEFALAcwgsAYDmEFwDAcsISXpWVlbruuus0bNgwXXrppZo9e7Z27NgRVGMYhsrKypSamqrk5GRNmzZNO3fuDKrx+/0qKCiQ0+mU0+lUQUGB/H5/OIaIPhT/zH7zAQBnQ1jCq6mpSXfeeac2btyo+vp6nXPOObrlllt08OBBs6a6ulo1NTUqLy9XQ0OD7Ha7cnJydOjQIbMmPz9fXq9XbrdbbrdbXq9XhYWF4RgiAKAfCcsdNtatWxf085NPPimn06mtW7cqOztbhmGotrZWCxcu1PTp0yVJtbW1crlccrvdysvLU0tLizZt2qQNGzYoLS1NkrRy5UplZ2ertbVVLpcrHEMFAPQDfXLOKxAI6OjRo4qPj5cktbW1yefzafLkyWZNTEyMJk2apG3btkmSPB6P4uLilJ6ebtZkZGQoNjbWrAEAQOqjexsuXbpU48aNM2dQPp9PkmS324Pq7Ha72tvbJUkdHR1KSEiQzWYz19tsNiUmJqqjo+O0r9Xa2hru4f9/5/fRdvu3E897vfWTz8/q6/fd/tB/0KPQ6FFofdWj3h5lC3t43Xfffdq6das2bNigAQMGhHvzJ+mzw4lNXHwQDmfzcC+Hl0OjR6HRo9CioUdhPWy4bNkyvfTSS6qvr9eIESPM5Q6HQ5LU2dkZVN/Z2amkpCRJUlJSkrq6umQYhrneMAwdOHDArAEAQApjeC1ZssQMrlGjRgWtGz58uBwOhxobG81lPT09am5uNs9xpaWlKRAIyOPxmDUej0fd3d1B58EAAAjLYcNFixbp+eef19/+9jfFx8eb57hiY2MVFxcnm82moqIiVVZWyuVyaeTIkVqxYoViY2OVm5srSUpJSVFmZqZKSkpUVVUlSSopKVFWVlbEp6cAgOgSlvCqq6uTJPMy+OOWLFmiZcuWSZIWLFigw4cPa/HixfL7/Zo4caLWrVunQYMGBW2ntLRUM2bMkCRlZ2eroqIiHEMEAPQjYQmv3twFw2azadmyZWaYnUp8fLxWrVoVjiEBAPqxPrlU3oq4tREAWAfhhT71zT8K/HkXR3AkAPoT7ioPALAcwgsAYDmEFwDAcggvAIDlEF4AAMshvAAAlkN4AQAsh/ACAFgO4QUAsBzusIGz5sRbcHHHDQDfFTMvAIDlEF4AAMshvAAAlkN4AQAsh/ACAFgO4QUAsBwulUfE8EWVAL4rZl4AAMshvAAAlsNhQ0QFDiECOBPMvAAAlkN4AQAsh/ACAFgO4QUAsBzCCwBgOT/oqw1P/H4pRAe+9wtAKD/o8II1cBk9gBNx2BAAYDnMvACEHYd+0dcILwDfyZkEVDgO/XL4GN8UdeFVV1enRx99VD6fT6mpqSorK9OkSZMiPSxECf6i7xu9vXiptwH1XV/3m9v/tu0RZIiq8Fq3bp2WLl2qP//5z8rIyFBdXZ1mzpyprVu3atiwYZEeHqLcVU3nS03Hfqn1t19ovf1l/W11x9b9r0ffdxx9sY1wbz8c+wF/MEUnm9/vNyI9iOOmTJmisWPH6tFHHzWXXXHFFZo+fboefPDBsL8el8r3X/3tF0zoUEJv9HZm9211/W3f+i5aW1vlcrkiOoaoCa8vv/xSgwcP1urVq3XLLbeYyxctWqQdO3bo1VdfjdzgAABRJWoule/q6tKRI0dkt9uDltvtdnV0dERoVACAaBQ14QUAQG9FTXglJCRowIAB6uzsDFre2dmppKSkCI0KABCNoia8zj33XE2YMEGNjY1ByxsbG5Wenh6hUQEAolFUXSp/9913q7CwUBMnTlR6erqefvppffzxx8rLy4v00AAAUSRqZl6SdOutt6qsrEyPPPKIfvrTn2rr1q164YUX5HQ6Iz20M1JWVqb4+Pigx6hRo8z1hmGorKxMqampSk5O1rRp07Rz586gbfj9fhUUFMjpdMrpdKqgoEB+vz+o5oMPPtCNN96o5ORkjR49WuXl5TKM4ItHX375ZaWnpyspKUnp6elav359n73vb7NlyxbNmTNHo0ePVnx8vNasWRO0Ppp60pux9IVQPSoqKjppv8rMzAyq+eKLL7R48WJdcsklGjJkiObMmaP9+4Mv9d67d69mz56tIUOG6JJLLlFpaam+/PLLoJqmpiZdc801cjgcuuyyy/T000+fNN66ujqNHz9eDodD11xzjd58880wdeL0Kisrdd1112nYsGG69NJLNXv2bO3YsSOo5oe8L/WmP/1lP4qq8JKk/Px8bd++XR0dHfr3v/+tH//4x5Ee0nficrnU0tJiPr75H6S6ulo1NTUqLy9XQ0OD7Ha7cnJydOjQIbMmPz9fXq9XbrdbbrdbXq9XhYWF5vrPPvtMOTk5SkpKUkNDg5YvX67HHntMjz/+uFnj8Xh0xx13aObMmdq8ebNmzpypX/3qV3r77bfPThO+obu7W2PGjNHy5csVExNz0vpo6klvxtIXQvVIkq699tqg/erFF18MWr9s2TKtX79eq1ev1quvvqpDhw5p9uzZOnLkiCTpyJEjmj17tgKBgF599VWtXr1a9fX1+u1vf2tuY8+ePZo1a5bS0tL0xhtv6De/+Y1KS0v18ssvmzXHbyhw77336o033lBaWppmzpypvXv39kFn/qepqUl33nmnNm7cqPr6ep1zzjm65ZZbdPDgQbPmh7wv9aY/Uv/Yj6Lmc179SVlZmerr69Xc3HzSOsMwlJqaqrvuukuLFi2SJB0+fFgul0t//OMflZeXp5aWFqWnp2vDhg3KyMiQJDU3Nys7O1tvvfWWXC6XVq9erd///vfatWuX+YvukUce0dNPP60dO3bIZrMpLy9PBw8e1D//+U/z9adPn67ExEStXr267xtxGhdffLEqKir0i1/8QlJ09aQ3Y4lEj6RjfzF/8sknev7550/5nE8//VQjR45UTU2NZs2aJUnat2+fxo0bJ7fbrSlTpui1117TrFmztH37dg0dOlSS9Pzzz+uee+5Ra2urLrjgAj344INav369/vOf/5jb/vWvf60PP/xQr732mqSzf0OB0wkEAnI6nVqzZo2ys7PZl0L0R+o/+1HUzbz6iz179ig1NVXjx4/XHXfcoT179kiS2tra5PP5NHnyZLM2JiZGkyZN0rZt2yQd+4suLi4u6EKVjIwMxcbGBtVcffXVQX+hT5kyRe3t7Wpra5MkvfXWW0Gvc7zm+DaiRTT1pDdjiaTm5maNHDlSEydO1D333BN0de57772nr776KmjsQ4cOVUpKSlCPUlJSzF840rH3/8UXX+i9994za07Vo3fffVdfffWVvvzyS7333nsn1UyePPms9ygQCOjo0aOKj4+XxL50ohP7c1x/2I8Irz5w5ZVX6oknnpDb7TZvMjx16lR98skn8vl8kvStH8bu6OhQQkKCbDabud5msykxMTGo5lTbOL5Oknw+nyU+9B1NPenNWCIlMzNTf/nLX/Tyyy/r4Ycf1jvvvKObb75ZX3zxhaRj73HAgAFKSEgIet6JfTzxvR3/mEqoPn799dfq6uqKqhsKLF26VOPGjVNaWpok9qUTndgfqf/sR1F1tWF/cf311wf9fOWVV2rChAlau3atrrrqqgiNClY3Y8YM899jx47VhAkTNG7cOG3cuFE333xzBEcWGffdd5+2bt2qDRs2aMCAAZEeTtQ5XX/6y37EzOssiIuLU2pqqnbv3i2HwyFJ3/ph7KSkJHV1dQVd2WQYhg4cOBBUc6ptHF8nSQ6HwxIf+o6mnvRmLNFi8ODBGjJkiHbv3i3p2Hs8cuSIurq6gupO7OOJ7+34X8Ch+njOOecoISEhKm4osGzZMr300kuqr6/XiBEjzOXsS8ecrj+nYtX9iPA6C3p6etTa2iqHw6Hhw4fL4XAEfRi7p6dHzc3N5jH4tLQ0BQIBeTwes8bj8ai7uzuoprm5WT09PWZNY2OjBg8erOHDh0uSrrrqKkt86DuaetKbsUSLrq4utbe3m78kJ0yYoIEDBwaNff/+/eYFCtKxHrW0tARd9tzY2KjzzjtPEyZMMGtO1aPLL79cAwcOjPgNBZYsWWL+Yv7mR1Ak9iXp2/tzKlbdjwivPnD//ferqalJe/bs0dtvv63bb79dn3/+uebOnSubzaaioiJVV1ervr5eO3bsUHFxsWJjY5WbmytJSklJUWZmpkpKSuTxeOTxeFRSUqKsrCzzawhyc3MVExOj4uJi7dixQ/X19aqqqlJxcbF5LH/evHl64403tHLlSu3atUuVlZXavHmzioqKznpPAoGAvF6vvF6vjh49qn379snr9Wrv3r1R1ZPejCUSPQoEArr//vvl8XjU1tamzZs3a86cObLb7fr5z38uSbrwwgv1y1/+Ug8++KBef/11vf/++yosLNTYsWN17bXXSjp2Mnz06NGaN2+e3n//fb3++ut64IEHdNttt+mCCy6QJOXl5am9vV1Lly5VS0uLnnvuOa1du1bz5883x3r33Xdr7dq1eu6559TS0qIlS5aclRsKLFq0SGvXrtVTTz2l+Ph4+Xw++Xw+BQIBSb3779ef96VQ/elX+5Hf7zd4hPdx6623GsnJycbAgQONwYMHGzfddJOxdetWc/3BgweNJUuWGA6HwzjvvPOMSZMmGW+++WbQNvbs2WPMmjXLGDRokDFo0CBj1qxZxp49e4JqtmzZYlx99dXGeeedZzgcDmPp0qXGwYMHg2qeffZZw+VyGQMHDjRGjRplPPfccxHpyfr16w1JJz3mzp0bdT3pzVjOdo/a29uNyZMnG4mJicbAgQONoUOHGnPnzjX++9//Bm3D5/MZd911l3HRRRcZMTExRlZW1kk127dvN7KysoyYmBjjoosuMgoKCgyfzxdU869//csYP368ce655xpOp9OorKw8abwrVqwwhg0bZpx77rnGZZddZrzyyit93qNT9UeSsWTJkqj8/+ts70uh+tOf9iM+5wUAsBwOGwIALIfwAgBYDuEFALAcwgsAYDmEFwDAcggvAIDlEF4AAMshvAAAlkN4AQAs5/8AFCPryqDSfM4AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X.Area.hist(bins = 100);\n",
    "small_bean_index = X_calib.Area < 70000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-4 {color: black;background-color: white;}#sk-container-id-4 pre{padding: 0;}#sk-container-id-4 div.sk-toggleable {background-color: white;}#sk-container-id-4 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-4 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-4 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-4 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-4 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-4 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-4 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-4 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-4 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-4 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-4 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-4 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-4 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-4 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-4 div.sk-item {position: relative;z-index: 1;}#sk-container-id-4 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-4 div.sk-item::before, #sk-container-id-4 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-4 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-4 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-4 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-4 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-4 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-4 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-4 div.sk-label-container {text-align: center;}#sk-container-id-4 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-4 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-4\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>MapieClassifier(cv=&#x27;prefit&#x27;, estimator=GaussianNB())</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-10\" type=\"checkbox\" ><label for=\"sk-estimator-id-10\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MapieClassifier</label><div class=\"sk-toggleable__content\"><pre>MapieClassifier(cv=&#x27;prefit&#x27;, estimator=GaussianNB())</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-11\" type=\"checkbox\" ><label for=\"sk-estimator-id-11\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: GaussianNB</label><div class=\"sk-toggleable__content\"><pre>GaussianNB()</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-12\" type=\"checkbox\" ><label for=\"sk-estimator-id-12\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GaussianNB</label><div class=\"sk-toggleable__content\"><pre>GaussianNB()</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "MapieClassifier(cv='prefit', estimator=GaussianNB())"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_calib_group = X_calib[small_bean_index]\n",
    "y_calib_group = y_calib[small_bean_index]\n",
    "mapie_group = MapieClassifier(model, cv=\"prefit\", method=\"score\")\n",
    "mapie_group.fit(X_calib_group, y_calib_group.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "group_index = X_new.Area < 70000\n",
    "X_group = X_new[group_index]\n",
    "y_pred, y_set = mapie_group.predict(X_group, alpha=0.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a63dbfce567fe45cf0ad99c7fc02df65ff5753f7718eaa046a5ef4c1827daabc"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}