ch05-outliers/Recipe-5-Capping-outliers.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Capping / Censoring outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.datasets import load_breast_cancer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from feature_engine.outliers import Winsorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((398, 30), (171, 30))"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load the California House price data from Scikit-learn\n",
    "breast_cancer = load_breast_cancer()\n",
    "X = pd.DataFrame(breast_cancer.data, columns=breast_cancer.feature_names)\n",
    "y = breast_cancer.target\n",
    "\n",
    "# let's separate the data into training and testing sets\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X,\n",
    "    y,\n",
    "    test_size=0.3,\n",
    "    random_state=0,\n",
    ")\n",
    "\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean radius</th>\n",
       "      <th>mean texture</th>\n",
       "      <th>mean perimeter</th>\n",
       "      <th>mean area</th>\n",
       "      <th>mean smoothness</th>\n",
       "      <th>mean compactness</th>\n",
       "      <th>mean concavity</th>\n",
       "      <th>mean concave points</th>\n",
       "      <th>mean symmetry</th>\n",
       "      <th>mean fractal dimension</th>\n",
       "      <th>...</th>\n",
       "      <th>worst radius</th>\n",
       "      <th>worst texture</th>\n",
       "      <th>worst perimeter</th>\n",
       "      <th>worst area</th>\n",
       "      <th>worst smoothness</th>\n",
       "      <th>worst compactness</th>\n",
       "      <th>worst concavity</th>\n",
       "      <th>worst concave points</th>\n",
       "      <th>worst symmetry</th>\n",
       "      <th>worst fractal dimension</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>478</th>\n",
       "      <td>11.490</td>\n",
       "      <td>14.59</td>\n",
       "      <td>73.99</td>\n",
       "      <td>404.9</td>\n",
       "      <td>0.10460</td>\n",
       "      <td>0.08228</td>\n",
       "      <td>0.05308</td>\n",
       "      <td>0.01969</td>\n",
       "      <td>0.1779</td>\n",
       "      <td>0.06574</td>\n",
       "      <td>...</td>\n",
       "      <td>12.40</td>\n",
       "      <td>21.90</td>\n",
       "      <td>82.04</td>\n",
       "      <td>467.6</td>\n",
       "      <td>0.1352</td>\n",
       "      <td>0.2010</td>\n",
       "      <td>0.25960</td>\n",
       "      <td>0.07431</td>\n",
       "      <td>0.2941</td>\n",
       "      <td>0.09180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>303</th>\n",
       "      <td>10.490</td>\n",
       "      <td>18.61</td>\n",
       "      <td>66.86</td>\n",
       "      <td>334.3</td>\n",
       "      <td>0.10680</td>\n",
       "      <td>0.06678</td>\n",
       "      <td>0.02297</td>\n",
       "      <td>0.01780</td>\n",
       "      <td>0.1482</td>\n",
       "      <td>0.06600</td>\n",
       "      <td>...</td>\n",
       "      <td>11.06</td>\n",
       "      <td>24.54</td>\n",
       "      <td>70.76</td>\n",
       "      <td>375.4</td>\n",
       "      <td>0.1413</td>\n",
       "      <td>0.1044</td>\n",
       "      <td>0.08423</td>\n",
       "      <td>0.06528</td>\n",
       "      <td>0.2213</td>\n",
       "      <td>0.07842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>12.250</td>\n",
       "      <td>17.94</td>\n",
       "      <td>78.27</td>\n",
       "      <td>460.3</td>\n",
       "      <td>0.08654</td>\n",
       "      <td>0.06679</td>\n",
       "      <td>0.03885</td>\n",
       "      <td>0.02331</td>\n",
       "      <td>0.1970</td>\n",
       "      <td>0.06228</td>\n",
       "      <td>...</td>\n",
       "      <td>13.59</td>\n",
       "      <td>25.22</td>\n",
       "      <td>86.60</td>\n",
       "      <td>564.2</td>\n",
       "      <td>0.1217</td>\n",
       "      <td>0.1788</td>\n",
       "      <td>0.19430</td>\n",
       "      <td>0.08211</td>\n",
       "      <td>0.3113</td>\n",
       "      <td>0.08132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186</th>\n",
       "      <td>18.310</td>\n",
       "      <td>18.58</td>\n",
       "      <td>118.60</td>\n",
       "      <td>1041.0</td>\n",
       "      <td>0.08588</td>\n",
       "      <td>0.08468</td>\n",
       "      <td>0.08169</td>\n",
       "      <td>0.05814</td>\n",
       "      <td>0.1621</td>\n",
       "      <td>0.05425</td>\n",
       "      <td>...</td>\n",
       "      <td>21.31</td>\n",
       "      <td>26.36</td>\n",
       "      <td>139.20</td>\n",
       "      <td>1410.0</td>\n",
       "      <td>0.1234</td>\n",
       "      <td>0.2445</td>\n",
       "      <td>0.35380</td>\n",
       "      <td>0.15710</td>\n",
       "      <td>0.3206</td>\n",
       "      <td>0.06938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>6.981</td>\n",
       "      <td>13.43</td>\n",
       "      <td>43.79</td>\n",
       "      <td>143.5</td>\n",
       "      <td>0.11700</td>\n",
       "      <td>0.07568</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.1930</td>\n",
       "      <td>0.07818</td>\n",
       "      <td>...</td>\n",
       "      <td>7.93</td>\n",
       "      <td>19.54</td>\n",
       "      <td>50.41</td>\n",
       "      <td>185.2</td>\n",
       "      <td>0.1584</td>\n",
       "      <td>0.1202</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.2932</td>\n",
       "      <td>0.09382</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     mean radius  mean texture  mean perimeter  mean area  mean smoothness  \\\n",
       "478       11.490         14.59           73.99      404.9          0.10460   \n",
       "303       10.490         18.61           66.86      334.3          0.10680   \n",
       "155       12.250         17.94           78.27      460.3          0.08654   \n",
       "186       18.310         18.58          118.60     1041.0          0.08588   \n",
       "101        6.981         13.43           43.79      143.5          0.11700   \n",
       "\n",
       "     mean compactness  mean concavity  mean concave points  mean symmetry  \\\n",
       "478           0.08228         0.05308              0.01969         0.1779   \n",
       "303           0.06678         0.02297              0.01780         0.1482   \n",
       "155           0.06679         0.03885              0.02331         0.1970   \n",
       "186           0.08468         0.08169              0.05814         0.1621   \n",
       "101           0.07568         0.00000              0.00000         0.1930   \n",
       "\n",
       "     mean fractal dimension  ...  worst radius  worst texture  \\\n",
       "478                 0.06574  ...         12.40          21.90   \n",
       "303                 0.06600  ...         11.06          24.54   \n",
       "155                 0.06228  ...         13.59          25.22   \n",
       "186                 0.05425  ...         21.31          26.36   \n",
       "101                 0.07818  ...          7.93          19.54   \n",
       "\n",
       "     worst perimeter  worst area  worst smoothness  worst compactness  \\\n",
       "478            82.04       467.6            0.1352             0.2010   \n",
       "303            70.76       375.4            0.1413             0.1044   \n",
       "155            86.60       564.2            0.1217             0.1788   \n",
       "186           139.20      1410.0            0.1234             0.2445   \n",
       "101            50.41       185.2            0.1584             0.1202   \n",
       "\n",
       "     worst concavity  worst concave points  worst symmetry  \\\n",
       "478          0.25960               0.07431          0.2941   \n",
       "303          0.08423               0.06528          0.2213   \n",
       "155          0.19430               0.08211          0.3113   \n",
       "186          0.35380               0.15710          0.3206   \n",
       "101          0.00000               0.00000          0.2932   \n",
       "\n",
       "     worst fractal dimension  \n",
       "478                  0.09180  \n",
       "303                  0.07842  \n",
       "155                  0.08132  \n",
       "186                  0.06938  \n",
       "101                  0.09382  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_limits(df, variable, fold):\n",
    "    lower_limit = df[variable].mean() - fold * df[variable].std()\n",
    "    upper_limit = df[variable].mean() + fold * df[variable].std()\n",
    "    return lower_limit, upper_limit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.06356074164705164, 0.20149734880520967)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# we find the limits\n",
    "\n",
    "lower_limit, upper_limit = find_limits(X_train, \"worst smoothness\", 3)\n",
    "lower_limit, upper_limit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.07117, 0.20149734880520967)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Cap outliers\n",
    "\n",
    "X_train[\"worst smoothness\"].clip(lower=lower_limit, upper=upper_limit, inplace=True)\n",
    "X_test[\"worst smoothness\"].clip(lower=lower_limit, upper=upper_limit, inplace=True)\n",
    "\n",
    "X_train[\"worst smoothness\"].min(), X_train[\"worst smoothness\"].max()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature-engine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((398, 30), (171, 30))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's separate the data into training and testing sets\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X,\n",
    "    y,\n",
    "    test_size=0.3,\n",
    "    random_state=0,\n",
    ")\n",
    "\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Winsorizer(tail=&#x27;both&#x27;, variables=[&#x27;worst smoothness&#x27;, &#x27;worst texture&#x27;])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Winsorizer</label><div class=\"sk-toggleable__content\"><pre>Winsorizer(tail=&#x27;both&#x27;, variables=[&#x27;worst smoothness&#x27;, &#x27;worst texture&#x27;])</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "Winsorizer(tail='both', variables=['worst smoothness', 'worst texture'])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "capper = Winsorizer(\n",
    "    variables=[\"worst smoothness\", \"worst texture\"],\n",
    "    capping_method=\"gaussian\",\n",
    "    tail=\"both\",\n",
    "    fold=3,\n",
    ")\n",
    "\n",
    "capper.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'worst smoothness': 0.06356074164705164, 'worst texture': 7.092123638591499}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "capper.left_tail_caps_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'worst smoothness': 0.20149734880520967, 'worst texture': 43.97692158753917}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "capper.right_tail_caps_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "worst smoothness     0.201497\n",
       "worst texture       43.976922\n",
       "dtype: float64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = capper.transform(X_train)\n",
    "X_test = capper.transform(X_test)\n",
    "\n",
    "X_train[capper.variables_].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fsml",
   "language": "python",
   "name": "fsml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}