diff --git a/README.md b/README.md index c87d13553..9fea931de 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,7 @@ transforming parameters from the data and then transform it. * RecursiveFeatureElimination * RecursiveFeatureAddition * DropHighPSIFeatures + * SelectByInformationValue ### Datetime * DatetimeFeatures diff --git a/docs/api_doc/selection/SelectByInformationValue.rst b/docs/api_doc/selection/SelectByInformationValue.rst new file mode 100644 index 000000000..7c2897cb6 --- /dev/null +++ b/docs/api_doc/selection/SelectByInformationValue.rst @@ -0,0 +1,5 @@ +SelectByInformationValue +======================== + +.. autoclass:: feature_engine.selection.SelectByInformationValue + :members: diff --git a/docs/api_doc/selection/index.rst b/docs/api_doc/selection/index.rst index 659b36591..b17b33c57 100644 --- a/docs/api_doc/selection/index.rst +++ b/docs/api_doc/selection/index.rst @@ -4,17 +4,18 @@ Feature Selection ================= -Feature-engine's feature selection transformers are used to drop subsets of variables, -or in other words, to select subsets of variables. Feature-engine hosts selection -algorithms that are, in general, not available in other libraries. These algorithms have -been gathered from data science competitions or used in the industry. +Feature-engine's feature selection transformers are used to drop subsets of variables +with low predictive value. Feature-engine hosts selection algorithms that are, in general, +not available in other libraries. These algorithms have been gathered from data science +competitions or used in the industry. -Feature-engine's transformers select features based on 2 strategies. They either select -features by looking at the features intrinsic characteristics, like distributions or their -relationship with other features. Or they select features based on their impact on the -machine learning model performance. +Feature-engine's transformers select features based on different strategies. Some algorithms +remove constant or quasi-constant features. Some algorithms remove duplicated or correlated +variables. Some algorithms select features based on a machine learning model performance. +Some transformers implement selection procedures used in finance. And some transformers support +functionality that has been developed in the industry or in data science competitions. -In the following tables you find the algorithms that belong to either category. +In the following tables you find the algorithms that belong to each category. Selection based on feature characteristics ------------------------------------------ @@ -27,20 +28,37 @@ Selection based on feature characteristics :class:`DropDuplicateFeatures()` √ √ Drops features that are duplicated :class:`DropCorrelatedFeatures()` × √ Drops features that are correlated :class:`SmartCorrelatedSelection()` × √ From a correlated feature group drops the less useful features +============================================ ======================= ============= ==================================================================================== + +Selection based on a machine learning model +------------------------------------------- + +============================================ ======================= ============= ==================================================================================== + Transformer Categorical variables Allows NA Description +============================================ ======================= ============= ==================================================================================== +:class:`SelectBySingleFeaturePerformance()` × × Selects features based on single feature model performance +:class:`RecursiveFeatureElimination()` × × Removes features recursively by evaluating model performance +:class:`RecursiveFeatureAddition()` × × Adds features recursively by evaluating model performance +============================================ ======================= ============= ==================================================================================== + +Selection methods commonly used in finance +------------------------------------------ + +============================================ ======================= ============= ==================================================================================== + Transformer Categorical variables Allows NA Description +============================================ ======================= ============= ==================================================================================== :class:`DropHighPSIFeatures()` × √ Drops features with high Population Stability Index +:class:`SelectByInformationValue()` √ x Drops features with low information value ============================================ ======================= ============= ==================================================================================== -Selection based on model performance ------------------------------------- +Alternative feature selection methods +------------------------------------- ============================================ ======================= ============= ==================================================================================== Transformer Categorical variables Allows NA Description ============================================ ======================= ============= ==================================================================================== :class:`SelectByShuffling()` × × Selects features if shuffling their values causes a drop in model performance -:class:`SelectBySingleFeaturePerformance()` × × Removes observations with missing data from the dataset :class:`SelectByTargetMeanPerformance()` √ × Using the target mean as performance proxy, selects high performing features -:class:`RecursiveFeatureElimination()` × × Removes features recursively by evaluating model performance -:class:`RecursiveFeatureAddition()` × × Adds features recursively by evaluating model performance ============================================ ======================= ============= ==================================================================================== @@ -53,12 +71,13 @@ Selection based on model performance DropDuplicateFeatures DropCorrelatedFeatures SmartCorrelatedSelection - DropHighPSIFeatures - SelectByShuffling SelectBySingleFeaturePerformance - SelectByTargetMeanPerformance RecursiveFeatureElimination RecursiveFeatureAddition + DropHighPSIFeatures + SelectByInformationValue + SelectByShuffling + SelectByTargetMeanPerformance Other Feature Selection Libraries --------------------------------- diff --git a/docs/images/selectionChart.png b/docs/images/selectionChart.png index 9e1f91d23..1caeb0484 100644 Binary files a/docs/images/selectionChart.png and b/docs/images/selectionChart.png differ diff --git a/docs/index.rst b/docs/index.rst index d66646b37..d2d0f33e3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -174,6 +174,7 @@ Feature Selection: - :doc:`api_doc/selection/DropCorrelatedFeatures`: drops correlated variables from a dataframe - :doc:`api_doc/selection/SmartCorrelatedSelection`: selects best features from correlated groups - :doc:`api_doc/selection/DropHighPSIFeatures`: selects features based on the Population Stability Index (PSI) +- :doc:`api_doc/selection/SelectByInformationValue`: selects features based on their information value - :doc:`api_doc/selection/SelectByShuffling`: selects features by evaluating model performance after feature shuffling - :doc:`api_doc/selection/SelectBySingleFeaturePerformance`: selects features based on their performance on univariate estimators - :doc:`api_doc/selection/SelectByTargetMeanPerformance`: selects features based on target mean encoding performance diff --git a/docs/user_guide/selection/SelectByInformationValue.rst b/docs/user_guide/selection/SelectByInformationValue.rst new file mode 100644 index 000000000..769efd41a --- /dev/null +++ b/docs/user_guide/selection/SelectByInformationValue.rst @@ -0,0 +1,279 @@ +.. _information_value: + +.. currentmodule:: feature_engine.selection + +SelectByInformationValue +======================== + +:class:`SelectByInformationValue()` selects features based on whether the feature's information value score is +greater than the threshold passed by the user. + +The IV is calculated as: + +.. math:: + + IV = ∑ (fraction of positive cases - fraction of negative cases) * WoE + +where: + +- the fraction of positive cases is the proportion of observations of class 1, from the total class 1 observations. +- the fraction of negative cases is the proportion of observations of class 0, from the total class 0 observations. +- WoE is the weight of the evidence. + +The WoE is calculated as: + +.. math:: + + WoE = ln(fraction of positive cases / fraction of negative cases) + +Information value (IV) is used to assess a feature's predictive power of a binary-class dependent +variable. To derive a feature's IV, the weight of evidence (WoE) must first be calculated for each +unique category or bin that comprises the feature. If a category or bin contains a large percentage +of true or positive labels compared to the percentage of false or negative labels, then that category +or bin will have a high WoE value. + +Once the WoE is derived, :class:`SelectByInformationValue()` calculates the IV for each variable. +A variable's IV is essentially the weighted sum of the individual WoE values for each category or bin +within that variable where the weights incorporate the absolute difference between the +numerator and denominator. This value assesses the feature's predictive power in capturing the binary +dependent variable. + +The table below presents a general framework for using IV to determine a variable's predictive power: + +.. list-table:: + :widths: 30 30 + :header-rows: 1 + + * - Information Value + - Predictive Power + * - < 0.02 + - Useless + * - 0.02 to 0.1 + - Weak + * - 0.1 to 0.3 + - Medium + * - 0.3 to 0.5 + - Strong + * - > 0.5 + - Suspicious, too good to be true + +Table taken from `listendata `_. + + +Example +------- +Let's see how to use this transformer to select variables from UC Irvine's credit approval data set which can +be found `here`_. This dataset concerns credit card applications. All attribute names and values have been changed +to meaningless symbols to protect confidentiality. + +The data is comprised of both numerical and categorical data. + +.. _here: https://archive-beta.ics.uci.edu/ml/datasets/credit+approval + +Let's import the required libraries and classes: + +.. code:: python + + import pandas as pd + import numpy as np + from sklearn.model_selection import train_test_split + from feature_engine.selection import SelectByInformationValue + +Let's now load and prepare the credit approval data: + +.. code:: python + + # load data + data = pd.read_csv('crx.data', header=None) + + # name variables + var_names = ['A' + str(s) for s in range(1,17)] + data.columns = var_names + data.rename(columns={'A16': 'target'}, inplace=True) + + # preprocess data + data = data.replace('?', np.nan) + data['A2'] = data['A2'].astype('float') + data['A14'] = data['A14'].astype('float') + data['target'] = data['target'].map({'+':1, '-':0}) + + # drop rows with missing data + data.dropna(axis=0, inplace=True) + + data.head() + +Let's now review the first 5 rows of the dataset: + +.. code:: python + + A1 A2 A3 A4 A5 A6 A7 A8 A9 A10 A11 A12 A13 A14 A15 target + 0 b 30.83 0.000 u g w v 1.25 t t 1 f g 202.0 0 1 + 1 a 58.67 4.460 u g q h 3.04 t t 6 f g 43.0 560 1 + 2 a 24.50 0.500 u g q h 1.50 t f 0 f g 280.0 824 1 + 3 b 27.83 1.540 u g w v 3.75 t t 5 t g 100.0 3 1 + 4 b 20.17 5.625 u g w v 1.71 t f 0 f s 120.0 0 1 + + +Let's now split the data into train and test sets: + +.. code:: python + + # separate train and test sets + X_train, X_test, y_train, y_test = train_test_split( + data.drop(['target'], axis=1), + data['target'], + test_size=0.2, + random_state=0) + + X_train.shape, X_test.shape + +We see the size of the datasets below. + +.. code:: python + + ((522, 15), (131, 15)) + +Now, we set up :class:`SelectByInformationValue()`. We will pass six categorical +variables to the parameter :code:`variables`. We will set the parameter :code:`threshold` +to `0.2`. We see from the above mentioned table that an IV score of 0.2 signifies medium +predictive power. + +.. code:: python + + sel = SelectByInformationValue( + variables=['A1', 'A6', 'A9', 'A10', 'A12', 'A13'], + threshold=0.2, + ) + + sel.fit(X_train, y_train) + +With :code:`fit()`, the transformer: + + - calculates the WoE for each variable + - calculates the the IV for each variable + - identifies the variables that have an IV score below the threshold + +In the attribute :code:`variables_`, we find the variables that were evaluated: + +.. code:: python + + ['A1', 'A6', 'A7', 'A9', 'A10', 'A12', 'A13'] + +In the attribute :code:`features_to_drop_`, we find the variables that were not selected: + +.. code:: python + + sel.features_to_drop_ + + ['A1', 'A12', 'A13'] + +The attribute :code:`information_values_` shows the IV scores for each variable. + +.. code:: python + + {'A1': 0.0009535686492270659, + 'A6': 0.6006252129425703, + 'A9': 2.9184484098456807, + 'A10': 0.8606638171665587, + 'A12': 0.012251943759377052, + 'A13': 0.04383964979386022} + +We see that the transformer correctly selected the features that have an IV score greater +than the :code:`threshold` which was set to 0.2. + +The transformer also has the method `get_support` with similar functionality to Scikit-learn's +selectors method. If you execute `sel.get_support()`, you obtain: + +.. code:: python + + [False, True, True, True, True, True, True, + True, True, True, True, False, False, True, + True] + +With :code:`transform()`, we can go ahead and drop the features that do not meet the threshold: + +.. code:: python + + Xtr = sel.transform(X_test) + + Xtr.head() + +.. code:: python + + A2 A3 A4 A5 A6 A7 A8 A9 A10 A11 A14 A15 + 564 42.17 5.04 u g q h 12.750 t f 0 92.0 0 + 519 39.17 1.71 u g x v 0.125 t t 5 480.0 0 + 14 45.83 10.50 u g q v 5.000 t t 7 0.0 0 + 257 20.00 0.00 u g d v 0.500 f f 0 144.0 0 + 88 34.00 4.50 u g aa v 1.000 t f 0 240.0 0 + + +Note that :code:`Xtr` includes all the numerical features - i.e., A2, A3, A8, A11, and A14 - because +we only evaluated a few of the categorical features. + +And, finally, we can also obtain the names of the features in the final transformed dataset: + +.. code:: python + + sel.get_feature_names_out() + + ['A2', 'A3', 'A4', 'A5', 'A6', 'A7', 'A8', 'A9', 'A10', 'A11', 'A14', 'A15'] + + +If we want to select from categorical and numerical variables, we can do so as well by +sorting the numerical variables into bins first. Let's sort them into 5 bins of equal-frequency: + +.. code:: python + + sel = SelectByInformationValue( + bins=5, + strategy="equal_frequency", + threshold=0.2, + ) + + sel.fit(X_train.drop(["A4", "A5", "A7"], axis=1), y_train) + +If we now inspect the information values: + +.. code:: python + + sel.information_values_ + +We see the following: + +.. code:: python + + {'A1': 0.0009535686492270659, + 'A2': 0.10319123021570434, + 'A3': 0.2596258749173557, + 'A6': 0.6006252129425703, + 'A8': 0.7291628533346297, + 'A9': 2.9184484098456807, + 'A10': 0.8606638171665587, + 'A11': 1.0634602064399297, + 'A12': 0.012251943759377052, + 'A13': 0.04383964979386022, + 'A14': 0.3316668794040285, + 'A15': 0.6228678069374612} + +And if we inspect the features to drop: + +.. code:: python + + sel.features_to_drop_ + +We see the following: + +.. code:: python + + ['A1', 'A2', 'A12', 'A13'] + + +Note +---- + +The WoE is given by a logarithm of a fraction. Thus, if for any category or bin, the fraction of +observations of class 0 is 0, the WoE is not defined, and the transformer will raise an error. + +If you encounter this problem try grouping variables into fewer bins if they are numerical, +or grouping rare categories with the RareLabelEncoder if they are categorical. \ No newline at end of file diff --git a/docs/user_guide/selection/index.rst b/docs/user_guide/selection/index.rst index fd7ef34d0..e53aed329 100644 --- a/docs/user_guide/selection/index.rst +++ b/docs/user_guide/selection/index.rst @@ -8,41 +8,42 @@ Feature Selection ================= Feature-engine's feature selection transformers identify features with low predictive -performance and drop them from the dataset. To our knowledge, the feature selection -algorithms supported by Feature-engine are not yet available in other libraries. These -algorithms have been gathered from data science competitions or used in the industry. +performance and drop them from the dataset. Most of the feature selection algorithms +supported by Feature-engine are not yet available in other libraries. These algorithms +have been gathered from data science competitions or used in the industry. Selection Mechanism Overview ---------------------------- -Feature-engine's transformers select features based on 2 mechanism. The first mechanism -involves selecting features based on the features intrinsic characteristics like distributions -or their relationship with other features. The second mechanism involves selecting features -based on their impact on the machine learning model performance. In this context, features -are evaluated individually or as part of a feature group by different algorithms. +Feature-engine’s transformers select features based on different strategies. + +The first strategy evaluates the features intrinsic characteristics, like their distributions. +For example, we can remove constant or quasi-constant features. Or we can remove features +whose distribution in unstable in time by using the Population Stability Index. + +A second strategy consists in determining the relationships between features. Among these, +we can remove features that are duplicated or correlated. + +We can also select features based on their relationship with the target. To assess this, +we can replace the feature values by the target mean, or calculate the information value. + +Some feature selection procedures involve training machine learning models. We can assess +features individually, or collectively, through various algorithms, as shown in the +following diagram: .. figure:: ../../images/selectionChart.png :align: center Selection mechanisms - Overview -For example, in the first pillar, features will be selected based on the diversity of their -values, changes in their distribution or their relation to other features. This way, -features that show the same value in all or almost all the observations will be dropped, -features which distribution changes in time will be dropped, or duplicated or correlated -features will be dropped. - -Algorithms that select features based on individual feature performance will select features -by either training a machine learning model using an individual feature, or estimating model -performance with a single feature using a prediction proxy. Algorithms that select features based on their performance within a group of variables, will normally train a model with all the features, and then remove or add or shuffle a feature and re-evaluate the model performance. -These methods are normally geared to improve the overall performance of the final machine learning model -as well as reducing the feature space. +These methods are normally geared towards improving the overall performance of the final +machine learning model as well as reducing the feature space. Selectors Characteristics Overview @@ -52,10 +53,10 @@ Some Feature-engine's selectors work with categorical variables off-the-shelf an missing data in the variables. These gives you the opportunity to quickly screen features before jumping into any feature engineering. -In the following tables we highlight the main Feature-engine selectors characteristics: +In the following tables, we highlight the main Feature-engine selectors characteristics: Selection based on feature characteristics -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------ ============================================ ======================= ============= ==================================================================================== Transformer Categorical variables Allows NA Description @@ -65,32 +66,37 @@ Selection based on feature characteristics :class:`DropDuplicateFeatures()` √ √ Drops features that are duplicated :class:`DropCorrelatedFeatures()` × √ Drops features that are correlated :class:`SmartCorrelatedSelection()` × √ From a correlated feature group drops the less useful features -:class:`DropHighPSIFeatures()` × √ Drops features with high Population Stability Index ============================================ ======================= ============= ==================================================================================== -Selection based on model performance -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Methods that determine duplication or the number of unique values, can work with both +numerical and categorical variables and support missing data as well. + +Selection procedures based on correlation work only with numerical variables but allow +missing data. + +Selection based on a machine learning model +------------------------------------------- ============================================ ======================= ============= ==================================================================================== Transformer Categorical variables Allows NA Description ============================================ ======================= ============= ==================================================================================== -:class:`SelectByShuffling()` × × Selects features if shuffling their values causes a drop in model performance -:class:`SelectBySingleFeaturePerformance()` × × Removes observations with missing data from the dataset -:class:`SelectByTargetMeanPerformance()` √ × Using the target mean as performance proxy, selects high performing features +:class:`SelectBySingleFeaturePerformance()` × × Selects features based on single feature model performance :class:`RecursiveFeatureElimination()` × × Removes features recursively by evaluating model performance :class:`RecursiveFeatureAddition()` × × Adds features recursively by evaluating model performance ============================================ ======================= ============= ==================================================================================== -In short, selection procedures that require training a machine learning model from Scikit-learn -require numerical variables without missing data. Selection procedures based on correlation work -only with numerical variables but allow missing data. Methods that determine duplication or -the number of unique values can work with both numerical and categorical variables and support -missing data as well. +Selection procedures that require training a machine learning model from Scikit-learn +require numerical variables without missing data. -The :class:`SelectBySingleFeaturePerformance()` uses the target mean value as proxy for prediction, -replacing categories or variable intervals by these values and then determining a performance metric. -Thus, it is suitable for both categorical and numerical variables. In its current implementation, -it does not support missing data. +Selection methods commonly used in finance +------------------------------------------ + +============================================ ======================= ============= ==================================================================================== + Transformer Categorical variables Allows NA Description +============================================ ======================= ============= ==================================================================================== +:class:`DropHighPSIFeatures()` × √ Drops features with high Population Stability Index +:class:`SelectByInformationValue()` √ x Drops features with low information value +============================================ ======================= ============= ==================================================================================== :class:`DropHighPSIFeatures()` allows to remove features with changes in their distribution. This is done by splitting the input dataframe in two parts and comparing the distribution of each feature in the two @@ -98,7 +104,22 @@ parts. The metric used to assess distribution shift is the Population Stability unstable features may lead to more robust models. In fields like Credit Risk Modelling, the Regulator often requires the PSI of the final feature set to be below are given threshold. -Throughout the user guide, you will find more details about each of the feature selection procedures. +Alternative feature selection methods +------------------------------------- + +============================================ ======================= ============= ==================================================================================== + Transformer Categorical variables Allows NA Description +============================================ ======================= ============= ==================================================================================== +:class:`SelectByShuffling()` × × Selects features if shuffling their values causes a drop in model performance +:class:`SelectByTargetMeanPerformance()` √ × Using the target mean as performance proxy, selects high performing features +============================================ ======================= ============= ==================================================================================== + +The :class:`SelectByTargetMeanPerformance()` uses the target mean value as proxy for prediction, +replacing categories or variable intervals by these values and then determining a performance metric. +Thus, it is suitable for both categorical and numerical variables. In its current implementation, +it does not support missing data. + +Throughout the rest of user guide, you will find more details about each of the feature selection procedures. Feature Selection Algorithms ---------------------------- @@ -113,12 +134,13 @@ Click below to find more details on how to use each one of the transformers. DropDuplicateFeatures DropCorrelatedFeatures SmartCorrelatedSelection - DropHighPSIFeatures - SelectByShuffling SelectBySingleFeaturePerformance - SelectByTargetMeanPerformance RecursiveFeatureElimination RecursiveFeatureAddition + SelectByShuffling + SelectByTargetMeanPerformance + DropHighPSIFeatures + SelectByInformationValue Additional Resources @@ -127,4 +149,5 @@ Additional Resources More details about feature selection can be found in the following resources: - `Feature Selection Online Course `_ -- `Feature Selection for Machine Learning: A comprehensive Overview `_ +- `Feature Selection book `_ +- `Train in data's blog `_ diff --git a/feature_engine/encoding/woe.py b/feature_engine/encoding/woe.py index 707957849..999f1a3ae 100644 --- a/feature_engine/encoding/woe.py +++ b/feature_engine/encoding/woe.py @@ -22,8 +22,8 @@ _transform_encoders_docstring, ) from feature_engine._docstrings.substitute import Substitution -from feature_engine.encoding._helper_functions import check_parameter_unseen from feature_engine.dataframe_checks import check_X_y +from feature_engine.encoding._helper_functions import check_parameter_unseen from feature_engine.encoding.base_encoder import ( CategoricalInitMixin, CategoricalMethodsMixin, @@ -31,6 +31,44 @@ from feature_engine.tags import _return_tags +class WoE: + def _check_fit_input(self, X: pd.DataFrame, y: pd.Series): + """ + Check that X is dataframe, and y a binary series with values 0 and 1. + """ + X, y = check_X_y(X, y) + + # check that y is binary + if y.nunique() != 2: + raise ValueError( + "This encoder is designed for binary classification. The target " + "used has more than 2 unique values." + ) + + # if target does not have values 0 and 1, we need to remap, to be able to + # compute the averages. + if y.min() != 0 or y.max() != 1: + y = pd.Series(np.where(y == y.min(), 0, 1)) + return X, y + + def _calculate_woe(self, X: pd.DataFrame, y: pd.Series, variable: Union[str, int]): + total_pos = y.sum() + inverse_y = y.ne(1).copy() + total_neg = inverse_y.sum() + + pos = y.groupby(X[variable]).sum() / total_pos + neg = inverse_y.groupby(X[variable]).sum() / total_neg + + if not (pos[:] == 0).sum() == 0 or not (neg[:] == 0).sum() == 0: + raise ValueError( + "The proportion of one of the classes for a category in " + "variable {} is zero, and log of zero is not defined".format(variable) + ) + + woe = np.log(pos / neg) + return pos, neg, woe + + @Substitution( ignore_format=_ignore_format_docstring, variables=_variables_categorical_docstring, @@ -42,7 +80,7 @@ transform=_transform_encoders_docstring, inverse_transform=_inverse_transform_docstring, ) -class WoEEncoder(CategoricalInitMixin, CategoricalMethodsMixin): +class WoEEncoder(CategoricalInitMixin, CategoricalMethodsMixin, WoE): """ The WoEEncoder() replaces categories by the weight of evidence (WoE). The WoE was used primarily in the financial sector to create credit risk @@ -145,40 +183,15 @@ def fit(self, X: pd.DataFrame, y: pd.Series): Target, must be binary. """ - X, y = check_X_y(X, y) - - # check that y is binary - if y.nunique() != 2: - raise ValueError( - "This encoder is designed for binary classification. The target " - "used has more than 2 unique values." - ) - - # if target does not have values 0 and 1, we need to remap, to be able to - # compute the averages. - if y.min() != 0 or y.max() != 1: - y = pd.Series(np.where(y == y.min(), 0, 1)) + X, y = self._check_fit_input(X, y) self._fit(X) self._get_feature_names_in(X) self.encoder_dict_ = {} - total_pos = y.sum() - inverse_y = y.ne(1).copy() - total_neg = inverse_y.sum() - for var in self.variables_: - pos = y.groupby(X[var]).sum() / total_pos - neg = inverse_y.groupby(X[var]).sum() / total_neg - - if not (pos[:] == 0).sum() == 0 or not (neg[:] == 0).sum() == 0: - raise ValueError( - "The proportion of one of the classes for a category in " - "variable {} is zero, and log of zero is not defined".format(var) - ) - - woe = np.log(pos / neg) + _, _, woe = self._calculate_woe(X, y, var) self.encoder_dict_[var] = woe.to_dict() diff --git a/feature_engine/selection/__init__.py b/feature_engine/selection/__init__.py index 5d1d82542..7ac07138c 100644 --- a/feature_engine/selection/__init__.py +++ b/feature_engine/selection/__init__.py @@ -12,6 +12,7 @@ from .single_feature_performance import SelectBySingleFeaturePerformance from .smart_correlation_selection import SmartCorrelatedSelection from .target_mean_selection import SelectByTargetMeanPerformance +from .information_value import SelectByInformationValue __all__ = [ "DropFeatures", @@ -25,4 +26,5 @@ "RecursiveFeatureAddition", "RecursiveFeatureElimination", "SelectByTargetMeanPerformance", + "SelectByInformationValue", ] diff --git a/feature_engine/selection/information_value.py b/feature_engine/selection/information_value.py new file mode 100644 index 000000000..4e2e18204 --- /dev/null +++ b/feature_engine/selection/information_value.py @@ -0,0 +1,259 @@ +from typing import List, Union + +import numpy as np +import pandas as pd + +from feature_engine._docstrings.fit_attributes import ( + _feature_names_in_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, +) +from feature_engine._docstrings.init_parameters import _ignore_format_docstring +from feature_engine._docstrings.methods import _fit_transform_docstring +from feature_engine._docstrings.substitute import Substitution +from feature_engine._variable_handling.init_parameter_checks import ( + _check_init_parameter_variables, +) +from feature_engine._variable_handling.variable_type_selection import ( + _find_all_variables, + _find_categorical_and_numerical_variables, +) +from feature_engine.dataframe_checks import _check_contains_inf, _check_contains_na +from feature_engine.discretisation import ( + EqualFrequencyDiscretiser, + EqualWidthDiscretiser, +) +from feature_engine.encoding.woe import WoE +from feature_engine.selection._docstring import ( + _features_to_drop_docstring, + _get_support_docstring, + _threshold_docstring, +) +from feature_engine.selection.base_selector import BaseSelector +from feature_engine.tags import _return_tags + +Variables = Union[None, int, str, List[Union[str, int]]] + + +@Substitution( + threshold=_threshold_docstring, + ignore_format=_ignore_format_docstring, + variables_=_variables_attribute_docstring, + features_to_drop=_features_to_drop_docstring, + feature_names_in=_feature_names_in_docstring, + n_features_in=_n_features_in_docstring, + fit_transform=_fit_transform_docstring, + confirm_variables=BaseSelector._confirm_variables_docstring, + get_support=_get_support_docstring, +) +class SelectByInformationValue(BaseSelector, WoE): + """ + SelectByInformationValue() selects features based on their information value (IV). + The IV is calculated as: + + .. math:: + + IV = ∑ (fraction of positive cases - fraction of negative cases) * WoE + + where: + + - the fraction of positive cases is the proportion of observations of class 1, + from the total class 1 observations. + - the fraction of negative cases is the proportion of observations of class 0, + from the total class 0 observations. + - WoE is the weight of the evidence. + + SelectByInformationValue() is only suitable to select features for binary + classification. + + SelectByInformationValue() can determine the IV for numerical and categorical + variables. For numerical variables, it first sorts the variables into intervals, + and then determines the IV. + + You can pass a list of variables to examine. Alternatively, the transformer will + examine all variables. + + The IV allows you to assess each variable's independent contribution to the target + variable. The transformer selects those variables whose IV is higher than the + threshold. + + More details in the :ref:`User Guide `. + + + Parameters + ---------- + variables: list, default=None + The list of variables to evaluate. If None, the transformer will evaluate all + variables in the dataset (except datetime). + + bins: int, default = 5 + If the dataset contains numerical variables, the number of bins into which + the values will be sorted. + + strategy: str, default = 'equal_width' + Whether the bins should be of equal width ('equal_width') or equal frequency + ('equal_frequency'). + + threshold: float, int, default = 0.2. + The threshold to drop a feature. If the IV for a feature is < threshold, the + feature will be dropped. + + {confirm_variables} + + Attributes + ---------- + {variables_} + + information_values_: + A dictionary with the information values for each feature. + + {features_to_drop} + + {feature_names_in} + + {n_features_in} + + Methods + ------- + fit: + Find features with high information value. + + {fit_transform} + + {get_support} + + transform: + Remove features with low information value. + + See Also + -------- + feature_engine.encoding.WoEEncoder + feature_engine.discretisation.EqualWidthDiscretiser + feature_engine.discretisation.EqualFrequencyDiscretiser + + References + ---------- + .. [1] Weight of evidence and information value explained + https://www.listendata.com/2015/03/weight-of-evidence-woe-and-information.html + + .. [2] WoE and IV for continuous variables + https://www.listendata.com/2019/08/WOE-IV-Continuous-Dependent.html + """ + + def __init__( + self, + variables: Variables = None, + bins: int = 5, + strategy: str = "equal_width", + threshold: Union[float, int] = 0.2, + confirm_variables: bool = False, + ) -> None: + + if not isinstance(bins, int) or isinstance(bins, int) and bins <= 0: + raise ValueError(f"bins must be an integer. Got {bins} instead.") + + if strategy not in ["equal_width", "equal_frequency"]: + raise ValueError( + "strategy takes only values 'equal_width' or 'equal_frequency'. " + f"Got {strategy} instead." + ) + + if not isinstance(threshold, (int, float)): + raise ValueError( + f"threshold must be a an integer or a float. Got {threshold} " + "instead." + ) + + self.variables = _check_init_parameter_variables(variables) + self.bins = bins + self.strategy = strategy + self.threshold = threshold + self.confirm_variables = confirm_variables + + def fit(self, X: pd.DataFrame, y: pd.Series): + """ + Learn the information value. Find features with IV above the threshold. + + Parameters + ---------- + X: pandas dataframe of shape = [n_samples, n_features] + The training input samples. + + y: pandas series of shape = [n_samples, ] + Target, must be binary. + """ + # check input dataframe + X, y = self._check_fit_input(X, y) + + # If required exclude variables that are not in the input dataframe + self._confirm_variables(X) + + # find categorical and numerical variables + # find all variables or check those entered are present in the dataframe + self.variables_ = _find_all_variables(X, self.variables_, exclude_datetime=True) + + _, variables_numerical = _find_categorical_and_numerical_variables( + X, self.variables_ + ) + + # check for missing values + _check_contains_na(X, self.variables_) + _check_contains_inf(X, variables_numerical) + + # get input df features number and name + self._get_feature_names_in(X) + + # If there are numerical variables, discretize them + if len(variables_numerical) > 0: + discretiser = self._make_discretiser(variables_numerical) + X = discretiser.fit_transform(X) + + self.information_values_ = {} + for var in self.variables_: + total_pos, total_neg, woe = self._calculate_woe(X, y, var) + iv = self._calculate_iv(total_pos, total_neg, woe) + self.information_values_[var] = iv + + self.features_to_drop_ = [ + f + for f in self.information_values_.keys() + if self.information_values_[f] < self.threshold + ] + + return self + + def _calculate_iv(self, pos, neg, woe): + return np.sum((pos - neg) * woe) + + def _make_discretiser(self, variables): + """ + Instantiate the EqualWidthDiscretiser or EqualFrequencyDiscretiser. + """ + if self.strategy == "equal_width": + discretiser = EqualWidthDiscretiser( + bins=self.bins, + variables=variables, + return_boundaries=True, + ) + else: + discretiser = EqualFrequencyDiscretiser( + q=self.bins, + variables=variables, + return_boundaries=True, + ) + + return discretiser + + def _more_tags(self): + tags_dict = _return_tags() + tags_dict["variables"] = "all" + tags_dict["requires_y"] = True + tags_dict["binary_only"] = True + # in the current format, the tests are performed using continuous np.arrays + # this means that when we encode some of the values, the denominator is 0 + # and this the transformer raises an error, and the test fails. + # For this reason, most sklearn transformers will fail. And it has nothing to + # do with the class not being compatible, it is just that the inputs passed + # are not suitable + tags_dict["_skip_test"] = True + return tags_dict diff --git a/feature_engine/selection/target_mean_selection.py b/feature_engine/selection/target_mean_selection.py index 6704aef58..e90d013ca 100644 --- a/feature_engine/selection/target_mean_selection.py +++ b/feature_engine/selection/target_mean_selection.py @@ -113,7 +113,7 @@ class SelectByTargetMeanPerformance(BaseSelector): the values will be sorted. strategy: str, default = 'equal_width' - Whether the bins should of equal width ('equal_width') or equal frequency + Whether the bins should be of equal width ('equal_width') or equal frequency ('equal_frequency'). {scoring} diff --git a/tests/test_encoding/test_woe_class.py b/tests/test_encoding/test_woe_class.py new file mode 100644 index 000000000..55404d641 --- /dev/null +++ b/tests/test_encoding/test_woe_class.py @@ -0,0 +1,16 @@ +import numpy as np +import pandas as pd + +from feature_engine.encoding.woe import WoE + + +def test_woe_calculation(df_enc): + pos_exp = pd.Series({"A": 0.333333, "B": 0.333333, "C": 0.333333}) + neg_exp = pd.Series({"A": 0.285714, "B": 0.571429, "C": 0.142857}) + + woe_class = WoE() + pos, neg, woe = woe_class._calculate_woe(df_enc, df_enc["target"], "var_A") + + pd.testing.assert_series_equal(pos, pos_exp, check_names=False) + pd.testing.assert_series_equal(neg, neg_exp, check_names=False) + pd.testing.assert_series_equal(np.log(pos / neg), woe, check_names=False) diff --git a/tests/test_selection/test_check_estimator_selectors.py b/tests/test_selection/test_check_estimator_selectors.py index 09f9797a9..67aba7226 100644 --- a/tests/test_selection/test_check_estimator_selectors.py +++ b/tests/test_selection/test_check_estimator_selectors.py @@ -14,6 +14,7 @@ SelectBySingleFeaturePerformance, SelectByTargetMeanPerformance, SmartCorrelatedSelection, + SelectByInformationValue, ) from tests.estimator_checks.estimator_checks import check_feature_engine_estimator from tests.estimator_checks.init_params_triggered_functionality_checks import ( @@ -35,6 +36,7 @@ SelectBySingleFeaturePerformance(estimator=_logreg, scoring="accuracy"), RecursiveFeatureAddition(estimator=_logreg, scoring="accuracy"), RecursiveFeatureElimination(estimator=_logreg, scoring="accuracy", threshold=-100), + SelectByInformationValue(bins=2), ] _multivariate_estimators = [ @@ -52,8 +54,9 @@ DropHighPSIFeatures(bins=5), SelectByTargetMeanPerformance(bins=3, regression=False, threshold=0), SelectBySingleFeaturePerformance( - estimator=_logreg, scoring="accuracy", threshold=0 + estimator=_logreg, scoring="accuracy", threshold=0, ), + SelectByInformationValue(bins=2), ] _model_based_estimators = [ diff --git a/tests/test_selection/test_information_value.py b/tests/test_selection/test_information_value.py new file mode 100644 index 000000000..35880ae3a --- /dev/null +++ b/tests/test_selection/test_information_value.py @@ -0,0 +1,160 @@ +import math +import numpy as np +import pandas as pd +import pytest + +from feature_engine.selection import SelectByInformationValue + + +@pytest.mark.parametrize("_threshold", ["python", (True, False), [4.3, 3]]) +def test_error_when_not_permitted_threshold(_threshold): + with pytest.raises(ValueError): + SelectByInformationValue(threshold=_threshold) + + +@pytest.mark.parametrize("_bins", ["python", (True, False), 4.3, -1]) +def test_error_when_not_permitted_bins(_bins): + with pytest.raises(ValueError): + SelectByInformationValue(bins=_bins) + + +@pytest.mark.parametrize("_strategy", ["python", (True, False), 4.3, -1]) +def test_error_when_not_permitted_strategy(_strategy): + with pytest.raises(ValueError): + SelectByInformationValue(strategy=_strategy) + + +def test_raises_error_when_target_not_binary(df_enc_numeric): + transformer = SelectByInformationValue() + with pytest.raises(ValueError): + transformer.fit(df_enc_numeric[["var_A", "target"]], df_enc_numeric["var_B"]) + + +def test_calculate_iv_method(): + # values taken from here: + # https://www.listendata.com/2015/03/weight-of-evidence-woe-and-information.html + pos = np.array( + [ + 0.05379574, + 0.12288367, + 0.134352813, + 0.163025669, + 0.166302567, + 0.158929547, + 0.105406881, + 0.045057346, + 0.050245767, + ] + ) + + neg = np.array( + [ + 0.059171598, + 0.100591716, + 0.115384615, + 0.150887574, + 0.159763314, + 0.162721893, + 0.121301775, + 0.068047337, + 0.062130178, + ] + ) + woe = np.log(pos / neg) + sel = SelectByInformationValue() + assert sel._calculate_iv(pos, neg, woe) == 0.0233856621001144 + + +def test_iv_dictionary(df_enc): + sel = SelectByInformationValue().fit(df_enc[["var_A", "var_B"]], df_enc["target"]) + expected_dict = {"var_A": 0.29706307738283366, "var_B": 0.29706307738283366} + assert sel.information_values_ == expected_dict + + +def test_transformer_with_default_params(): + df = pd.DataFrame( + { + "var_A": ["A"] * 6 + ["B"] * 10 + ["C"] * 4, + "var_B": ["A"] * 10 + ["B"] * 6 + ["C"] * 4, + "var_C": ["X"] * 7 + ["Y"] * 5 + ["Z"] * 8, + "var_D": ["L"] * 3 + ["M"] * 9 + ["N"] * 8, + "var_E": ["R"] * 7 + ["S"] * 4 + ["T"] * 9, + "target": [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0], + } + ) + X = df.drop("target", axis=1).copy() + y = df["target"].copy() + + sel = SelectByInformationValue() + sel.fit(df.drop("target", axis=1), df["target"]) + X_tr = sel.fit_transform(X, y) + + exp_dict = { + "var_A": 0.29706307738283366, + "var_B": 0.29706307738283366, + "var_C": 0.07817653204775647, + "var_D": 0.494962117149986, + "var_E": 0.024620803988822354, + } + + features_to_drop = ["var_C", "var_E"] + exp_df = X.drop(features_to_drop, axis=1) + + for key in exp_dict.keys(): + assert math.isclose(exp_dict[key], sel.information_values_[key]) + assert sel.features_to_drop_ == features_to_drop + assert X_tr.equals(exp_df) + + +def test_transformer_with_numerical_and_categorical_variables(df_enc): + df = pd.DataFrame( + { + "var_A": ["A"] * 6 + ["B"] * 10 + ["C"] * 4, + "var_B": ["A"] * 10 + ["B"] * 6 + ["C"] * 4, + "var_C": np.linspace(0, 20, num=20), + "var_D": np.linspace(0, 20, num=20), + "var_E": ["R"] * 7 + ["S"] * 4 + ["T"] * 9, + "target": [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0], + } + ) + X = df.drop("target", axis=1).copy() + y = df["target"].copy() + + sel = SelectByInformationValue(bins=3) + sel.fit(df.drop("target", axis=1), df["target"]) + X_tr = sel.fit_transform(X, y) + + exp_dict = { + "var_A": 0.29706307738283366, + "var_B": 0.29706307738283366, + "var_C": 0.010625883395914762, + "var_D": 0.010625883395914762, + "var_E": 0.024620803988822354, + } + + features_to_drop = ["var_C", "var_D", "var_E"] + exp_df = X.drop(features_to_drop, axis=1) + + for key in exp_dict.keys(): + assert math.isclose(exp_dict[key], sel.information_values_[key]) + assert sel.features_to_drop_ == features_to_drop + assert X_tr.equals(exp_df) + + +def test_transformer_with_equal_frequency_discretization(df_enc): + df = pd.DataFrame({ + "var_C": np.linspace(0, 20, num=20), + "var_D": np.linspace(0, 20, num=20), + "target": [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0], + }) + X = df.drop("target", axis=1).copy() + y = df["target"].copy() + + sel = SelectByInformationValue(bins=3, strategy="equal_frequency") + sel.fit(df.drop("target", axis=1), df["target"]) + sel.fit(X, y) + + exp_dict = {'var_C': 0.010625883395914762, 'var_D': 0.010625883395914762} + + for key in exp_dict.keys(): + assert math.isclose(exp_dict[key], sel.information_values_[key])