diff --git a/feature_engine/creation/_docstring.py b/feature_engine/creation/_docstring.py new file mode 100644 index 000000000..b78869dc0 --- /dev/null +++ b/feature_engine/creation/_docstring.py @@ -0,0 +1,14 @@ +_drop_original_docstring = """drop_original: bool, default=False + If True, the original variables will be dropped from the dataframe after + creating the features. + """.rstrip() + +_missing_values_docstring = """missing_values: string, default='raise' + Indicates if missing values should be ignored or raised. If 'raise' the + transformer will return an error if the the datasets to `fit` or `transform` + contain missing values. If 'ignore', missing data will be ignored when creating + the features. + """ +_transform_docstring = """transform: + Create and add the new features. + """.rstrip() diff --git a/feature_engine/creation/combine_with_reference_feature.py b/feature_engine/creation/combine_with_reference_feature.py index b3d2e47cc..91d5dd1c2 100644 --- a/feature_engine/creation/combine_with_reference_feature.py +++ b/feature_engine/creation/combine_with_reference_feature.py @@ -4,16 +4,35 @@ from sklearn.base import BaseEstimator, TransformerMixin from sklearn.utils.validation import check_is_fitted +from feature_engine.creation._docstring import ( + _drop_original_docstring, + _missing_values_docstring, + _transform_docstring, +) from feature_engine.dataframe_checks import ( _check_contains_inf, _check_contains_na, _check_input_matches_training_df, _is_dataframe, ) +from feature_engine.docstrings import ( + Substitution, + _fit_not_learn_docstring, + _fit_transform_docstring, + _n_features_in_docstring, +) from feature_engine.validation import _return_tags from feature_engine.variable_manipulation import _find_or_check_numerical_variables +@Substitution( + missing_values=_missing_values_docstring, + drop_original=_drop_original_docstring, + n_features_in_=_n_features_in_docstring, + fit=_fit_not_learn_docstring, + transform=_transform_docstring, + fit_transform=_fit_transform_docstring, +) class CombineWithReferenceFeature(BaseEstimator, TransformerMixin): """ CombineWithReferenceFeature() applies basic mathematical operations between a group @@ -42,9 +61,9 @@ class CombineWithReferenceFeature(BaseEstimator, TransformerMixin): operations: list, default=['sub'] The list of basic mathematical operations to be used in the transformation. - If None, all of ['sub', 'div','add','mul'] will be performed. Alternatively, + If None, all of ['sub', 'div', 'add', 'mul'] will be performed. Alternatively, you can enter a list of operations to carry out. Each operation should - be a string and must be one of the elements in `['sub', 'div','add', 'mul']`. + be a string and must be one of the elements in `['sub', 'div', 'add', 'mul']`. Each operation will result in a new variable that will be added to the transformed dataset. @@ -59,29 +78,21 @@ class CombineWithReferenceFeature(BaseEstimator, TransformerMixin): If `new_variable_names` is None, the transformer will assign an arbitrary name to the features. The name will be var + operation + ref_var. - missing_values: string, default='ignore' - Indicates if missing values should be ignored or raised. If 'ignore', the - transformer will ignore missing data when transforming the data. If 'raise' the - transformer will return an error if the training or the datasets to transform - contain missing values. + {missing_values} - drop_original: bool, default=False - If True, the original variables will be dropped from the dataframe - after their combination. + {drop_original} Attributes ---------- - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- - fit: - This transformer does not learn parameters. - transform: - Combine the variables with the mathematical operations. - fit_transform: - Fit to the data, then transform it. + {fit} + + {transform} + + {fit_transform} Notes ----- diff --git a/feature_engine/creation/cyclical.py b/feature_engine/creation/cyclical.py index ad42b0bd8..aade97611 100644 --- a/feature_engine/creation/cyclical.py +++ b/feature_engine/creation/cyclical.py @@ -4,9 +4,28 @@ import pandas as pd from feature_engine.base_transformers import BaseNumericalTransformer +from feature_engine.creation._docstring import ( + _drop_original_docstring, + _transform_docstring, +) +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, + _variables_numerical_docstring, +) from feature_engine.variable_manipulation import _check_input_parameter_variables +@Substitution( + variables=_variables_numerical_docstring, + drop_original=_drop_original_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + transform=_transform_docstring, + fit_transform=_fit_transform_docstring, +) class CyclicalTransformer(BaseNumericalTransformer): """ The CyclicalTransformer() applies cyclical transformations to numerical @@ -27,39 +46,32 @@ class CyclicalTransformer(BaseNumericalTransformer): Parameters ---------- - variables: list, default=None - The list of numerical variables to transform. If None, the transformer will - automatically find and select all numerical variables. + {variables} max_values: dict, default=None A dictionary with the maximum value of each variable to transform. Useful when the maximum value is not present in the dataset. If None, the transformer will automatically find the maximum value of each variable. - drop_original: bool, default=False - If True, the original variables to transform will be dropped from the dataframe. + {drop_original} Attributes ---------- max_values_: The maximum value of the cyclical feature. - variables_: - The group of variables that will be transformed. - - n_features_in_: - The number of features in the train set used in fit. + {variables_} + {n_features_in_} Methods ------- fit: Learns the maximum values of the cyclical features. - transform: - Applies the cyclical transformation. - fit_transform: - Fit to data, then transform it. + {transform} + + {fit_transform} References ---------- diff --git a/feature_engine/creation/mathematical_combination.py b/feature_engine/creation/mathematical_combination.py index 69d28b4ee..6db2953f9 100644 --- a/feature_engine/creation/mathematical_combination.py +++ b/feature_engine/creation/mathematical_combination.py @@ -4,16 +4,35 @@ from sklearn.base import BaseEstimator, TransformerMixin from sklearn.utils.validation import check_is_fitted +from feature_engine.creation._docstring import ( + _drop_original_docstring, + _missing_values_docstring, + _transform_docstring, +) from feature_engine.dataframe_checks import ( _check_contains_inf, _check_contains_na, _check_input_matches_training_df, _is_dataframe, ) +from feature_engine.docstrings import ( + Substitution, + _fit_not_learn_docstring, + _fit_transform_docstring, + _n_features_in_docstring, +) from feature_engine.validation import _return_tags from feature_engine.variable_manipulation import _find_or_check_numerical_variables +@Substitution( + missing_values=_missing_values_docstring, + drop_original=_drop_original_docstring, + n_features_in_=_n_features_in_docstring, + fit=_fit_not_learn_docstring, + transform=_transform_docstring, + fit_transform=_fit_transform_docstring, +) class MathematicalCombination(BaseEstimator, TransformerMixin): """ MathematicalCombination() applies basic mathematical operations to multiple @@ -55,11 +74,9 @@ class MathematicalCombination(BaseEstimator, TransformerMixin): to the newly created features starting by the name of the mathematical operation, followed by the variables combined separated by -. - missing_values: string, default='raise' - Indicates if missing values should be ignored or raised. If 'raise' the - transformer will return an error if the the datasets to `fit` or `transform` - contain missing values. If 'ignore', missing data will be ignored when - performing the calculations. + {missing_values} + + {drop_original} Attributes ---------- @@ -70,17 +87,15 @@ class MathematicalCombination(BaseEstimator, TransformerMixin): List with the mathematical operations to be applied to the `variables_to_combine`. - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- - fit: - This transformer does not learn parameters. - transform: - Combine the variables with the mathematical operations. - fit_transform: - Fit to the data, then transform it. + {fit} + + {transform} + + {fit_transform} Notes ----- diff --git a/feature_engine/datetime/datetime.py b/feature_engine/datetime/datetime.py index 8e25bc84f..318728c64 100644 --- a/feature_engine/datetime/datetime.py +++ b/feature_engine/datetime/datetime.py @@ -18,12 +18,23 @@ FEATURES_SUFFIXES, FEATURES_SUPPORTED, ) +from feature_engine.docstrings import ( + Substitution, + _fit_not_learn_docstring, + _fit_transform_docstring, + _n_features_in_docstring, +) from feature_engine.variable_manipulation import ( _check_input_parameter_variables, _find_or_check_datetime_variables, ) +@Substitution( + n_features_in_=_n_features_in_docstring, + fit=_fit_not_learn_docstring, + fit_transform=_fit_transform_docstring, +) class DatetimeFeatures(BaseEstimator, TransformerMixin): """ DatetimeFeatures extracts date and time features from datetime variables, adding @@ -103,17 +114,16 @@ class DatetimeFeatures(BaseEstimator, TransformerMixin): features_to_extract_: The date and time features that will be extracted from each variable. - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- - fit: - This transformer does not learn parameters. + {fit} + transform: Add the date and time features. - fit_transform: - Fit to the data, then transform it. + + {fit_transform} See also -------- @@ -130,7 +140,6 @@ def __init__( dayfirst: bool = False, yearfirst: bool = False, utc: Union[None, bool] = None, - ) -> None: if features_to_extract: @@ -162,10 +171,7 @@ def __init__( ) if utc is not None and not isinstance(utc, bool): - raise ValueError( - "utc takes only booleans or None. " - f"Got {utc} instead." - ) + raise ValueError("utc takes only booleans or None. " f"Got {utc} instead.") self.variables = _check_input_parameter_variables(variables) self.drop_original = drop_original @@ -244,8 +250,10 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: datetime_df = pd.concat( [ pd.to_datetime( - X[variable], dayfirst=self.dayfirst, - yearfirst=self.yearfirst, utc=self.utc + X[variable], + dayfirst=self.dayfirst, + yearfirst=self.yearfirst, + utc=self.utc, ) for variable in self.variables_ ], @@ -255,9 +263,9 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: non_dt_columns = datetime_df.columns[~datetime_df.apply(is_datetime)].tolist() if non_dt_columns: raise ValueError( - "ValueError: variable(s) " + - (len(non_dt_columns) * '{} ').format(*non_dt_columns) + - "could not be converted to datetime. Try setting utc=True" + "ValueError: variable(s) " + + (len(non_dt_columns) * "{} ").format(*non_dt_columns) + + "could not be converted to datetime. Try setting utc=True" ) # create new features diff --git a/feature_engine/discretisation/arbitrary.py b/feature_engine/discretisation/arbitrary.py index 02a227455..3bccff0f0 100644 --- a/feature_engine/discretisation/arbitrary.py +++ b/feature_engine/discretisation/arbitrary.py @@ -7,19 +7,35 @@ import pandas as pd from feature_engine.discretisation.base_discretiser import BaseDiscretiser +from feature_engine.docstrings import ( + Substitution, + _fit_not_learn_docstring, + _fit_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, +) from feature_engine.validation import _return_tags +@Substitution( + return_object=BaseDiscretiser._return_object_docstring, + return_boundaries=BaseDiscretiser._return_boundaries_docstring, + binner_dict_=BaseDiscretiser._binner_dict_docstring, + transform=BaseDiscretiser._transform_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit=_fit_not_learn_docstring, + fit_transform=_fit_transform_docstring, +) class ArbitraryDiscretiser(BaseDiscretiser): """ The ArbitraryDiscretiser() divides numerical variables into intervals which limits are determined by the user. Thus, it works only with numerical variables. You need to enter a dictionary with variable names as keys, and a list with - the limits of the intervals as values. For example `{'var1':[0, 10, 100, 1000], - 'var2':[5, 10, 15, 20]}`. - - The ArbitraryDiscretiser() will then sort var1 values into the intervals 0-10, + the limits of the intervals as values. For example the key could be the variable + name 'var1' and the value the following list: [0, 10, 100, 1000]. The + ArbitraryDiscretiser() will then sort var1 values into the intervals 0-10, 10-100, 100-1000, and var2 into 5-10, 10-15 and 15-20. Similar to `pandas.cut`. More details in the :ref:`User Guide `. @@ -27,18 +43,11 @@ class ArbitraryDiscretiser(BaseDiscretiser): Parameters ---------- binning_dict: dict - The dictionary with the variable to interval limits pairs. A valid dictionary - looks like this: - `binning_dict = {'var1':[0, 10, 100, 1000], 'var2':[5, 10, 15, 20]}` + The dictionary with the variable to interval limits pairs. - return_object: bool, default=False - Whether the the discrete variable should be returned as numeric or as object. - If you would like to proceed with the engineering of the variable as if - it was categorical, use True. Alternatively, keep the default to False. + {return_object} - return_boundaries: bool, default=False - Whether the output, that is the bins, should be the interval boundaries. If - True, it returns the interval boundaries. If False, it returns integers. + {return_boundaries} errors: string, default='ignore' Indicates what to do when a value is outside the limits indicated in the @@ -48,23 +57,19 @@ class ArbitraryDiscretiser(BaseDiscretiser): Attributes ---------- - binner_dict_: - Dictionary with the interval limits per variable. + {binner_dict_} - variables_: - The variables that will be discretised. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- - fit: - This transformer does not learn any parameter. - transform: - Sort variable values into the intervals. - fit_transform: - Fit to the data, then transform it. + {fit} + + {transform} + + {fit_transform} See Also -------- @@ -122,16 +127,16 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): def transform(self, X: pd.DataFrame) -> pd.DataFrame: """Sort the variable values into the intervals. - Parameters - ---------- - X: pandas dataframe of shape = [n_samples, n_features] - The dataframe to be transformed. + Parameters + ---------- + X: pandas dataframe of shape = [n_samples, n_features] + The dataframe to be transformed. - Returns - ------- - X_new: pandas dataframe of shape = [n_samples, n_features] - The transformed data with the discrete variables. - """ + Returns + ------- + X_new: pandas dataframe of shape = [n_samples, n_features] + The transformed data with the discrete variables. + """ X = super().transform(X) # check if NaN values were introduced by the discretisation procedure. diff --git a/feature_engine/discretisation/base_discretiser.py b/feature_engine/discretisation/base_discretiser.py index 4db2e0b01..07da489c1 100644 --- a/feature_engine/discretisation/base_discretiser.py +++ b/feature_engine/discretisation/base_discretiser.py @@ -7,23 +7,28 @@ class BaseDiscretiser(BaseNumericalTransformer): - """ - Shared set-up checks and methods across numerical discretisers. + """Shared set-up checks and methods across numerical discretisers.""" - Parameters - ---------- - return_object: bool, default=False + _return_object_docstring = """return_object: bool, default=False Whether the the discrete variable should be returned as numeric or as object. If you would like to proceed with the engineering of the variable as if it was categorical, use True. Alternatively, keep the default to False. + """.rstrip() - return_boundaries: bool, default=False + _return_boundaries_docstring = """return_boundaries: bool, default=False Whether the output should be the interval boundaries. If True, it returns the interval boundaries. If False, it returns integers. + """.rstrip() + + _binner_dict_docstring = """binner_dict_: + Dictionary with the interval limits per variable. + """.rstrip() + + _fit_docstring = """fit: + Find the interval limits. + """.rstrip() - Methods - ------- - transform: + _transform_docstring = """transform: Sort continuous variable values into the intervals. """ diff --git a/feature_engine/discretisation/decision_tree.py b/feature_engine/discretisation/decision_tree.py index 30a09fd0b..e8bb0ebe7 100644 --- a/feature_engine/discretisation/decision_tree.py +++ b/feature_engine/discretisation/decision_tree.py @@ -9,12 +9,25 @@ from sklearn.utils.multiclass import check_classification_targets, type_of_target from feature_engine.base_transformers import BaseNumericalTransformer +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, + _variables_numerical_docstring, +) from feature_engine.variable_manipulation import ( _check_input_parameter_variables, _find_or_check_numerical_variables, ) +@Substitution( + variables=_variables_numerical_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit_transform=_fit_transform_docstring, +) class DecisionTreeDiscretiser(BaseNumericalTransformer): """ The DecisionTreeDiscretiser() replaces numerical variables by discrete, i.e., @@ -35,9 +48,7 @@ class DecisionTreeDiscretiser(BaseNumericalTransformer): Parameters ---------- - variables: list, default=None - The list of numerical variables to transform. If None, the discretiser will - automatically select all numerical variables. + {variables} cv: int, default=3 Desired cross-validation fold to fit the decision tree. @@ -51,9 +62,8 @@ class DecisionTreeDiscretiser(BaseNumericalTransformer): param_grid: dictionary, default=None The hyperparameters for the decision tree to test with a grid search. The `param_grid` can contain any of the permitted hyperparameters for Scikit-learn's - DecisionTreeRegressor() or DecisionTreeClassifier(). - - If None, then `param_grid = {'max_depth': [1, 2, 3, 4]}`. + DecisionTreeRegressor() or DecisionTreeClassifier(). If None, then param_grid + will optimise the 'max_depth' over `[1, 2, 3, 4]`. regression: boolean, default=True Indicates whether the discretiser should train a regression or a classification @@ -73,11 +83,9 @@ class DecisionTreeDiscretiser(BaseNumericalTransformer): scores_dict_: Dictionary with the score of the best decision tree per variable. - variables_: - The variables that will be discretised. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- @@ -85,8 +93,7 @@ class DecisionTreeDiscretiser(BaseNumericalTransformer): Fit a decision tree per variable. transform: Replace continuous variable values by the predictions of the decision tree. - fit_transform: - Fit to the data, then transform it. + {fit_transform} See Also -------- diff --git a/feature_engine/discretisation/equal_frequency.py b/feature_engine/discretisation/equal_frequency.py index 331cc8700..0eb9a77d0 100644 --- a/feature_engine/discretisation/equal_frequency.py +++ b/feature_engine/discretisation/equal_frequency.py @@ -6,9 +6,27 @@ import pandas as pd from feature_engine.discretisation.base_discretiser import BaseDiscretiser +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, + _variables_numerical_docstring, +) from feature_engine.variable_manipulation import _check_input_parameter_variables +@Substitution( + return_object=BaseDiscretiser._return_object_docstring, + return_boundaries=BaseDiscretiser._return_boundaries_docstring, + binner_dict_=BaseDiscretiser._binner_dict_docstring, + fit=BaseDiscretiser._fit_docstring, + transform=BaseDiscretiser._transform_docstring, + variables=_variables_numerical_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit_transform=_fit_transform_docstring, +) class EqualFrequencyDiscretiser(BaseDiscretiser): """ The EqualFrequencyDiscretiser() divides continuous numerical variables @@ -27,41 +45,30 @@ class EqualFrequencyDiscretiser(BaseDiscretiser): Parameters ---------- - variables: list, default=None - The list of numerical variables that will be discretised. If None, the - EqualFrequencyDiscretiser() will select all numerical variables. + {variables} q: int, default=10 Desired number of equal frequency intervals / bins. - return_object: bool, default=False - Whether the the discrete variable should be returned as numeric or as - object. If you would like to proceed with the engineering of the variable as if - it was categorical, use True. Alternatively, keep the default to False. + {return_object} - return_boundaries: bool, default=False - Whether the output should be the interval boundaries. If True, it returns - the interval boundaries. If False, it returns integers. + {return_boundaries} Attributes ---------- - binner_dict_: - Dictionary with the interval limits per variable. + {binner_dict_} - variables_: - The variables that will be discretised. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- - fit: - Find the interval limits. - transform: - Sort continuous variable values into the intervals. - fit_transform: - Fit to the data, then transform it. + {fit} + + {transform} + + {fit_transform} See Also -------- diff --git a/feature_engine/discretisation/equal_width.py b/feature_engine/discretisation/equal_width.py index 6d6560e37..91deebf76 100644 --- a/feature_engine/discretisation/equal_width.py +++ b/feature_engine/discretisation/equal_width.py @@ -6,9 +6,27 @@ import pandas as pd from feature_engine.discretisation.base_discretiser import BaseDiscretiser +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, + _variables_numerical_docstring, +) from feature_engine.variable_manipulation import _check_input_parameter_variables +@Substitution( + return_object=BaseDiscretiser._return_object_docstring, + return_boundaries=BaseDiscretiser._return_boundaries_docstring, + binner_dict_=BaseDiscretiser._binner_dict_docstring, + fit=BaseDiscretiser._fit_docstring, + transform=BaseDiscretiser._transform_docstring, + variables=_variables_numerical_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit_transform=_fit_transform_docstring, +) class EqualWidthDiscretiser(BaseDiscretiser): """ The EqualWidthDiscretiser() divides continuous numerical variables into @@ -35,41 +53,30 @@ class EqualWidthDiscretiser(BaseDiscretiser): Parameters ---------- - variables: list, default=None - The list of numerical variables to transform. If None, the - discretiser will automatically select all numerical type variables. + {variables} bins: int, default=10 Desired number of equal width intervals / bins. - return_object: bool, default=False - Whether the the discrete variable should be returned as numeric or as - object. If you would like to proceed with the engineering of the variable as if - it was categorical, use True. Alternatively, keep the default to False. + {return_object} - return_boundaries : bool, default=False - Whether the output should be the interval boundaries. If True, it returns - the interval boundaries. If False, it returns integers. + {return_boundaries} Attributes ---------- - binner_dict_: - Dictionary with the interval limits per variable. + {binner_dict_} - variables_: - The variables that will be discretised. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- - fit: - Find the interval limits. - transform: - Sort continuous variable values into the intervals. - fit_transform: - Fit to the data, then transform it. + {fit} + + {transform} + + {fit_transform} See Also -------- diff --git a/feature_engine/docstrings.py b/feature_engine/docstrings.py new file mode 100644 index 000000000..ee84534b1 --- /dev/null +++ b/feature_engine/docstrings.py @@ -0,0 +1,65 @@ +"""Utilities for docstring in Feature-engine. + +Taken from the project imbalanced-learn: + +https://github.com/scikit-learn-contrib/imbalanced-learn/blob/ +imblearn/utils/_docstring.py#L7 +""" + + +class Substitution: + """Decorate a function's or a class' docstring to perform string + substitution on it. + This decorator should be robust even if obj.__doc__ is None + (for example, if -OO was passed to the interpreter). + """ + + def __init__(self, *args, **kwargs): + if args and kwargs: + raise AssertionError("Only positional or keyword args are allowed") + + self.params = args or kwargs + + def __call__(self, obj): + obj.__doc__ = obj.__doc__.format(**self.params) + return obj + + +# input parameters +_variables_numerical_docstring = """variables: list, default=None + The list of numerical variables to transform. If None, the transformer will + automatically find and select all numerical variables. + """.rstrip() + +_drop_original_docstring = """drop_original: bool, default=False + If True, the original variables to transform will be dropped from the dataframe. + """.rstrip() + +_missing_values_docstring = """missing_values: string, default='raise' + Indicates if missing values should be ignored or raised. If 'raise' the + transformer will return an error if the the datasets to `fit` or `transform` + contain missing values. If 'ignore', missing data will be ignored when learning + parameters or performing the transformation. + """ + +# Attributes +_variables_attribute_docstring = """variables_: + The group of variables that will be transformed. + """.rstrip() + +_n_features_in_docstring = """n_features_in_: + The number of features in the train set used in fit. + """.rstrip() + +# Methods +_fit_not_learn_docstring = """fit: + This transformer does not learn parameters. + """.rstrip() + +_fit_transform_docstring = """fit_transform: + Fit to data, then transform it. + """.rstrip() + +_inverse_transform_docstring = """inverse_transform: + Convert the data back to the original representation. + """.rstrip() diff --git a/feature_engine/encoding/_docstrings.py b/feature_engine/encoding/_docstrings.py new file mode 100644 index 000000000..af305e4d2 --- /dev/null +++ b/feature_engine/encoding/_docstrings.py @@ -0,0 +1,30 @@ +_variables_numerical_docstring = """variables: list, default=None + The list of numerical variables to transform. If None, the transformer will + automatically find and select all numerical variables. + """.rstrip() + +_variables_docstring = """variables: list, default=None + The list of categorical variables that will be encoded. If None, the + encoder will find and transform all variables of type object or categorical by + default. You can also make the transformer accept numerical variables, see the + next parameter. + """.rstrip() + +_ignore_format_docstring = """ignore_format: bool, default=False + Whether the format in which the categorical variables are cast should be + ignored. If False, the encoder will automatically select variables of type + object or categorical, or check that the variables entered by the user are of + type object or categorical. If True, the encoder will select all variables or + accept all variables entered by the user, including those cast as numeric. + """.rstrip() + +_errors_docstring = """errors: string, default='ignore' + Indicates what to do, when categories not present in the train set are + encountered during transform. If 'raise', then rare categories will raise an + error. If 'ignore', then rare categories will be set as NaN and a warning will + be raised instead. + """.rstrip() + +_transform_docstring = """transform: + Encode the categories to numbers. + """.rstrip() diff --git a/feature_engine/encoding/base_encoder.py b/feature_engine/encoding/base_encoder.py index b117fc75d..c01eda11a 100644 --- a/feature_engine/encoding/base_encoder.py +++ b/feature_engine/encoding/base_encoder.py @@ -10,31 +10,32 @@ _check_input_matches_training_df, _is_dataframe, ) +from feature_engine.docstrings import Substitution +from feature_engine.encoding._docstrings import ( + _errors_docstring, + _ignore_format_docstring, + _variables_docstring, +) from feature_engine.validation import _return_tags from feature_engine.variable_manipulation import ( + _check_input_parameter_variables, _find_all_variables, _find_or_check_categorical_variables, - _check_input_parameter_variables, ) +@Substitution( + ignore_format=_ignore_format_docstring, + variables=_variables_docstring, +) class BaseCategoricalTransformer(BaseEstimator, TransformerMixin): """shared set-up checks and methods across categorical transformers Parameters ---------- - variables: list, default=None - The list of categorical variables that will be encoded. If None, the - encoder will find and transform all variables of type object or categorical by - default. You can also make the transformer accept numerical variables, see the - next parameter. - - ignore_format: bool, default=False - Whether the format in which the categorical variables are cast should be - ignored. If False, the encoder will automatically select variables of type - object or categorical, or check that the variables entered by the user are of - type object or categorical. If True, the encoder will select all variables or - accept all variables entered by the user, including those cast as numeric. + {variables}. + + {ignore_format} """ def __init__( @@ -235,6 +236,11 @@ def _more_tags(self): return tags_dict +@Substitution( + ignore_format=_ignore_format_docstring, + variables=_variables_docstring, + errors=_errors_docstring, +) class BaseCategorical(BaseCategoricalTransformer): """ BaseCategorical() is the parent class to some of the encoders. @@ -242,24 +248,11 @@ class BaseCategorical(BaseCategoricalTransformer): Parameters ---------- - variables: list, default=None - The list of categorical variables that will be encoded. If None, the - encoder will find and transform all variables of type object or categorical by - default. You can also make the transformer accept numerical variables, see the - next parameter. - - ignore_format: bool, default=False - Whether the format in which the categorical variables are cast should be - ignored. If False, the encoder will automatically select variables of type - object or categorical, or check that the variables entered by the user are of - type object or categorical. If True, the encoder will select all variables or - accept all variables entered by the user, including those cast as numeric. - - errors: string, default='ignore' - Indicates what to do, when categories not present in the train set are - encountered during transform. If 'raise', then rare categories will raise an - error. If 'ignore', then rare categories will be set as NaN and a warning will - be raised instead. + {variables} + + {ignore_format} + + {errors} """ def __init__( diff --git a/feature_engine/encoding/count_frequency.py b/feature_engine/encoding/count_frequency.py index e1f4cc96c..d09a48e70 100644 --- a/feature_engine/encoding/count_frequency.py +++ b/feature_engine/encoding/count_frequency.py @@ -5,9 +5,32 @@ import pandas as pd +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _inverse_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, +) +from feature_engine.encoding._docstrings import ( + _errors_docstring, + _ignore_format_docstring, + _transform_docstring, + _variables_docstring, +) from feature_engine.encoding.base_encoder import BaseCategorical +@Substitution( + ignore_format=_ignore_format_docstring, + variables=_variables_docstring, + errors=_errors_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit_transform=_fit_transform_docstring, + transform=_transform_docstring, + inverse_transform=_inverse_transform_docstring, +) class CountFrequencyEncoder(BaseCategorical): """ The CountFrequencyEncoder() replaces categories by either the count or the @@ -41,46 +64,31 @@ class CountFrequencyEncoder(BaseCategorical): **'frequency'**: percentage of observations per category - variables: list, default=None - The list of categorical variables that will be encoded. If None, the - encoder will find and transform all variables of type object or categorical by - default. You can also make the transformer accept numerical variables, see the - next parameter. - - ignore_format: bool, default=False - Whether the format in which the categorical variables are cast should be - ignored. If False, the encoder will automatically select variables of type - object or categorical, or check that the variables entered by the user are of - type object or categorical. If True, the encoder will select all variables or - accept all variables entered by the user, including those cast as numeric. - - errors: string, default='ignore' - Indicates what to do when categories not present in the train set are - encountered during transform. If 'raise', then rare categories will raise an - error. If 'ignore', then rare categories will be set as NaN and a warning will - be raised instead. + {variables} + + {ignore_format} + + {errors} Attributes ---------- encoder_dict_: Dictionary with the count or frequency per category, per variable. - variables_: - The group of variables that will be transformed. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- fit: Learn the count or frequency per category, per variable. - transform: - Encode the categories to numbers. - fit_transform: - Fit to the data, then transform it. - inverse_transform: - Encode the numbers into the original categories. + + {transform} + + {fit_transform} + + {inverse_transform} Notes ----- @@ -102,7 +110,7 @@ def __init__( encoding_method: str = "count", variables: Union[None, int, str, List[Union[str, int]]] = None, ignore_format: bool = False, - errors: str = "ignore" + errors: str = "ignore", ) -> None: if encoding_method not in ["count", "frequency"]: diff --git a/feature_engine/encoding/decision_tree.py b/feature_engine/encoding/decision_tree.py index 0d69198ce..5be6a70c9 100644 --- a/feature_engine/encoding/decision_tree.py +++ b/feature_engine/encoding/decision_tree.py @@ -8,10 +8,27 @@ from sklearn.utils.multiclass import check_classification_targets, type_of_target from feature_engine.discretisation import DecisionTreeDiscretiser +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, +) +from feature_engine.encoding._docstrings import ( + _ignore_format_docstring, + _variables_docstring, +) from feature_engine.encoding.base_encoder import BaseCategoricalTransformer from feature_engine.encoding.ordinal import OrdinalEncoder +@Substitution( + ignore_format=_ignore_format_docstring, + variables=_variables_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit_transform=_fit_transform_docstring, +) class DecisionTreeEncoder(BaseCategoricalTransformer): """ The DecisionTreeEncoder() encodes categorical variables with predictions @@ -55,8 +72,8 @@ class DecisionTreeEncoder(BaseCategoricalTransformer): param_grid: dictionary, default=None The hyperparameters for the decision tree to test with a grid search. The `param_grid` can contain any of the permitted hyperparameters for Scikit-learn's - DecisionTreeRegressor() or DecisionTreeClassifier(). If None, then - `param_grid = {'max_depth': [1, 2, 3, 4]}`. + DecisionTreeRegressor() or DecisionTreeClassifier(). If None, then param_grid + will optimise the 'max_depth' over `[1, 2, 3, 4]`. regression: boolean, default=True Indicates whether the encoder should train a regression or a classification @@ -68,38 +85,28 @@ class DecisionTreeEncoder(BaseCategoricalTransformer): DecisionTreeClassifier(). For reproducibility it is recommended to set the random_state to an integer. - variables: list, default=None - The list of categorical variables that will be encoded. If None, the - encoder will find and transform all variables of type object or categorical by - default. You can also make the transformer accept numerical variables, see the - next parameter. + {variables} - ignore_format: bool, default=False - Whether the format in which the categorical variables are cast should be - ignored. If False, the encoder will automatically select variables of type - object or categorical, or check that the variables entered by the user are of - type object or categorical. If True, the encoder will select all variables or - accept all variables entered by the user, including those cast as numeric. + {ignore_format} Attributes ---------- encoder_: sklearn Pipeline containing the ordinal encoder and the decision tree. - variables_: - The group of variables that will be transformed. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- fit: Fit a decision tree per variable. + transform: Replace categorical variable by the predictions of the decision tree. - fit_transform: - Fit to the data, then transform it. + + {fit_transform} Notes ----- diff --git a/feature_engine/encoding/mean_encoding.py b/feature_engine/encoding/mean_encoding.py index 83ef8c685..a25c84d17 100644 --- a/feature_engine/encoding/mean_encoding.py +++ b/feature_engine/encoding/mean_encoding.py @@ -5,9 +5,32 @@ import pandas as pd +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _inverse_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, +) +from feature_engine.encoding._docstrings import ( + _errors_docstring, + _ignore_format_docstring, + _transform_docstring, + _variables_docstring, +) from feature_engine.encoding.base_encoder import BaseCategorical +@Substitution( + ignore_format=_ignore_format_docstring, + variables=_variables_docstring, + errors=_errors_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit_transform=_fit_transform_docstring, + transform=_transform_docstring, + inverse_transform=_inverse_transform_docstring, +) class MeanEncoder(BaseCategorical): """ The MeanEncoder() replaces categories by the mean value of the target for each @@ -33,46 +56,31 @@ class MeanEncoder(BaseCategorical): Parameters ---------- - variables: list, default=None - The list of categorical variables that will be encoded. If None, the - encoder will find and transform all variables of type object or categorical by - default. You can also make the transformer accept numerical variables, see the - next parameter. - - ignore_format: bool, default=False - Whether the format in which the categorical variables are cast should be - ignored. If False, the encoder will automatically select variables of type - object or categorical, or check that the variables entered by the user are of - type object or categorical. If True, the encoder will select all variables or - accept all variables entered by the user, including those cast as numeric. - - errors: string, default='ignore' - Indicates what to do when categories not present in the train set are - encountered during transform. If 'raise', then rare categories will raise an - error. If 'ignore', then rare categories will be set as NaN and a warning will - be raised instead. + {variables} + + {ignore_format} + + {errors} Attributes ---------- encoder_dict_: Dictionary with the target mean value per category per variable. - variables_: - The group of variables that will be transformed. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- fit: Learn the target mean value per category, per variable. - transform: - Encode the categories to numbers. - fit_transform: - Fit to the data, then transform it. - inverse_transform: - Encode the numbers into the original categories. + + {transform} + + {fit_transform} + + {inverse_transform} Notes ----- @@ -100,7 +108,7 @@ def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, ignore_format: bool = False, - errors: str = "ignore" + errors: str = "ignore", ) -> None: super().__init__(variables, ignore_format, errors) diff --git a/feature_engine/encoding/one_hot.py b/feature_engine/encoding/one_hot.py index 6db7f34c8..b7403d242 100644 --- a/feature_engine/encoding/one_hot.py +++ b/feature_engine/encoding/one_hot.py @@ -6,9 +6,26 @@ import numpy as np import pandas as pd +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, +) +from feature_engine.encoding._docstrings import ( + _ignore_format_docstring, + _variables_docstring, +) from feature_engine.encoding.base_encoder import BaseCategoricalTransformer +@Substitution( + ignore_format=_ignore_format_docstring, + variables=_variables_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit_transform=_fit_transform_docstring, +) class OneHotEncoder(BaseCategoricalTransformer): """ The OneHotEncoder() replaces categorical variables by a set of binary variables @@ -71,42 +88,32 @@ class OneHotEncoder(BaseCategoricalTransformer): to `True`, will ensure that for every binary variable in the dataset, only 1 dummy is created. - variables: list, default=None - The list of categorical variables that will be encoded. If None, the - encoder will find and transform all variables of type object or categorical by - default. You can also make the transformer accept numerical variables, see the - next parameter. + {variables} - ignore_format: bool, default=False - Whether the format in which the categorical variables are cast should be - ignored. If False, the encoder will automatically select variables of type - object or categorical, or check that the variables entered by the user are of - type object or categorical. If True, the encoder will select all variables or - accept all variables entered by the user, including those cast as numeric. + {ignore_format} Attributes ---------- encoder_dict_: Dictionary with the categories for which dummy variables will be created. - variables_: - The group of variables that will be transformed. + {variables_} variables_binary_: List with binary variables identified in the data. That is, variables with only 2 categories. - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- fit: Learn the unique categories per variable + transform: Replace the categorical variables by the binary variables. - fit_transform: - Fit to the data, then transform it. + + {fit_transform} Notes ----- diff --git a/feature_engine/encoding/ordinal.py b/feature_engine/encoding/ordinal.py index 29ff95d7c..c8b827017 100644 --- a/feature_engine/encoding/ordinal.py +++ b/feature_engine/encoding/ordinal.py @@ -5,12 +5,35 @@ import pandas as pd +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _inverse_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, +) +from feature_engine.encoding._docstrings import ( + _errors_docstring, + _ignore_format_docstring, + _transform_docstring, + _variables_docstring, +) from feature_engine.encoding.base_encoder import BaseCategorical +@Substitution( + ignore_format=_ignore_format_docstring, + variables=_variables_docstring, + errors=_errors_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit_transform=_fit_transform_docstring, + transform=_transform_docstring, + inverse_transform=_inverse_transform_docstring, +) class OrdinalEncoder(BaseCategorical): """ - The OrdinalCategoricalEncoder() replaces categories by ordinal numbers + The OrdinalEncoder() replaces categories by ordinal numbers (0, 1, 2, 3, etc). The numbers can be ordered based on the mean of the target per category, or assigned arbitrarily. @@ -38,46 +61,31 @@ class OrdinalEncoder(BaseCategorical): **'arbitrary'**: categories are numbered arbitrarily. - variables: list, default=None - The list of categorical variables that will be encoded. If None, the - encoder will find and transform all variables of type object or categorical by - default. You can also make the transformer accept numerical variables, see the - next parameter. - - ignore_format: bool, default=False - Whether the format in which the categorical variables are cast should be - ignored. If False, the encoder will automatically select variables of type - object or categorical, or check that the variables entered by the user are of - type object or categorical. If True, the encoder will select all variables or - accept all variables entered by the user, including those cast as numeric. - - errors: string, default='ignore' - Indicates what to do when categories not present in the train set are - encountered during transform. If 'raise', then rare categories will raise an - error. If 'ignore', then rare categories will be set as NaN and a warning will - be raised instead. + {variables} + + {ignore_format} + + {errors} Attributes ---------- encoder_dict_: Dictionary with the ordinal number per category, per variable. - variables_: - The group of variables that will be transformed. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- fit: Find the integer to replace each category in each variable. - transform: - Encode the categories to numbers. - fit_transform: - Fit to the data, then transform it. - inverse_transform: - Encode the numbers into the original categories. + + {transform} + + {fit_transform} + + {inverse_transform} Notes ----- @@ -107,7 +115,7 @@ def __init__( encoding_method: str = "ordered", variables: Union[None, int, str, List[Union[str, int]]] = None, ignore_format: bool = False, - errors: str = "ignore" + errors: str = "ignore", ) -> None: if encoding_method not in ["ordered", "arbitrary"]: diff --git a/feature_engine/encoding/probability_ratio.py b/feature_engine/encoding/probability_ratio.py index 24483e86e..50230249f 100644 --- a/feature_engine/encoding/probability_ratio.py +++ b/feature_engine/encoding/probability_ratio.py @@ -6,10 +6,33 @@ import numpy as np import pandas as pd +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _inverse_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, +) +from feature_engine.encoding._docstrings import ( + _errors_docstring, + _ignore_format_docstring, + _transform_docstring, + _variables_docstring, +) from feature_engine.encoding.base_encoder import BaseCategorical from feature_engine.validation import _return_tags +@Substitution( + ignore_format=_ignore_format_docstring, + variables=_variables_docstring, + errors=_errors_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit_transform=_fit_transform_docstring, + transform=_transform_docstring, + inverse_transform=_inverse_transform_docstring, +) class PRatioEncoder(BaseCategorical): """ The PRatioEncoder() replaces categories by the ratio of the probability of the @@ -58,46 +81,31 @@ class PRatioEncoder(BaseCategorical): **'log_ratio'**: log probability ratio - variables: list, default=None - The list of categorical variables that will be encoded. If None, the - encoder will find and transform all variables of type object or categorical by - default. You can also make the transformer accept numerical variables, see the - next parameter. - - ignore_format: bool, default=False - Whether the format in which the categorical variables are cast should be - ignored. If False, the encoder will automatically select variables of type - object or categorical, or check that the variables entered by the user are of - type object or categorical. If True, the encoder will select all variables or - accept all variables entered by the user, including those cast as numeric. - - errors: string, default='ignore' - Indicates what to do when categories not present in the train set are - encountered during transform. If 'raise', then rare categories will raise an - error. If 'ignore', then rare categories will be set as NaN and a warning will - be raised instead. + {variables} + + {ignore_format} + + {errors} Attributes ---------- encoder_dict_: Dictionary with the probability ratio per category per variable. - variables_: - The group of variables that will be transformed. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- fit: Learn probability ratio per category, per variable. - transform: - Encode categories into numbers. - fit_transform: - Fit to the data, then transform it. - inverse_transform: - Encode the numbers into the original categories. + + {transform} + + {fit_transform} + + {inverse_transform} Notes ----- @@ -115,7 +123,7 @@ def __init__( encoding_method: str = "ratio", variables: Union[None, int, str, List[Union[str, int]]] = None, ignore_format: bool = False, - errors: str = "ignore" + errors: str = "ignore", ) -> None: if encoding_method not in ["ratio", "log_ratio"]: diff --git a/feature_engine/encoding/rare_label.py b/feature_engine/encoding/rare_label.py index 22bbceda2..825c6e9b3 100644 --- a/feature_engine/encoding/rare_label.py +++ b/feature_engine/encoding/rare_label.py @@ -7,12 +7,29 @@ import numpy as np import pandas as pd +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, +) +from feature_engine.encoding._docstrings import ( + _ignore_format_docstring, + _variables_docstring, +) from feature_engine.encoding.base_encoder import BaseCategoricalTransformer +@Substitution( + ignore_format=_ignore_format_docstring, + variables=_variables_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit_transform=_fit_transform_docstring, +) class RareLabelEncoder(BaseCategoricalTransformer): """ - The RareLabelCategoricalEncoder() groups rare or infrequent categories in + The RareLabelEncoder() groups rare or infrequent categories in a new category called "Rare", or any other name entered by the user. For example in the variable colour, if the percentage of observations @@ -61,18 +78,9 @@ class RareLabelEncoder(BaseCategoricalTransformer): replace_with: string, intege or float, default='Rare' The value that will be used to replace infrequent categories. - variables: list, default=None - The list of categorical variables that will be encoded. If None, the - encoder will find and transform all variables of type object or categorical by - default. You can also make the transformer accept numerical variables, see the - next parameter. + {variables} - ignore_format: bool, default=False - Whether the format in which the categorical variables are cast should be - ignored. If False, the encoder will automatically select variables of type - object or categorical, or check that the variables entered by the user are of - type object or categorical. If True, the encoder will select all variables or - accept all variables entered by the user, including those cast as numeric. + {ignore_format} Attributes ---------- @@ -80,20 +88,19 @@ class RareLabelEncoder(BaseCategoricalTransformer): Dictionary with the frequent categories, i.e., those that will be kept, per variable. - variables_: - The variables that will be transformed. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- fit: Find frequent categories. + transform: Group rare categories - fit_transform: - Fit to data, then transform it. + + {fit_transform} """ def __init__( diff --git a/feature_engine/encoding/woe.py b/feature_engine/encoding/woe.py index 1d319abb4..2363a22e2 100644 --- a/feature_engine/encoding/woe.py +++ b/feature_engine/encoding/woe.py @@ -6,10 +6,33 @@ import numpy as np import pandas as pd +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _inverse_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, +) +from feature_engine.encoding._docstrings import ( + _errors_docstring, + _ignore_format_docstring, + _transform_docstring, + _variables_docstring, +) from feature_engine.encoding.base_encoder import BaseCategorical from feature_engine.validation import _return_tags +@Substitution( + ignore_format=_ignore_format_docstring, + variables=_variables_docstring, + errors=_errors_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit_transform=_fit_transform_docstring, + transform=_transform_docstring, + inverse_transform=_inverse_transform_docstring, +) class WoEEncoder(BaseCategorical): """ The WoEEncoder() replaces categories by the weight of evidence @@ -41,46 +64,31 @@ class WoEEncoder(BaseCategorical): Parameters ---------- - variables: list, default=None - The list of categorical variables that will be encoded. If None, the - encoder will find and transform all variables of type object or categorical by - default. You can also make the transformer accept numerical variables, see the - next parameter. - - ignore_format: bool, default=False - Whether the format in which the categorical variables are cast should be - ignored. If False, the encoder will automatically select variables of type - object or categorical, or check that the variables entered by the user are of - type object or categorical. If True, the encoder will select all variables or - accept all variables entered by the user, including those cast as numeric. - - errors: string, default='ignore' - Indicates what to do when categories not present in the train set are - encountered during transform. If 'raise', then rare categories will raise an - error. If 'ignore', then rare categories will be set as NaN and a warning will - be raised instead. + {variables} + + {ignore_format} + + {errors} Attributes ---------- encoder_dict_: Dictionary with the WoE per variable. - variables_: - The group of variables that will be transformed. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- fit: Learn the WoE per category, per variable. - transform: - Encode the categories to numbers. - fit_transform: - Fit to the data, then transform it. - inverse_transform: - Encode the numbers into the original categories. + + {transform} + + {fit_transform} + + {inverse_transform} Notes ----- @@ -105,7 +113,7 @@ def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, ignore_format: bool = False, - errors: str = "ignore" + errors: str = "ignore", ) -> None: super().__init__(variables, ignore_format, errors) diff --git a/feature_engine/imputation/arbitrary_number.py b/feature_engine/imputation/arbitrary_number.py index fc6a5cab4..fbd5ed396 100644 --- a/feature_engine/imputation/arbitrary_number.py +++ b/feature_engine/imputation/arbitrary_number.py @@ -6,6 +6,13 @@ import pandas as pd from feature_engine.dataframe_checks import _is_dataframe +from feature_engine.docstrings import ( + Substitution, + _fit_not_learn_docstring, + _fit_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, +) from feature_engine.imputation.base_imputer import BaseImputer from feature_engine.parameter_checks import _define_numerical_dict from feature_engine.variable_manipulation import ( @@ -14,6 +21,14 @@ ) +@Substitution( + imputer_dict_=BaseImputer._imputer_dict_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit=_fit_not_learn_docstring, + transform=BaseImputer._transform_docstring, + fit_transform=_fit_transform_docstring, +) class ArbitraryNumberImputer(BaseImputer): """ The ArbitraryNumberImputer() replaces missing data by an arbitrary @@ -41,25 +56,23 @@ class ArbitraryNumberImputer(BaseImputer): The dictionary of variables and the arbitrary numbers for their imputation. If specified, it overrides the above parameters. + Attributes ---------- - imputer_dict_: - Dictionary with the values to replace NAs in each variable. - variables_: - The group of variables that will be transformed. + {imputer_dict_} - n_features_in_: - The number of features in the train set used in fit. + {variables_} + + {n_features_in_} Methods ------- - fit: - This transformer does not learn parameters. - transform: - Impute missing data. - fit_transform: - Fit to the data, then transform it. + {fit} + + {transform} + + {fit_transform} See Also -------- diff --git a/feature_engine/imputation/base_imputer.py b/feature_engine/imputation/base_imputer.py index 598ec085a..306036d09 100644 --- a/feature_engine/imputation/base_imputer.py +++ b/feature_engine/imputation/base_imputer.py @@ -12,6 +12,19 @@ class BaseImputer(BaseEstimator, TransformerMixin): """shared set-up checks and methods across imputers""" + _variables_numerical_docstring = """variables: list, default=None + The list of variables to impute. If None, the imputer will select + all numerical variables. + """.rstrip() + + _imputer_dict_docstring = """imputer_dict_: + Dictionary with the values to replace missing data in each variable. + """.rstrip() + + _transform_docstring = """transform: + Impute missing data. + """.rstrip() + def _check_transform_input_and_state(self, X: pd.DataFrame) -> pd.DataFrame: """ Check that the input is a dataframe and of the same size than the one used diff --git a/feature_engine/imputation/categorical.py b/feature_engine/imputation/categorical.py index 665fe1ff1..555f3d67f 100644 --- a/feature_engine/imputation/categorical.py +++ b/feature_engine/imputation/categorical.py @@ -6,6 +6,12 @@ import pandas as pd from feature_engine.dataframe_checks import _is_dataframe +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, +) from feature_engine.imputation.base_imputer import BaseImputer from feature_engine.variable_manipulation import ( _check_input_parameter_variables, @@ -14,19 +20,26 @@ ) +@Substitution( + imputer_dict_=BaseImputer._imputer_dict_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + transform=BaseImputer._transform_docstring, + fit_transform=_fit_transform_docstring, +) class CategoricalImputer(BaseImputer): """ The CategoricalImputer() replaces missing data in categorical variables by an arbitrary value or by the most frequent category. - The CategoricalVariableImputer() imputes by default only categorical variables + The CategoricalImputer() imputes by default only categorical variables (type 'object' or 'categorical'). You can pass a list of variables to impute, or alternatively, the encoder will find and impute all categorical variables. If you want to impute numerical variables with this transformer, there are 2 ways of doing it: - **Option 1**: Cast your numerical variables as object in the input dataframe, before + **Option 1**: Cast your numerical variables as object in the input dataframe before passing it to the transformer. **Option 2**: Set `ignore_format=True`. Note that if you do this and do not pass the @@ -42,8 +55,8 @@ class CategoricalImputer(BaseImputer): or 'missing' to impute with an arbitrary value. fill_value: str, int, float, default='Missing' - Only used when `imputation_method='missing'`. User-defined value to replace the - missing data. + User-defined value to replace missing data. Only used when + `imputation_method='missing'`. variables: list, default=None The list of categorical variables that will be imputed. If None, the @@ -59,30 +72,28 @@ class CategoricalImputer(BaseImputer): ignore_format: bool, default=False Whether the format in which the categorical variables are cast should be - ignored. If false, the encoder will automatically select variables of type + ignored. If false, the imputer will automatically select variables of type object or categorical, or check that the variables entered by the user are of - type object or categorical. If True, the encoder will select all variables or + type object or categorical. If True, the imputer will select all variables or accept all variables entered by the user, including those cast as numeric. Attributes ---------- - imputer_dict_: - Dictionary with most frequent category or arbitrary value per variable. + {imputer_dict_} - variables_: - The group of variables that will be transformed. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- fit: Learn the most frequent category or assign arbitrary value to variable. - transform: - Impute missing data. - fit_transform: - Fit to the data, than transform it. + + {transform} + + {fit_transform} + """ def __init__( diff --git a/feature_engine/imputation/drop_missing_data.py b/feature_engine/imputation/drop_missing_data.py index 787a2bfcc..3dd559fed 100644 --- a/feature_engine/imputation/drop_missing_data.py +++ b/feature_engine/imputation/drop_missing_data.py @@ -6,10 +6,19 @@ import pandas as pd from feature_engine.dataframe_checks import _is_dataframe +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _n_features_in_docstring, +) from feature_engine.imputation.base_imputer import BaseImputer from feature_engine.variable_manipulation import _check_input_parameter_variables +@Substitution( + n_features_in_=_n_features_in_docstring, + fit_transform=_fit_transform_docstring, +) class DropMissingData(BaseImputer): """ DropMissingData() will delete rows containing missing values. It provides @@ -53,8 +62,7 @@ class DropMissingData(BaseImputer): when the latter is `None`, or when only a subset of the indicated variables show NA in the train set if `missing_only=True`. - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- @@ -62,8 +70,9 @@ class DropMissingData(BaseImputer): Find the variables for which missing data should be evaluated. transform: Remove rows with missing data. - fit_transform: - Fit to the data, then transform it. + + {fit_transform} + return_na_data: Returns a dataframe with the rows that contain missing data. """ diff --git a/feature_engine/imputation/end_tail.py b/feature_engine/imputation/end_tail.py index 80c9f2a85..3995c072b 100644 --- a/feature_engine/imputation/end_tail.py +++ b/feature_engine/imputation/end_tail.py @@ -6,6 +6,12 @@ import pandas as pd from feature_engine.dataframe_checks import _is_dataframe +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, +) from feature_engine.imputation.base_imputer import BaseImputer from feature_engine.variable_manipulation import ( _check_input_parameter_variables, @@ -13,6 +19,14 @@ ) +@Substitution( + variables=BaseImputer._variables_numerical_docstring, + imputer_dict_=BaseImputer._imputer_dict_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + transform=BaseImputer._transform_docstring, + fit_transform=_fit_transform_docstring, +) class EndTailImputer(BaseImputer): """ The EndTailImputer() replaces missing data by a value at either tail of the @@ -71,29 +85,25 @@ class EndTailImputer(BaseImputer): Factor to multiply the std, the IQR or the Max values. Recommended values are 2 or 3 for Gaussian, or 1.5 or 3 for IQR. - variables: list, default=None - The list of variables to impute. If None, the imputer will select - all numerical variables. + {variables} Attributes ---------- - imputer_dict_: - Dictionary with the values at the end of the distribution per variable. + {imputer_dict_} - variables_: - The group of variables that will be transformed. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- fit: Learn values to replace missing data. - transform: - Impute missing data. - fit_transform: - Fit to the data, then transform it. + + {transform} + + {fit_transform} + """ def __init__( diff --git a/feature_engine/imputation/mean_median.py b/feature_engine/imputation/mean_median.py index f5d9c62a6..212d760d5 100644 --- a/feature_engine/imputation/mean_median.py +++ b/feature_engine/imputation/mean_median.py @@ -6,6 +6,12 @@ import pandas as pd from feature_engine.dataframe_checks import _is_dataframe +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, +) from feature_engine.imputation.base_imputer import BaseImputer from feature_engine.variable_manipulation import ( _check_input_parameter_variables, @@ -13,6 +19,14 @@ ) +@Substitution( + variables=BaseImputer._variables_numerical_docstring, + imputer_dict_=BaseImputer._imputer_dict_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + transform=BaseImputer._transform_docstring, + fit_transform=_fit_transform_docstring, +) class MeanMedianImputer(BaseImputer): """ The MeanMedianImputer() replaces missing data by the mean or median value of the @@ -29,29 +43,25 @@ class MeanMedianImputer(BaseImputer): imputation_method: str, default='median' Desired method of imputation. Can take 'mean' or 'median'. - variables: list, default=None - The list of variables to impute. If None, the imputer will impute - all numerical variables. + {variables} Attributes ---------- - imputer_dict_: - Dictionary with the mean or median values per variable. + {imputer_dict_} - variables_: - The group of variables that will be transformed. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- fit: Learn the mean or median values. - transform: - Impute missing data. - fit_transform: - Fit to the data, then transform it. + + {transform} + + {fit_transform} + """ def __init__( diff --git a/feature_engine/imputation/missing_indicator.py b/feature_engine/imputation/missing_indicator.py index e8d78a535..9c75edec2 100644 --- a/feature_engine/imputation/missing_indicator.py +++ b/feature_engine/imputation/missing_indicator.py @@ -7,10 +7,19 @@ import pandas as pd from feature_engine.dataframe_checks import _is_dataframe +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _n_features_in_docstring, +) from feature_engine.imputation.base_imputer import BaseImputer from feature_engine.variable_manipulation import _check_input_parameter_variables +@Substitution( + n_features_in_=_n_features_in_docstring, + fit_transform=_fit_transform_docstring, +) class AddMissingIndicator(BaseImputer): """ The AddMissingIndicator() adds binary variables that indicate if data is @@ -50,17 +59,18 @@ class AddMissingIndicator(BaseImputer): variables_: List of variables for which the missing indicators will be created. - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- fit: Find the variables for which the missing indicators will be created + transform: Add the missing indicators. - fit_transform: - Fit to the data, then transform it. + + {fit_transform} + """ def __init__( diff --git a/feature_engine/imputation/random_sample.py b/feature_engine/imputation/random_sample.py index 292c0121c..b4616ca11 100644 --- a/feature_engine/imputation/random_sample.py +++ b/feature_engine/imputation/random_sample.py @@ -7,6 +7,12 @@ import pandas as pd from feature_engine.dataframe_checks import _is_dataframe +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, +) from feature_engine.imputation.base_imputer import BaseImputer from feature_engine.variable_manipulation import _check_input_parameter_variables @@ -27,6 +33,12 @@ def _define_seed( return internal_seed +@Substitution( + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + transform=BaseImputer._transform_docstring, + fit_transform=_fit_transform_docstring, +) class RandomSampleImputer(BaseImputer): """ The RandomSampleImputer() replaces missing data with a random sample extracted from @@ -75,20 +87,19 @@ class RandomSampleImputer(BaseImputer): X_: Copy of the training dataframe from which to extract the random samples. - variables_: - The group of variables that will be transformed. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- fit: Make a copy of the train set - transform: - Impute missing data. - fit_transform: - Fit to the data, then transform it. + + {transform} + + {fit_transform} + """ def __init__( diff --git a/feature_engine/outliers/__init__.py b/feature_engine/outliers/__init__.py index 8d740ff89..312778348 100644 --- a/feature_engine/outliers/__init__.py +++ b/feature_engine/outliers/__init__.py @@ -3,7 +3,7 @@ """ from .artbitrary import ArbitraryOutlierCapper -from .winsorizer import Winsorizer from .trimmer import OutlierTrimmer +from .winsorizer import Winsorizer __all__ = ["Winsorizer", "ArbitraryOutlierCapper", "OutlierTrimmer"] diff --git a/feature_engine/outliers/artbitrary.py b/feature_engine/outliers/artbitrary.py index afa84c114..c5494efea 100644 --- a/feature_engine/outliers/artbitrary.py +++ b/feature_engine/outliers/artbitrary.py @@ -11,19 +11,37 @@ _check_contains_na, _is_dataframe, ) +from feature_engine.docstrings import ( + Substitution, + _fit_not_learn_docstring, + _fit_transform_docstring, + _missing_values_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, +) from feature_engine.outliers.base_outlier import BaseOutlier from feature_engine.parameter_checks import _define_numerical_dict from feature_engine.validation import _return_tags from feature_engine.variable_manipulation import _find_or_check_numerical_variables +@Substitution( + missing_values=_missing_values_docstring, + right_tail_caps_=BaseOutlier._right_tail_caps_docstring, + left_tail_caps_=BaseOutlier._left_tail_caps_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit=_fit_not_learn_docstring, + fit_transform=_fit_transform_docstring, +) class ArbitraryOutlierCapper(BaseOutlier): """ The ArbitraryOutlierCapper() caps the maximum or minimum values of a variable at an arbitrary value indicated by the user. You must provide the maximum or minimum values that will be used to cap each - variable in a dictionary {feature:capping value} + variable in a dictionary containing the features as keys and the capping values as + values. More details in the :ref:`User Guide `. @@ -31,39 +49,34 @@ class ArbitraryOutlierCapper(BaseOutlier): ---------- max_capping_dict: dictionary, default=None Dictionary containing the user specified capping values for the right tail of - the distribution of each variable (maximum values). + the distribution of each variable to cap (maximum values). min_capping_dict: dictionary, default=None Dictionary containing user specified capping values for the eft tail of the - distribution of each variable (minimum values). + distribution of each variable to cap (minimum values). - missing_values : string, default='raise' - Indicates if missing values should be ignored or raised. If - `missing_values='raise'` the transformer will return an error if the - training or the datasets to transform contain missing values. + {missing_values} Attributes ---------- - right_tail_caps_: - Dictionary with the maximum values at which variables will be capped. + {right_tail_caps_} - left_tail_caps_: - Dictionary with the minimum values at which variables will be capped. + {left_tail_caps_} - variables_: - The group of variables that will be transformed. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- - fit: - This transformer does not learn any parameter. + + {fit} + transform: Cap the variables. - fit_transform: - Fit to the data. Then transform it. + + {fit_transform} + """ def __init__( diff --git a/feature_engine/outliers/base_outlier.py b/feature_engine/outliers/base_outlier.py index a57caf218..af1864798 100644 --- a/feature_engine/outliers/base_outlier.py +++ b/feature_engine/outliers/base_outlier.py @@ -1,8 +1,9 @@ +from typing import List, Optional, Union + import numpy as np import pandas as pd from sklearn.base import BaseEstimator, TransformerMixin from sklearn.utils.validation import check_is_fitted -from typing import List, Optional, Union from feature_engine.dataframe_checks import ( _check_contains_inf, @@ -20,6 +21,16 @@ class BaseOutlier(BaseEstimator, TransformerMixin): """shared set-up checks and methods across outlier transformers""" + _right_tail_caps_docstring = """right_tail_caps_: + Dictionary with the maximum values beyond which a value will be considered an + outlier. + """.rstrip() + + _left_tail_caps_docstring = """left_tail_caps_: + Dictionary with the minimum values beyond which a value will be considered an + outlier. + """.rstrip() + def _check_transform_input_and_state(self, X: pd.DataFrame) -> pd.DataFrame: """Checks that the input is a dataframe and of the same size than the one used in the fit method. Checks absence of NA. @@ -97,6 +108,71 @@ def _more_tags(self): class WinsorizerBase(BaseOutlier): + + _intro_docstring = """The extreme values beyond which an observation is considered + an outlier are determined using: + + - a Gaussian approximation + - the inter-quantile range proximity rule (IQR) + - percentiles + + **Gaussian limits:** + + - right tail: mean + 3* std + - left tail: mean - 3* std + + **IQR limits:** + + - right tail: 75th quantile + 3* IQR + - left tail: 25th quantile - 3* IQR + + where IQR is the inter-quartile range: 75th quantile - 25th quantile. + + **percentiles:** + + - right tail: 95th percentile + - left tail: 5th percentile + + You can select how far out to cap the maximum or minimum values with the + parameter `'fold'`. + + If `capping_method='gaussian'` fold gives the value to multiply the std. + + If `capping_method='iqr'` fold is the value to multiply the IQR. + + If `capping_method='quantiles'`, fold is the percentile on each tail that should + be censored. For example, if fold=0.05, the limits will be the 5th and 95th + percentiles. If fold=0.1, the limits will be the 10th and 90th percentiles. + """.rstrip() + + _capping_method_docstring = """capping_method: str, default='gaussian' + Desired outlier detection method. Can take 'gaussian', 'iqr' or 'quantiles'. + + The transformer will find the maximum and / or minimum values beyond which a + data point will be considered an outlier using: + **'gaussian'**: the Gaussian approximation. + **'iqr'**: the IQR proximity rule. + **'quantiles'**: the percentiles. + """.rstrip() + + _tail_docstring = """tail: str, default='right' + Whether to look for outliers on the right, left or both tails of the + distribution. Can take 'left', 'right' or 'both'. + """.rstrip() + + _fold_docstring = """fold: int or float, default=3 + The factor used to multiply the std or IQR to calculate the maximum or minimum + allowed values. Recommended values are 2 or 3 for the gaussian approximation, + and 1.5 or 3 for the IQR proximity rule. + + If `capping_method='quantile'`, then `'fold'` indicates the percentile. So if + `fold=0.05`, the limits will be the 95th and 5th percentiles. + + **Note**: Outliers will be removed up to a maximum of the 20th percentiles on + both sides. Thus, when `capping_method='quantile'`, then `'fold'` takes values + between 0 and 0.20. + """.rstrip() + def __init__( self, capping_method: str = "gaussian", diff --git a/feature_engine/outliers/trimmer.py b/feature_engine/outliers/trimmer.py index 69365868b..5a8f3ce53 100644 --- a/feature_engine/outliers/trimmer.py +++ b/feature_engine/outliers/trimmer.py @@ -4,48 +4,37 @@ import numpy as np import pandas as pd +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _missing_values_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, + _variables_numerical_docstring, +) from feature_engine.outliers.base_outlier import WinsorizerBase +@Substitution( + intro_docstring=WinsorizerBase._intro_docstring, + capping_method=WinsorizerBase._capping_method_docstring, + tail=WinsorizerBase._tail_docstring, + fold=WinsorizerBase._fold_docstring, + variables=_variables_numerical_docstring, + missing_values=_missing_values_docstring, + right_tail_caps_=WinsorizerBase._right_tail_caps_docstring, + left_tail_caps_=WinsorizerBase._left_tail_caps_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit_transform=_fit_transform_docstring, +) class OutlierTrimmer(WinsorizerBase): """The OutlierTrimmer() removes observations with outliers from the dataset. The OutlierTrimmer() first calculates the maximum and /or minimum values beyond which a value will be considered an outlier, and thus removed. - Limits are determined using: - - - a Gaussian approximation - - the inter-quantile range proximity rule - - percentiles. - - **Gaussian limits:** - - - right tail: mean + 3* std - - left tail: mean - 3* std - - **IQR limits:** - - - right tail: 75th quantile + 3* IQR - - left tail: 25th quantile - 3* IQR - - where IQR is the inter-quartile range: 75th quantile - 25th quantile. - - **percentiles or quantiles:** - - - right tail: 95th percentile - - left tail: 5th percentile - - You can select how far out to cap the maximum or minimum values with the - parameter `'fold'`. - - If `capping_method='gaussian'` fold gives the value to multiply the std. - - If `capping_method='iqr'` fold is the value to multiply the IQR. - - If `capping_method='quantile'`, fold is the percentile on each tail that should - be censored. For example, if fold=0.05, the limits will be the 5th and 95th - percentiles. If fold=0.1, the limits will be the 10th and 90th percentiles. + {intro_docstring} The OutlierTrimmer() works only with numerical variables. A list of variables can be indicated. Alternatively, it will select all numerical variables. @@ -58,68 +47,36 @@ class OutlierTrimmer(WinsorizerBase): Parameters ---------- - capping_method: str, default='gaussian' - Desired capping method. Can take 'gaussian', 'iqr' or 'quantiles'. - - **'gaussian'**: the transformer will find the maximum and / or minimum values - to cap the variables using the Gaussian approximation. - - **'iqr'**: the transformer will find the boundaries using the IQR proximity - rule. - - **'quantiles'**: the limits are given by the percentiles. + {capping_method} - tail: str, default='right' - Whether to cap outliers on the right, left or both tails of the distribution. - Can take 'left', 'right' or 'both'. + {tail} - fold: int or float, default=3 - How far out to to place the capping values. The number that will multiply - the std or IQR to calculate the capping values. Recommended values, 2 - or 3 for the gaussian approximation, or 1.5 or 3 for the IQR proximity - rule. + {fold} - If `capping_method='quantile'`, then `'fold'` indicates the percentile. So if - `fold=0.05`, the limits will be the 95th and 5th percentiles. + {variables} - **Note**: Outliers will be removed up to a maximum of the 20th percentiles on - both sides. Thus, when `capping_method='quantile'`, then `'fold'` takes values - between 0 and 0.20. - - variables: list, default=None - The list of variables for which the outliers will be removed. If None, - the transformer will find and select all numerical variables. - - missing_values: string, default='raise' - Indicates if missing values should be ignored or raised. Sometimes we want to - remove outliers in the raw, original data, sometimes, we may want to remove - outliers in the already pre-transformed data. If missing_values='ignore', the - transformer will ignore missing data when learning the capping parameters or - transforming the data. If missing_values='raise' the transformer will return - an error if the training or the datasets to transform contain missing values. + {missing_values} Attributes ---------- - right_tail_caps_: - Dictionary with the maximum values above which values will be removed. + {right_tail_caps_} - left_tail_caps_ : - Dictionary with the minimum values below which values will be removed. + {left_tail_caps_} - variables_: - The group of variables that will be transformed. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- fit: Find maximum and minimum values. + transform: Remove outliers. - fit_transform: - Fit to the data. Then transform it. + + {fit_transform} + """ def transform(self, X: pd.DataFrame) -> pd.DataFrame: diff --git a/feature_engine/outliers/winsorizer.py b/feature_engine/outliers/winsorizer.py index 4d76aa909..b3b70bdb4 100644 --- a/feature_engine/outliers/winsorizer.py +++ b/feature_engine/outliers/winsorizer.py @@ -7,47 +7,36 @@ import pandas as pd from feature_engine.dataframe_checks import _is_dataframe +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _missing_values_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, + _variables_numerical_docstring, +) from feature_engine.outliers.base_outlier import WinsorizerBase +@Substitution( + intro_docstring=WinsorizerBase._intro_docstring, + capping_method=WinsorizerBase._capping_method_docstring, + tail=WinsorizerBase._tail_docstring, + fold=WinsorizerBase._fold_docstring, + variables=_variables_numerical_docstring, + missing_values=_missing_values_docstring, + right_tail_caps_=WinsorizerBase._right_tail_caps_docstring, + left_tail_caps_=WinsorizerBase._left_tail_caps_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit_transform=_fit_transform_docstring, +) class Winsorizer(WinsorizerBase): """ The Winsorizer() caps maximum and/or minimum values of a variable at automatically determined values, and optionally adds indicators. - The values to cap variables are determined using: - - - a Gaussian approximation - - the inter-quantile range proximity rule (IQR) - - percentiles - - **Gaussian limits:** - - - right tail: mean + 3* std - - left tail: mean - 3* std - - **IQR limits:** - - - right tail: 75th quantile + 3* IQR - - left tail: 25th quantile - 3* IQR - - where IQR is the inter-quartile range: 75th quantile - 25th quantile. - - **percentiles:** - - - right tail: 95th percentile - - left tail: 5th percentile - - You can select how far out to cap the maximum or minimum values with the - parameter `'fold'`. - - If `capping_method='gaussian'` fold gives the value to multiply the std. - - If `capping_method='iqr'` fold is the value to multiply the IQR. - - If `capping_method='quantiles'`, fold is the percentile on each tail that should - be censored. For example, if fold=0.05, the limits will be the 5th and 95th - percentiles. If fold=0.1, the limits will be the 10th and 90th percentiles. + {intro_docstring} The Winsorizer() works only with numerical variables. A list of variables can be indicated. Alternatively, the Winsorizer() will select and cap all numerical @@ -60,73 +49,41 @@ class Winsorizer(WinsorizerBase): Parameters ---------- - capping_method: str, default='gaussian' - Desired capping method. Can take 'gaussian', 'iqr' or 'quantiles'. - - **'gaussian'**: the transformer will find the maximum and / or minimum values - to cap the variables using the Gaussian approximation. - - **'iqr'**: the transformer will find the boundaries using the IQR proximity - rule. - - **'quantiles'**: the limits are given by the percentiles. + {capping_method} - tail: str, default='right' - Whether to cap outliers on the right, left or both tails of the distribution. - Can take 'left', 'right' or 'both'. + {tail} - fold: int or float, default=3 - How far out to to place the capping values. The number that will multiply - the std or IQR to calculate the capping values. Recommended values, 2 - or 3 for the gaussian approximation, or 1.5 or 3 for the IQR proximity - rule. - - If `capping_method='quantiles'`, then `'fold'` indicates the percentile. So if - `fold=0.05`, the limits will be the 95th and 5th percentiles. - - **Note**: Outliers will be removed up to a maximum of the 20th percentiles on - both sides. Thus, when `capping_method='quantiles'`, then `'fold'` takes values - between 0 and 0.20. + {fold} add_indicators: bool, default=False Whether to add indicator variables to flag the capped outliers. If 'True', binary variables will be added to flag outliers on the left and right tails of the distribution. One binary variable per tail, per variable. - variables: list, default=None - The list of variables for which the outliers will be capped. If None, - the transformer will select and cap all numerical variables. + {variables} - missing_values: string, default='raise' - Indicates if missing values should be ignored or raised. Sometimes we want to - remove outliers in the raw, original data, sometimes, we may want to remove - outliers in the already pre-transformed data. If `missing_values='ignore'`, the - transformer will ignore missing data when learning the capping parameters or - transforming the data. If `missing_values='raise'` the transformer will return - an error if the training or the datasets to transform contain missing values. + {missing_values} Attributes ---------- - right_tail_caps_: - Dictionary with the maximum values at which variables will be capped. + {right_tail_caps_} - left_tail_caps_ : - Dictionary with the minimum values at which variables will be capped. + {left_tail_caps_} - variables_: - The group of variables that will be transformed. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- fit: - Learn the values that should be used to replace outliers. + Learn the values that will replace the outliers. + transform: Cap the variables. - fit_transform: - Fit to the data. Then transform it. + + {fit_transform} + """ def __init__( @@ -184,9 +141,12 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: X_out = pd.concat([X_out, X_right.astype(np.float64)], axis=1) else: X_both = pd.concat([X_left, X_right], axis=1).astype(np.float64) - X_both = X_both[[ - cl1 for cl2 in zip(X_left.columns.values, X_right.columns.values) - for cl1 in cl2 - ]] + X_both = X_both[ + [ + cl1 + for cl2 in zip(X_left.columns.values, X_right.columns.values) + for cl1 in cl2 + ] + ] X_out = pd.concat([X_out, X_both], axis=1) return X_out diff --git a/feature_engine/preprocessing/match_columns.py b/feature_engine/preprocessing/match_columns.py index be268c9b1..5f7ade51b 100644 --- a/feature_engine/preprocessing/match_columns.py +++ b/feature_engine/preprocessing/match_columns.py @@ -63,10 +63,10 @@ class MatchVariables(BaseEstimator, TransformerMixin): The values for the variables that will be added to the transformed dataset. missing_values: string, default='ignore' - Indicates if missing values should be ignored or raised. If 'ignore', the - transformer will ignore missing data when transforming the data. If 'raise' the - transformer will return an error if the training or the datasets to transform - contain missing values. + Indicates if missing values should be ignored or raised. If 'raise' the + transformer will return an error if the the datasets to `fit` or `transform` + contain missing values. If 'ignore', missing data will be ignored when learning + parameters or performing the transformation. verbose: bool, default=True If True, the transformer will print out the names of the variables that are @@ -100,16 +100,20 @@ def __init__( if missing_values not in ["raise", "ignore"]: raise ValueError( "missing_values takes only values 'raise' or 'ignore'." - f"Got '{missing_values} instead.") + f"Got '{missing_values} instead." + ) if not isinstance(verbose, bool): - raise ValueError("verbose takes only booleans True and False." - f"Got '{verbose} instead.") + raise ValueError( + "verbose takes only booleans True and False." f"Got '{verbose} instead." + ) # note: np.nan is an instance of float!!! if not isinstance(fill_value, (str, int, float)): - raise ValueError("fill_value takes integers, floats or strings." - f"Got '{fill_value} instead.") + raise ValueError( + "fill_value takes integers, floats or strings." + f"Got '{fill_value} instead." + ) self.fill_value = fill_value self.missing_values = missing_values @@ -169,12 +173,15 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: if self.verbose: if len(_columns_to_add) > 0: - print("The following variables are added to the DataFrame: " - f"{_columns_to_add}") + print( + "The following variables are added to the DataFrame: " + f"{_columns_to_add}" + ) if len(_columns_to_drop) > 0: print( "The following variables are dropped from the DataFrame: " - f"{_columns_to_drop}") + f"{_columns_to_drop}" + ) X = X.drop(_columns_to_drop, axis=1) @@ -191,7 +198,8 @@ def _more_tags(self): msg = ( "transformer takes categorical variables, and inf cannot be determined" - "on these variables. Thus, check is not implemented") + "on these variables. Thus, check is not implemented" + ) tags_dict["_xfail_checks"]["check_estimators_nan_inf"] = msg return tags_dict diff --git a/feature_engine/transformation/boxcox.py b/feature_engine/transformation/boxcox.py index 44e01f5e7..d0a4e3f82 100644 --- a/feature_engine/transformation/boxcox.py +++ b/feature_engine/transformation/boxcox.py @@ -7,10 +7,23 @@ import scipy.stats as stats from feature_engine.base_transformers import BaseNumericalTransformer +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, + _variables_numerical_docstring, +) from feature_engine.validation import _return_tags from feature_engine.variable_manipulation import _check_input_parameter_variables +@Substitution( + variables=_variables_numerical_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit_transform=_fit_transform_docstring, +) class BoxCoxTransformer(BaseNumericalTransformer): """ The BoxCoxTransformer() applies the BoxCox transformation to numerical @@ -39,29 +52,26 @@ class BoxCoxTransformer(BaseNumericalTransformer): Parameters ---------- - variables: list, default=None - The list of numerical variables to transform. If None, the transformer will - automatically find and select all numerical variables. + {variables} Attributes ---------- lambda_dict_: Dictionary with the best BoxCox exponent per variable. - variables_: - The group of variables that will be transformed. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- fit: Learn the optimal lambda for the BoxCox transformation. + transform: Apply the BoxCox transformation. - fit_transform: - Fit to data, then transform it. + + {fit_transform} References ---------- diff --git a/feature_engine/transformation/log.py b/feature_engine/transformation/log.py index 3166445fd..6a378ae3b 100644 --- a/feature_engine/transformation/log.py +++ b/feature_engine/transformation/log.py @@ -7,10 +7,27 @@ import pandas as pd from feature_engine.base_transformers import BaseNumericalTransformer +from feature_engine.docstrings import ( + Substitution, + _fit_not_learn_docstring, + _fit_transform_docstring, + _inverse_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, + _variables_numerical_docstring, +) from feature_engine.validation import _return_tags from feature_engine.variable_manipulation import _check_input_parameter_variables +@Substitution( + variables=_variables_numerical_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit=_fit_not_learn_docstring, + fit_transform=_fit_transform_docstring, + inverse_transform=_inverse_transform_docstring, +) class LogTransformer(BaseNumericalTransformer): """ The LogTransformer() applies the natural logarithm or the base 10 logarithm to @@ -26,9 +43,7 @@ class LogTransformer(BaseNumericalTransformer): Parameters ---------- - variables: list, default=None - The list of numerical variables to transform. If None, the transformer - will find and select all numerical variables. + {variables} base: string, default='e' Indicates if the natural or base 10 logarithm should be applied. Can take @@ -36,22 +51,21 @@ class LogTransformer(BaseNumericalTransformer): Attributes ---------- - variables_: - The group of variables that will be transformed. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- - fit: - This transformer does not learn parameters. + {fit} + transform: Transform the variables using the logarithm. - fit_transform: - Fit to data, then transform it. - inverse_transform: - Convert the data back to the original representation. + + {fit_transform} + + {inverse_transform} + """ def __init__( @@ -173,6 +187,12 @@ def _more_tags(self): return tags_dict +@Substitution( + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit_transform=_fit_transform_docstring, + inverse_transform=_inverse_transform_docstring, +) class LogCpTransformer(BaseNumericalTransformer): """ The LogCpTransformer() applies the transformation log(x + C), where C is a positive @@ -211,26 +231,26 @@ class LogCpTransformer(BaseNumericalTransformer): Attributes ---------- - variables_: - The group of variables that will be transformed. + {variables_} C_: The constant C to add to each variable. If C = "auto" a dictionary with C = abs(min(variable)) + 1. - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- fit: Learn the constant C. + transform: Transform the variables with the logarithm of x plus C. - fit_transform: - Fit to data, then transform it. - inverse_transform: - Convert the data back to the original representation. + + {fit_transform} + + {inverse_transform} + """ def __init__( diff --git a/feature_engine/transformation/power.py b/feature_engine/transformation/power.py index 71626feb6..20598bf38 100644 --- a/feature_engine/transformation/power.py +++ b/feature_engine/transformation/power.py @@ -7,9 +7,26 @@ import pandas as pd from feature_engine.base_transformers import BaseNumericalTransformer +from feature_engine.docstrings import ( + Substitution, + _fit_not_learn_docstring, + _fit_transform_docstring, + _inverse_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, + _variables_numerical_docstring, +) from feature_engine.variable_manipulation import _check_input_parameter_variables +@Substitution( + variables=_variables_numerical_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit=_fit_not_learn_docstring, + fit_transform=_fit_transform_docstring, + inverse_transform=_inverse_transform_docstring, +) class PowerTransformer(BaseNumericalTransformer): """ The PowerTransformer() applies power or exponential transformations to @@ -25,31 +42,28 @@ class PowerTransformer(BaseNumericalTransformer): Parameters ---------- - variables: list, default=None - The list of numerical variables to transform. If None, the transformer will - automatically find and select all numerical variables. + {variables} exp: float or int, default=0.5 The power (or exponent). Attributes ---------- - variables_: - The group of variables that will be transformed. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- - fit: - This transformer does not learn parameters. + {fit} + transform: Apply the power transformation to the variables. - fit_transform: - Fit to data, then transform it. - inverse_transform: - Convert the data back to the original representation. + + {fit_transform} + + {inverse_transform} + """ def __init__( diff --git a/feature_engine/transformation/reciprocal.py b/feature_engine/transformation/reciprocal.py index c8b0e6198..e275181ed 100644 --- a/feature_engine/transformation/reciprocal.py +++ b/feature_engine/transformation/reciprocal.py @@ -7,10 +7,27 @@ import pandas as pd from feature_engine.base_transformers import BaseNumericalTransformer +from feature_engine.docstrings import ( + Substitution, + _fit_not_learn_docstring, + _fit_transform_docstring, + _inverse_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, + _variables_numerical_docstring, +) from feature_engine.validation import _return_tags from feature_engine.variable_manipulation import _check_input_parameter_variables +@Substitution( + variables=_variables_numerical_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit=_fit_not_learn_docstring, + fit_transform=_fit_transform_docstring, + inverse_transform=_inverse_transform_docstring, +) class ReciprocalTransformer(BaseNumericalTransformer): """ The ReciprocalTransformer() applies the reciprocal transformation 1 / x @@ -27,28 +44,25 @@ class ReciprocalTransformer(BaseNumericalTransformer): Parameters ---------- - variables: list, default=None - The list of numerical variables to transform. If None, the transformer will - automatically find and select all numerical variables. + {variables} Attributes ---------- - variables_: - The group of variables that will be transformed. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- - fit: - This transformer does not learn parameters. + {fit} + transform: Apply the reciprocal 1 / x transformation. - fit_transform: - Fit to data, then transform it. - inverse_transform: - Convert the data back to the original representation. + + {fit_transform} + + {inverse_transform} + """ def __init__( diff --git a/feature_engine/transformation/yeojohnson.py b/feature_engine/transformation/yeojohnson.py index ff9bf6579..b7b2e4b53 100644 --- a/feature_engine/transformation/yeojohnson.py +++ b/feature_engine/transformation/yeojohnson.py @@ -7,9 +7,22 @@ import scipy.stats as stats from feature_engine.base_transformers import BaseNumericalTransformer +from feature_engine.docstrings import ( + Substitution, + _fit_transform_docstring, + _n_features_in_docstring, + _variables_attribute_docstring, + _variables_numerical_docstring, +) from feature_engine.variable_manipulation import _check_input_parameter_variables +@Substitution( + variables=_variables_numerical_docstring, + variables_=_variables_attribute_docstring, + n_features_in_=_n_features_in_docstring, + fit_transform=_fit_transform_docstring, +) class YeoJohnsonTransformer(BaseNumericalTransformer): """ The YeoJohnsonTransformer() applies the Yeo-Johnson transformation to the @@ -29,29 +42,26 @@ class YeoJohnsonTransformer(BaseNumericalTransformer): Parameters ---------- - variables: list, default=None - The list of numerical variables to transform. If None, the transformer will - automatically find and select all numerical variables. + {variables} Attributes ---------- lambda_dict_ Dictionary containing the best lambda for the Yeo-Johnson per variable. - variables_: - The group of variables that will be transformed. + {variables_} - n_features_in_: - The number of features in the train set used in fit. + {n_features_in_} Methods ------- fit: Learn the optimal lambda for the Yeo-Johnson transformation. + transform: Apply the Yeo-Johnson transformation. - fit_transform: - Fit to data, then transform it. + + {fit_transform} References ---------- diff --git a/tests/test_creation/test_combine_with_reference_feature.py b/tests/test_creation/test_combine_with_reference_feature.py index 63bb8d339..4d898a5b9 100644 --- a/tests/test_creation/test_combine_with_reference_feature.py +++ b/tests/test_creation/test_combine_with_reference_feature.py @@ -75,7 +75,7 @@ def test_error_when_drop_original_not_bool(): CombineWithReferenceFeature( variables_to_combine=["Age"], reference_variables=["Marks"], - drop_original="not_a_bool" + drop_original="not_a_bool", ) @@ -206,7 +206,7 @@ def test_drop_original_variables(df_vartypes): transformer = CombineWithReferenceFeature( variables_to_combine=["Age", "Marks"], reference_variables=["Age", "Marks"], - drop_original=True + drop_original=True, ) X = transformer.fit_transform(df_vartypes) diff --git a/tests/test_datetime/test_datetime_features.py b/tests/test_datetime/test_datetime_features.py index 97a89c37b..203470f57 100644 --- a/tests/test_datetime/test_datetime_features.py +++ b/tests/test_datetime/test_datetime_features.py @@ -225,9 +225,9 @@ def test_extract_all_datetime_features(df_datetime, df_datetime_transformed): def test_extract_specified_datetime_features(df_datetime, df_datetime_transformed): - X = DatetimeFeatures( - features_to_extract=["semester", "week"] - ).fit_transform(df_datetime) + X = DatetimeFeatures(features_to_extract=["semester", "week"]).fit_transform( + df_datetime + ) pd.testing.assert_frame_equal( X, df_datetime_transformed[ @@ -277,8 +277,10 @@ def test_extract_features_from_different_timezones( lambda x: x.subtract(time_zones) ), ) - exp_err_msg = "ValueError: variable(s) time_obj " \ + exp_err_msg = ( + "ValueError: variable(s) time_obj " "could not be converted to datetime. Try setting utc=True" + ) with pytest.raises(ValueError) as errinfo: assert DatetimeFeatures( variables="time_obj", features_to_extract=["hour"], utc=False diff --git a/tests/test_outliers/test_winsorizer.py b/tests/test_outliers/test_winsorizer.py index 1737d9fb3..c597dc5ff 100644 --- a/tests/test_outliers/test_winsorizer.py +++ b/tests/test_outliers/test_winsorizer.py @@ -166,20 +166,23 @@ def test_quantile_capping_both_tails_with_fold_15_percent(df_normal_dist): def test_indicators_are_added(df_normal_dist): transformer = Winsorizer( - tail="both", capping_method="quantiles", fold=0.1, add_indicators=True) + tail="both", capping_method="quantiles", fold=0.1, add_indicators=True + ) X = transformer.fit_transform(df_normal_dist) # test that the number of output variables is correct assert X.shape[1] == 3 * df_normal_dist.shape[1] assert np.all(X.iloc[:, df_normal_dist.shape[1]:].sum(axis=0) > 0) transformer = Winsorizer( - tail="left", capping_method="quantiles", fold=0.1, add_indicators=True) + tail="left", capping_method="quantiles", fold=0.1, add_indicators=True + ) X = transformer.fit_transform(df_normal_dist) assert X.shape[1] == 2 * df_normal_dist.shape[1] assert np.all(X.iloc[:, df_normal_dist.shape[1]:].sum(axis=0) > 0) transformer = Winsorizer( - tail="right", capping_method="quantiles", fold=0.1, add_indicators=True) + tail="right", capping_method="quantiles", fold=0.1, add_indicators=True + ) X = transformer.fit_transform(df_normal_dist) assert X.shape[1] == 2 * df_normal_dist.shape[1] assert np.all(X.iloc[:, df_normal_dist.shape[1]:].sum(axis=0) > 0) @@ -191,7 +194,7 @@ def test_indicators_filter_variables(df_vartypes): tail="both", capping_method="quantiles", fold=0.1, - add_indicators=True + add_indicators=True, ) X = transformer.fit_transform(df_vartypes) assert X.shape[1] == df_vartypes.shape[1] + 4 @@ -207,17 +210,13 @@ def test_indicators_filter_variables(df_vartypes): def test_indicators_are_correct(): transformer = Winsorizer( - tail="left", - capping_method="quantiles", - fold=0.1, - add_indicators=True + tail="left", capping_method="quantiles", fold=0.1, add_indicators=True ) df = pd.DataFrame({"col": np.arange(100).astype(np.float64)}) df_out = transformer.fit_transform(df) expected_ind = np.r_[np.repeat(True, 10), np.repeat(False, 90)].astype(np.float64) pd.testing.assert_frame_equal( - df_out.drop("col", axis=1), - df.assign(col_left=expected_ind).drop("col", axis=1) + df_out.drop("col", axis=1), df.assign(col_left=expected_ind).drop("col", axis=1) ) transformer.set_params(tail="right") @@ -225,23 +224,22 @@ def test_indicators_are_correct(): expected_ind = np.r_[np.repeat(False, 90), np.repeat(True, 10)].astype(np.float64) pd.testing.assert_frame_equal( df_out.drop("col", axis=1), - df.assign(col_right=expected_ind).drop("col", axis=1) + df.assign(col_right=expected_ind).drop("col", axis=1), ) transformer.set_params(tail="both") df_out = transformer.fit_transform(df) - expected_ind_left = np.r_[ - np.repeat(True, 10), np.repeat(False, 90) - ].astype(np.float64) - expected_ind_right = np.r_[ - np.repeat(False, 90), np.repeat(True, 10) - ].astype(np.float64) + expected_ind_left = np.r_[np.repeat(True, 10), np.repeat(False, 90)].astype( + np.float64 + ) + expected_ind_right = np.r_[np.repeat(False, 90), np.repeat(True, 10)].astype( + np.float64 + ) pd.testing.assert_frame_equal( df_out.drop("col", axis=1), - df.assign( - col_left=expected_ind_left, - col_right=expected_ind_right - ).drop("col", axis=1) + df.assign(col_left=expected_ind_left, col_right=expected_ind_right).drop( + "col", axis=1 + ), ) diff --git a/tests/test_preprocessing/test_match_columns.py b/tests/test_preprocessing/test_match_columns.py index ec1be0aa7..6c4efd2f6 100644 --- a/tests/test_preprocessing/test_match_columns.py +++ b/tests/test_preprocessing/test_match_columns.py @@ -8,10 +8,8 @@ _params_fill_value = [ (1, [1, 1, 1, 1], [1, 1, 1, 1]), (0.1, [0.1, 0.1, 0.1, 0.1], [0.1, 0.1, 0.1, 0.1]), - ("none", ["none", "none", "none", - "none"], ["none", "none", "none", "none"]), - (np.nan, [np.nan, np.nan, np.nan, - np.nan], [np.nan, np.nan, np.nan, np.nan]), + ("none", ["none", "none", "none", "none"], ["none", "none", "none", "none"]), + (np.nan, [np.nan, np.nan, np.nan, np.nan], [np.nan, np.nan, np.nan, np.nan]), ] _params_allowed = [ @@ -21,10 +19,12 @@ ] -@pytest.mark.parametrize("fill_value, expected_studies, expected_age", - _params_fill_value) -def test_drop_and_add_columns(fill_value, expected_studies, expected_age, - df_vartypes, df_na): +@pytest.mark.parametrize( + "fill_value, expected_studies, expected_age", _params_fill_value +) +def test_drop_and_add_columns( + fill_value, expected_studies, expected_age, df_vartypes, df_na +): train = df_na.copy() test = df_vartypes.copy() test = test.drop("Age", axis=1) # to add more than one column @@ -41,14 +41,16 @@ def test_drop_and_add_columns(fill_value, expected_studies, expected_age, transformed_df = match_columns.transform(test) - expected_result = pd.DataFrame({ - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Studies": expected_studies, - "Age": expected_age, - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": pd.date_range("2020-02-24", periods=4, freq="T"), - }) + expected_result = pd.DataFrame( + { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Studies": expected_studies, + "Age": expected_age, + "Marks": [0.9, 0.8, 0.7, 0.6], + "dob": pd.date_range("2020-02-24", periods=4, freq="T"), + } + ) # test init params if fill_value is np.nan: @@ -64,10 +66,12 @@ def test_drop_and_add_columns(fill_value, expected_studies, expected_age, pd.testing.assert_frame_equal(expected_result, transformed_df) -@pytest.mark.parametrize("fill_value, expected_studies, expected_age", - _params_fill_value) +@pytest.mark.parametrize( + "fill_value, expected_studies, expected_age", _params_fill_value +) def test_columns_addition_when_more_columns_in_train_than_test( - fill_value, expected_studies, expected_age, df_vartypes, df_na): + fill_value, expected_studies, expected_age, df_vartypes, df_na +): train = df_na.copy() test = df_vartypes.copy() @@ -81,14 +85,16 @@ def test_columns_addition_when_more_columns_in_train_than_test( transformed_df = match_columns.transform(test) - expected_result = pd.DataFrame({ - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Studies": expected_studies, - "Age": expected_age, - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": pd.date_range("2020-02-24", periods=4, freq="T"), - }) + expected_result = pd.DataFrame( + { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Studies": expected_studies, + "Age": expected_age, + "Marks": [0.9, 0.8, 0.7, 0.6], + "dob": pd.date_range("2020-02-24", periods=4, freq="T"), + } + ) # test init params if fill_value is np.nan: @@ -127,20 +133,17 @@ def test_drop_columns_when_more_columns_in_test_than_train(df_vartypes, df_na): pd.testing.assert_frame_equal(expected_result, transformed_df) -@pytest.mark.parametrize("fill_value, missing_values, verbose", - _params_allowed) -def test_error_if_param_values_not_allowed(fill_value, missing_values, - verbose): +@pytest.mark.parametrize("fill_value, missing_values, verbose", _params_allowed) +def test_error_if_param_values_not_allowed(fill_value, missing_values, verbose): with pytest.raises(ValueError): - MatchVariables(fill_value=fill_value, - missing_values=missing_values, - verbose=verbose) + MatchVariables( + fill_value=fill_value, missing_values=missing_values, verbose=verbose + ) def test_verbose_print_out(capfd, df_vartypes, df_na): - match_columns = MatchVariables(missing_values="ignore", - verbose=True) + match_columns = MatchVariables(missing_values="ignore", verbose=True) train = df_na.copy() train.loc[:, "new_variable"] = 5 @@ -149,19 +152,23 @@ def test_verbose_print_out(capfd, df_vartypes, df_na): match_columns.transform(df_vartypes) out, err = capfd.readouterr() - assert (out == "The following variables are added to the DataFrame: " - "['new_variable', 'Studies']\n" - or out == "The following variables are added to the DataFrame: " - "['Studies', 'new_variable']\n") + assert ( + out == "The following variables are added to the DataFrame: " + "['new_variable', 'Studies']\n" + or out == "The following variables are added to the DataFrame: " + "['Studies', 'new_variable']\n" + ) match_columns.fit(df_vartypes) match_columns.transform(train) out, err = capfd.readouterr() - assert (out == "The following variables are dropped from the DataFrame: " - "['new_variable', 'Studies']\n" or out - == "The following variables are dropped from the DataFrame: " - "['Studies', 'new_variable']\n") + assert ( + out == "The following variables are dropped from the DataFrame: " + "['new_variable', 'Studies']\n" + or out == "The following variables are dropped from the DataFrame: " + "['Studies', 'new_variable']\n" + ) def test_raises_error_if_na_in_df(df_na, df_vartypes):