From fdc7b9207fd6a28a016d9ce0fe1f699e161be18a Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Thu, 7 Apr 2022 09:08:18 +0200 Subject: [PATCH 01/10] adds check_y --- feature_engine/dataframe_checks.py | 75 ++++++++++++++++++++++++++++++ tests/test_dataframe_checks.py | 31 +++++++++++- 2 files changed, 105 insertions(+), 1 deletion(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index b85d415bd..6dd520083 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -7,6 +7,7 @@ import numpy as np import pandas as pd from scipy.sparse import issparse +from sklearn.utils.validation import _check_y def check_X(X: Union[np.generic, np.ndarray, pd.DataFrame]) -> pd.DataFrame: @@ -85,6 +86,80 @@ def check_X(X: Union[np.generic, np.ndarray, pd.DataFrame]) -> pd.DataFrame: return X +def check_y( + y: Union[np.generic, np.ndarray, pd.Series, List], + multi_output: bool = False, + y_numeric: bool = True, +) -> pd.Series: + """ + Checks that y is a series, or alternatively, if it can be converted to a series. + + Parameters + ---------- + y : pd.Series, np.array, list + + multi_output : bool, default=False + Whether to allow 2D y (array). If false, y will be + validated as a vector. y cannot have np.nan or np.inf values if + multi_output=True. + + y_numeric : bool, default=False + Whether to ensure that y has a numeric type. If dtype of y is object, + it is converted to float64. Should only be used for regression + algorithms. + + Returns + ------- + y: pd.Series + """ + + if y is None: + raise ValueError("y cannot be None.") + + elif isinstance(y, pd.Series): + _check_y(y, multi_output=multi_output, y_numeric=y_numeric) + y = y.copy() + + else: + y = _check_y(y, multi_output=multi_output, y_numeric=y_numeric) + y = pd.Series(y) + + return y + + +def _check_pd_X_y( + X: Union[np.generic, np.ndarray, pd.DataFrame], + y: Union[np.generic, np.ndarray, pd.Series, List], + multi_output: bool = False, + y_numeric: bool = True, +) -> (pd.DataFrame, pd.Series): + """ + Ensures X and y are compatible pandas DataFrame and Series. If both are pandas + objects, checks that their indexes match. If any is a numpy array, converts to + pandas object with compatible index. + + This transformer ensures that we can concatenate X and y using `pandas.concat`, + functionality needed in the encoders. + + Parameters + ---------- + X: Pandas DataFrame or numpy ndarray + y: Pandas Series or numpy ndarray + + Raises + ------ + ValueError: if X and y are pandas objects with inconsistent indexes. + TypeError: if X is sparse matrix, empty dataframe or not a dataframe. + TypeError: if y can't be parsed as pandas Series. + + Returns + ------- + X: Pandas DataFrame + y: Pandas Series + """ + pass + + def _check_X_matches_training_df(X: pd.DataFrame, reference: int) -> None: """ Checks that DataFrame to transform has the same number of columns that the diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 6ed847214..74ab559c1 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -1,7 +1,7 @@ import numpy as np import pandas as pd import pytest -from pandas.testing import assert_frame_equal +from pandas.testing import assert_frame_equal, assert_series_equal from scipy.sparse import csr_matrix from feature_engine.dataframe_checks import ( @@ -9,6 +9,7 @@ _check_contains_na, _check_X_matches_training_df, check_X, + check_y, ) @@ -42,6 +43,34 @@ def test_raises_error_if_empty_df(): check_X(df) +def test_check_y_returns_series(): + s = pd.Series([0,1,2,3,4]) + assert_series_equal(check_y(s), s) + + +def test_check_y_converts_np_array(): + a1D = np.array([1, 2, 3, 4]) + s = pd.Series(a1D) + assert_series_equal(check_y(a1D), s) + + +def test_check_y_raises_none_error(): + with pytest.raises(ValueError): + check_y(None) + + +def test_check_y_raises_nan_error(): + s = pd.Series([0, np.nan, 2, 3, 4]) + with pytest.raises(ValueError): + check_y(s) + + +def test_check_y_raises_inf_error(): + s = pd.Series([0, np.inf, 2, 3, 4]) + with pytest.raises(ValueError): + check_y(s) + + def test_check_X_matches_training_df(df_vartypes): with pytest.raises(ValueError): assert _check_X_matches_training_df(df_vartypes, 4) From 272901c5ea8b7cfe2a718fece588b3aec71fb617 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Thu, 7 Apr 2022 09:25:45 +0200 Subject: [PATCH 02/10] adds check_x_y --- feature_engine/dataframe_checks.py | 19 ++++++++++---- tests/test_dataframe_checks.py | 42 +++++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index 6dd520083..74eaf9701 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -2,12 +2,12 @@ transform(). """ -from typing import List, Union +from typing import List, Union, Tuple import numpy as np import pandas as pd from scipy.sparse import issparse -from sklearn.utils.validation import _check_y +from sklearn.utils.validation import _check_y, check_consistent_length def check_X(X: Union[np.generic, np.ndarray, pd.DataFrame]) -> pd.DataFrame: @@ -127,12 +127,12 @@ def check_y( return y -def _check_pd_X_y( +def check_X_y( X: Union[np.generic, np.ndarray, pd.DataFrame], y: Union[np.generic, np.ndarray, pd.Series, List], multi_output: bool = False, y_numeric: bool = True, -) -> (pd.DataFrame, pd.Series): +) -> Tuple[pd.DataFrame, pd.Series]: """ Ensures X and y are compatible pandas DataFrame and Series. If both are pandas objects, checks that their indexes match. If any is a numpy array, converts to @@ -157,7 +157,16 @@ def _check_pd_X_y( X: Pandas DataFrame y: Pandas Series """ - pass + X = check_X(X) + y = check_y(y, multi_output=multi_output, y_numeric=y_numeric) + check_consistent_length(X, y) + + # If X and y were a DataFrame and a Series, they are copied without transformation. + # Check that their indexes match. + if not all(y.index == X.index): + raise ValueError("The indexes of X and y do not match.") + + return X, y def _check_X_matches_training_df(X: pd.DataFrame, reference: int) -> None: diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 74ab559c1..9790f4b8f 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -9,6 +9,7 @@ _check_contains_na, _check_X_matches_training_df, check_X, + check_X_y, check_y, ) @@ -44,7 +45,7 @@ def test_raises_error_if_empty_df(): def test_check_y_returns_series(): - s = pd.Series([0,1,2,3,4]) + s = pd.Series([0, 1, 2, 3, 4]) assert_series_equal(check_y(s), s) @@ -71,6 +72,45 @@ def test_check_y_raises_inf_error(): check_y(s) +def test_check_x_y_returns_pandas(df_vartypes): + s = pd.Series([0, 1, 2, 3]) + x, y = check_X_y(df_vartypes, s) + assert_frame_equal(df_vartypes, x) + assert_series_equal(s, y) + + +def test_check_x_y_converts_numpy_to_pandas(): + a2D = np.array([[1, 2], [3, 4], [3, 4], [3, 4]]) + df_2D = pd.DataFrame(a2D, columns=["0", "1"]) + + a1D = np.array([1, 2, 3, 4]) + s = pd.Series(a1D) + + x, y = check_X_y(df_2D, s) + assert_frame_equal(df_2D, x) + assert_series_equal(s, y) + + +def test_check_x_y_inconsistent_length(df_vartypes): + s = pd.Series([0, 1, 2, 3, 5]) + with pytest.raises(ValueError): + check_X_y(df_vartypes, s) + + +def test_check_x_y_raises_index_mismatch(df_vartypes): + s = pd.Series( + [ + 0, + 1, + 2, + 3, + ], + index=[2, 3, 4, 5], + ) + with pytest.raises(ValueError): + check_X_y(df_vartypes, s) + + def test_check_X_matches_training_df(df_vartypes): with pytest.raises(ValueError): assert _check_X_matches_training_df(df_vartypes, 4) From 17133bbe2b33ab8de08b7fa51eb366f2d3cae8d7 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Thu, 7 Apr 2022 09:54:41 +0200 Subject: [PATCH 03/10] reformates encoders to accomodate check_x_y --- feature_engine/encoding/base_encoder.py | 34 +++++++++----------- feature_engine/encoding/count_frequency.py | 5 +-- feature_engine/encoding/decision_tree.py | 5 +-- feature_engine/encoding/mean_encoding.py | 7 ++-- feature_engine/encoding/one_hot.py | 4 ++- feature_engine/encoding/ordinal.py | 13 ++++---- feature_engine/encoding/probability_ratio.py | 8 ++--- feature_engine/encoding/rare_label.py | 4 ++- feature_engine/encoding/woe.py | 8 ++--- 9 files changed, 44 insertions(+), 44 deletions(-) diff --git a/feature_engine/encoding/base_encoder.py b/feature_engine/encoding/base_encoder.py index 2bb15a641..20a6762e3 100644 --- a/feature_engine/encoding/base_encoder.py +++ b/feature_engine/encoding/base_encoder.py @@ -1,5 +1,5 @@ import warnings -from typing import List, Union +from typing import List, Union, Tuple import pandas as pd from sklearn.base import BaseEstimator, TransformerMixin @@ -9,6 +9,7 @@ _check_contains_na, _check_X_matches_training_df, check_X, + check_X_y, ) from feature_engine.docstrings import Substitution from feature_engine.encoding._docstrings import ( @@ -53,10 +54,18 @@ def __init__( self.variables = _check_input_parameter_variables(variables) self.ignore_format = ignore_format - def _check_fit_input_and_variables(self, X: pd.DataFrame) -> pd.DataFrame: + def _check_X(self, X: pd.DataFrame) -> pd.DataFrame: + return check_X(X) + + def _check_X_y( + self, X: pd.DataFrame, y: pd.Series + ) -> Tuple[pd.DataFrame, pd.Series]: + return check_X_y(X, y) + + def _check_or_select_variables(self, X: pd.DataFrame): """ - Checks that input is a dataframe, finds categorical variables, or alternatively - checks that the variables entered by the user are of type object (categorical). + Finds categorical variables, or alternatively checks that the variables + entered by the user are of type object (categorical). Checks absence of NA. Parameters @@ -66,23 +75,11 @@ def _check_fit_input_and_variables(self, X: pd.DataFrame) -> pd.DataFrame: Raises ------ TypeError - If the input is not a Pandas DataFrame. If any user provided variable is not categorical ValueError If there are no categorical variables in the df or the df is empty If the variable(s) contain null values - - Returns - ------- - X: Pandas DataFrame - The same dataframe entered as parameter - variables : list - list of categorical variables """ - - # check input dataframe - X = check_X(X) - if not self.ignore_format: # find categorical variables or check variables entered by user are object self.variables_: List[ @@ -95,14 +92,14 @@ def _check_fit_input_and_variables(self, X: pd.DataFrame) -> pd.DataFrame: # check if dataset contains na _check_contains_na(X, self.variables_) + def _get_feature_names_in(self, X: pd.DataFrame): + # save input features self.feature_names_in_ = X.columns.tolist() # save train set shape self.n_features_in_ = X.shape[1] - return X - def _check_transform_input_and_state(self, X: pd.DataFrame) -> pd.DataFrame: """ Checks that the input is a dataframe and of the same size than the one used @@ -283,7 +280,6 @@ def __init__( ignore_format: bool = False, errors: str = "ignore", ) -> None: - if errors not in ["raise", "ignore"]: raise ValueError( "errors takes only values 'raise' and 'ignore ." diff --git a/feature_engine/encoding/count_frequency.py b/feature_engine/encoding/count_frequency.py index 420b97b40..afb9240b3 100644 --- a/feature_engine/encoding/count_frequency.py +++ b/feature_engine/encoding/count_frequency.py @@ -138,8 +138,9 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): y: pandas Series, default = None y is not needed in this encoder. You can pass y or None. """ - - X = self._check_fit_input_and_variables(X) + X = self._check_X(X) + self._check_or_select_variables(X) + self._get_feature_names_in(X) self.encoder_dict_ = {} diff --git a/feature_engine/encoding/decision_tree.py b/feature_engine/encoding/decision_tree.py index 82e4604c0..590c01263 100644 --- a/feature_engine/encoding/decision_tree.py +++ b/feature_engine/encoding/decision_tree.py @@ -188,6 +188,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): The target variable. Required to train the decision tree and for ordered ordinal encoding. """ + X, y = self._check_X_y(X, y) # confirm model type and target variables are compatible. if self.regression is True: @@ -201,8 +202,8 @@ def fit(self, X: pd.DataFrame, y: pd.Series): else: check_classification_targets(y) - # check input dataframe - X = self._check_fit_input_and_variables(X) + self._check_or_select_variables(X) + self._get_feature_names_in(X) if self.param_grid: param_grid = self.param_grid diff --git a/feature_engine/encoding/mean_encoding.py b/feature_engine/encoding/mean_encoding.py index 9a69da7ca..42bb2f455 100644 --- a/feature_engine/encoding/mean_encoding.py +++ b/feature_engine/encoding/mean_encoding.py @@ -131,10 +131,9 @@ def fit(self, X: pd.DataFrame, y: pd.Series): The target. """ - X = self._check_fit_input_and_variables(X) - - if not isinstance(y, pd.Series): - y = pd.Series(y) + X, y = self._check_X_y(X, y) + self._check_or_select_variables(X) + self._get_feature_names_in(X) temp = pd.concat([X, y], axis=1) temp.columns = list(X.columns) + ["target"] diff --git a/feature_engine/encoding/one_hot.py b/feature_engine/encoding/one_hot.py index 9da3dde30..475d45baa 100644 --- a/feature_engine/encoding/one_hot.py +++ b/feature_engine/encoding/one_hot.py @@ -180,7 +180,9 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): None. """ - X = self._check_fit_input_and_variables(X) + X = self._check_X(X) + self._check_or_select_variables(X) + self._get_feature_names_in(X) self.encoder_dict_ = {} diff --git a/feature_engine/encoding/ordinal.py b/feature_engine/encoding/ordinal.py index cca752a8d..8327d2f90 100644 --- a/feature_engine/encoding/ordinal.py +++ b/feature_engine/encoding/ordinal.py @@ -146,16 +146,15 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): Otherwise, y needs to be passed when fitting the transformer. """ - X = self._check_fit_input_and_variables(X) - - # join target to predictor variables if self.encoding_method == "ordered": - if y is None: - raise ValueError("Please provide a target y for this encoding method") + X, y = self._check_X_y(X, y) + else: + X = self._check_X(X) - if not isinstance(y, pd.Series): - y = pd.Series(y) + self._check_or_select_variables(X) + self._get_feature_names_in(X) + if self.encoding_method == "ordered": temp = pd.concat([X, y], axis=1) temp.columns = list(X.columns) + ["target"] diff --git a/feature_engine/encoding/probability_ratio.py b/feature_engine/encoding/probability_ratio.py index bfc266945..facb0777a 100644 --- a/feature_engine/encoding/probability_ratio.py +++ b/feature_engine/encoding/probability_ratio.py @@ -154,10 +154,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): Target, must be binary. """ - X = self._check_fit_input_and_variables(X) - - if not isinstance(y, pd.Series): - y = pd.Series(y) + X, y = self._check_X_y(X, y) # check that y is binary if y.nunique() != 2: @@ -166,6 +163,9 @@ def fit(self, X: pd.DataFrame, y: pd.Series): "used has more than 2 unique values." ) + self._check_or_select_variables(X) + self._get_feature_names_in(X) + temp = pd.concat([X, y], axis=1) temp.columns = list(X.columns) + ["target"] diff --git a/feature_engine/encoding/rare_label.py b/feature_engine/encoding/rare_label.py index e891398b9..18aa6788a 100644 --- a/feature_engine/encoding/rare_label.py +++ b/feature_engine/encoding/rare_label.py @@ -147,7 +147,9 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): y is not required. You can pass y or None. """ - X = self._check_fit_input_and_variables(X) + X = self._check_X(X) + self._check_or_select_variables(X) + self._get_feature_names_in(X) self.encoder_dict_ = {} diff --git a/feature_engine/encoding/woe.py b/feature_engine/encoding/woe.py index fd32421f4..a8d6dbbed 100644 --- a/feature_engine/encoding/woe.py +++ b/feature_engine/encoding/woe.py @@ -136,10 +136,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): Target, must be binary. """ - X = self._check_fit_input_and_variables(X) - - if not isinstance(y, pd.Series): - y = pd.Series(y) + X, y = self._check_X_y(X, y) # check that y is binary if y.nunique() != 2: @@ -148,6 +145,9 @@ def fit(self, X: pd.DataFrame, y: pd.Series): "used has more than 2 unique values." ) + self._check_or_select_variables(X) + self._get_feature_names_in(X) + temp = pd.concat([X, y], axis=1) temp.columns = list(X.columns) + ["target"] From 7d3d948503d50779070c01cb99f93d971db54324 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Thu, 7 Apr 2022 10:29:00 +0200 Subject: [PATCH 04/10] adds special cases when 1 object is pandas --- feature_engine/dataframe_checks.py | 49 +++++++++++++++++++++--------- tests/test_dataframe_checks.py | 32 +++++++++++++++++++ 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index 74eaf9701..d60cccb7e 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -87,9 +87,9 @@ def check_X(X: Union[np.generic, np.ndarray, pd.DataFrame]) -> pd.DataFrame: def check_y( - y: Union[np.generic, np.ndarray, pd.Series, List], - multi_output: bool = False, - y_numeric: bool = True, + y: Union[np.generic, np.ndarray, pd.Series, List], + multi_output: bool = False, + y_numeric: bool = True, ) -> pd.Series: """ Checks that y is a series, or alternatively, if it can be converted to a series. @@ -128,10 +128,10 @@ def check_y( def check_X_y( - X: Union[np.generic, np.ndarray, pd.DataFrame], - y: Union[np.generic, np.ndarray, pd.Series, List], - multi_output: bool = False, - y_numeric: bool = True, + X: Union[np.generic, np.ndarray, pd.DataFrame], + y: Union[np.generic, np.ndarray, pd.Series, List], + multi_output: bool = False, + y_numeric: bool = True, ) -> Tuple[pd.DataFrame, pd.Series]: """ Ensures X and y are compatible pandas DataFrame and Series. If both are pandas @@ -157,14 +157,33 @@ def check_X_y( X: Pandas DataFrame y: Pandas Series """ - X = check_X(X) - y = check_y(y, multi_output=multi_output, y_numeric=y_numeric) - check_consistent_length(X, y) - - # If X and y were a DataFrame and a Series, they are copied without transformation. - # Check that their indexes match. - if not all(y.index == X.index): - raise ValueError("The indexes of X and y do not match.") + + def _check_X_y(X, y): + X = check_X(X) + y = check_y(y, multi_output=multi_output, y_numeric=y_numeric) + check_consistent_length(X, y) + return X, y + + # case 1: both are pandas objects + if isinstance(X, pd.DataFrame) and isinstance(y, pd.Series): + X, y = _check_X_y(X, y) + # Check that their indexes match. + if not all(y.index == X.index): + raise ValueError("The indexes of X and y do not match.") + + # case 2: X is dataframe and y is something else + if isinstance(X, pd.DataFrame) and not isinstance(y, pd.Series): + X, y = _check_X_y(X, y) + y.index = X.index + + # case 3: X is not a dataframe and y is a series + elif not isinstance(X, pd.DataFrame) and isinstance(y, pd.Series): + X, y = _check_X_y(X, y) + X.index = y.index + + # all other cases + else: + X, y = _check_X_y(X, y) return X, y diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 9790f4b8f..6a006bcb6 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -79,6 +79,38 @@ def test_check_x_y_returns_pandas(df_vartypes): assert_series_equal(s, y) +def test_check_X_y_pandas_non_typical_index(): + df = pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ) + s = pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]) + x, y = check_X_y(df, s) + assert_frame_equal(df, x) + assert_series_equal(s, y) + + +def test_check_x_y_reassings_index(): + # case 1: X is dataframe, y is something else + df = pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ) + s = np.array([1, 2, 3, 4]) + s_exp = pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]) + x, y = check_X_y(df, s) + assert_frame_equal(df, x) + assert_series_equal(s_exp.astype(int), y.astype(int)) + + # case 2: X is not a df, y is a series + df = np.array([[1, 2, 3, 4], [5, 6, 7, 8]]).T + s = pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]) + df_exp = pd.DataFrame(df, columns=["0", "1"]) + df_exp.index = s.index + + x, y = check_X_y(df, s) + assert_frame_equal(df_exp, x) + assert_series_equal(s, y) + + def test_check_x_y_converts_numpy_to_pandas(): a2D = np.array([[1, 2], [3, 4], [3, 4], [3, 4]]) df_2D = pd.DataFrame(a2D, columns=["0", "1"]) From 7528219c8f2bcc32f53078b1caf52f950bca1cfa Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Thu, 7 Apr 2022 11:14:01 +0200 Subject: [PATCH 05/10] adds noahs tests for estimators --- tests/test_dataframe_checks.py | 9 + .../test_check_estimator_encoders.py | 215 ++++++++++++++++++ 2 files changed, 224 insertions(+) diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 6a006bcb6..0d61a4c71 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -89,6 +89,15 @@ def test_check_X_y_pandas_non_typical_index(): assert_series_equal(s, y) +def test_check_X_y_pandas_index_dont_match(): + df = pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ) + s = pd.Series([1, 2, 3, 4], index=[22, 99, 101, 999]) + with pytest.raises(ValueError): + check_X_y(df, s) + + def test_check_x_y_reassings_index(): # case 1: X is dataframe, y is something else df = pd.DataFrame( diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index 9600d36d9..402535e31 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -1,4 +1,5 @@ import pytest +import pandas as pd from sklearn.utils.estimator_checks import check_estimator from feature_engine.encoding import ( @@ -50,3 +51,217 @@ def test_check_estimator_from_sklearn(estimator): @pytest.mark.parametrize("estimator", _estimators) def test_check_estimator_from_feature_engine(estimator): return check_feature_engine_estimator(estimator) + + +@pytest.mark.parametrize( + # Key to all: - "non-standard" index that is not the usual + # contiguous range starting a t 0 + "encoder, df_test, df_expected", + [ + ( + DecisionTreeEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [21, 30, 21, 30, 51, 40]}, + index=[101, 105, 42, 76, 88, 92], + ), + pd.DataFrame( + {"0": [25.5, 25.5, 25.5, 25.5, 45.5, 45.5]}, + ), + ), + ( + MeanEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + pd.DataFrame( + {"0": [0.5, 0.5, 0.5, 0.5, 0.5, 0.5]}, + ), + ), + ( + OrdinalEncoder(encoding_method="ordered"), + pd.DataFrame( + { + "x": ["a", "a", "a", "b", "b", "b", "c", "c", "c"], + "y": [3, 3, 3, 2, 2, 2, 1, 1, 1], + }, + index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1], + ), + pd.DataFrame({"0": [2, 2, 2, 1, 1, 1, 0, 0, 0]}), + ), + ( + PRatioEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + pd.DataFrame( + {"0": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, + ), + ), + ( + WoEEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + pd.DataFrame( + {"0": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, + ), + ), + ], +) +def test_fix_index_mismatch_from_x_numpy_y_pandas(encoder, df_test, df_expected): + """ + Created 2022-03-27 to test fix to issue # 376 + Code adapted from: + https://github.com/scikit-learn-contrib/category_encoders/issues/280 + """ + + X = df_test[["x"]] + y = df_test["y"] + + # Test issue where X is array, + # y remains Series with original index + X_2 = X.to_numpy() + df_result = encoder.fit_transform(X_2, y) + assert df_result.equals(df_expected) + + +@pytest.mark.parametrize( + # Key to all: - "non-standard" index that is not the usual + # contiguous range starting a t 0 + "encoder, df_test, df_expected", + [ + ( + DecisionTreeEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [21, 30, 21, 30, 51, 40]}, + index=[101, 105, 42, 76, 88, 92], + ), + pd.DataFrame( + {"x": [25.5, 25.5, 25.5, 25.5, 45.5, 45.5]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ( + MeanEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + pd.DataFrame( + {"x": [0.5, 0.5, 0.5, 0.5, 0.5, 0.5]}, index=[101, 105, 42, 76, 88, 92] + ), + ), + ( + OrdinalEncoder(encoding_method="ordered"), + pd.DataFrame( + { + "x": ["a", "a", "a", "b", "b", "b", "c", "c", "c"], + "y": [3, 3, 3, 2, 2, 2, 1, 1, 1], + }, + index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1], + ), + pd.DataFrame( + {"x": [2, 2, 2, 1, 1, 1, 0, 0, 0]}, + index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1], + ), + ), + ( + PRatioEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + pd.DataFrame( + {"x": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, index=[101, 105, 42, 76, 88, 92] + ), + ), + ( + WoEEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + pd.DataFrame( + {"x": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, index=[101, 105, 42, 76, 88, 92] + ), + ), + ], +) +def test_fix_index_mismatch_from_x_pandas_y_numpy(encoder, df_test, df_expected): + """ + Created 2022-03-27 to test fix to issue # 376 + Code adapted from: + https://github.com/scikit-learn-contrib/category_encoders/issues/280 + """ + + X = df_test[["x"]] + y = df_test["y"] + + # Test issue fix where X becomes array, + # y remains Series with original DataFrame index + y_2 = y.to_numpy() + df_result = encoder.fit_transform(X, y_2) + assert df_result.equals(df_expected) + + +@pytest.mark.parametrize( + "encoder, df_test", + [ + ( + DecisionTreeEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [21, 30, 21, 30, 51, 40]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ( + MeanEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ( + OrdinalEncoder(encoding_method="ordered"), + pd.DataFrame( + { + "x": ["a", "a", "a", "b", "b", "b", "c", "c", "c"], + "y": [3, 3, 3, 2, 2, 2, 1, 1, 1], + }, + index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1], + ), + ), + ( + PRatioEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ( + WoEEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ], +) +def test_detect_index_mismatch_from_x_pandas_y_pandas(encoder, df_test): + """ + Created 2022-03-27 to test fix to issue # 376 + """ + + X: pd.DataFrame = df_test[["x"]] + y: pd.Series = df_test["y"] + + # Test issue fix where indexes of pandas objects are mismatched + y = y.reset_index(drop=True) + + e: Exception + with pytest.raises(Exception) as e: + encoder.fit_transform(X, y) + assert "indexes" in e.value.args[0].lower() From ee95e31b012ea1160e119e540725c14b141d3b91 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Wed, 13 Apr 2022 10:43:10 +0200 Subject: [PATCH 06/10] fixes pd.series checks and changes defo param --- feature_engine/dataframe_checks.py | 9 +++++++-- tests/test_dataframe_checks.py | 5 +++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index d60cccb7e..f3f6c7b66 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -89,7 +89,7 @@ def check_X(X: Union[np.generic, np.ndarray, pd.DataFrame]) -> pd.DataFrame: def check_y( y: Union[np.generic, np.ndarray, pd.Series, List], multi_output: bool = False, - y_numeric: bool = True, + y_numeric: bool = False, ) -> pd.Series: """ Checks that y is a series, or alternatively, if it can be converted to a series. @@ -117,7 +117,12 @@ def check_y( raise ValueError("y cannot be None.") elif isinstance(y, pd.Series): - _check_y(y, multi_output=multi_output, y_numeric=y_numeric) + if y.isnull().any(): + raise ValueError("y contains NaN infinity values.") + if y.dtype != "O" and not np.isfinite(y).all(): + raise ValueError("y contains infinity values.") + if y_numeric and y.dtype == "O": + y = y.astype("float") y = y.copy() else: diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 0d61a4c71..8e25ed699 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -72,6 +72,11 @@ def test_check_y_raises_inf_error(): check_y(s) +def test_check_y_converts_string_to_number(): + s = pd.Series(["0", "1", "2", "3", "4"]) + assert_series_equal(check_y(s, y_numeric=True), s.astype("float")) + + def test_check_x_y_returns_pandas(df_vartypes): s = pd.Series([0, 1, 2, 3]) x, y = check_X_y(df_vartypes, s) From 37d0f9adee8301ed19f94dcdab068b483412b951 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Wed, 13 Apr 2022 10:50:19 +0200 Subject: [PATCH 07/10] add adds missing params to docstring --- feature_engine/dataframe_checks.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index f3f6c7b66..62ec7378e 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -136,7 +136,7 @@ def check_X_y( X: Union[np.generic, np.ndarray, pd.DataFrame], y: Union[np.generic, np.ndarray, pd.Series, List], multi_output: bool = False, - y_numeric: bool = True, + y_numeric: bool = False, ) -> Tuple[pd.DataFrame, pd.Series]: """ Ensures X and y are compatible pandas DataFrame and Series. If both are pandas @@ -149,8 +149,19 @@ def check_X_y( Parameters ---------- X: Pandas DataFrame or numpy ndarray + y: Pandas Series or numpy ndarray + multi_output : bool, default=False + Whether to allow 2D y (array). If false, y will be + validated as a vector. y cannot have np.nan or np.inf values if + multi_output=True. + + y_numeric : bool, default=False + Whether to ensure that y has a numeric type. If dtype of y is object, + it is converted to float64. Should only be used for regression + algorithms. + Raises ------ ValueError: if X and y are pandas objects with inconsistent indexes. From 92e87e21dc5d826b541822264c0a15cab7e26dfa Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Wed, 13 Apr 2022 10:55:35 +0200 Subject: [PATCH 08/10] minor word changes --- feature_engine/dataframe_checks.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index 62ec7378e..4bb7d2601 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -97,6 +97,7 @@ def check_y( Parameters ---------- y : pd.Series, np.array, list + The input to check and copy or transform. multi_output : bool, default=False Whether to allow 2D y (array). If false, y will be @@ -118,7 +119,7 @@ def check_y( elif isinstance(y, pd.Series): if y.isnull().any(): - raise ValueError("y contains NaN infinity values.") + raise ValueError("y contains NaN values.") if y.dtype != "O" and not np.isfinite(y).all(): raise ValueError("y contains infinity values.") if y_numeric and y.dtype == "O": @@ -149,8 +150,10 @@ def check_X_y( Parameters ---------- X: Pandas DataFrame or numpy ndarray + The input to check and copy or transform. - y: Pandas Series or numpy ndarray + y: pd.Series, np.array, list + The input to check and copy or transform. multi_output : bool, default=False Whether to allow 2D y (array). If false, y will be From 95a4cc1c8fdf762155153a4378099b8f8fb50beb Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Thu, 14 Apr 2022 06:50:59 +0200 Subject: [PATCH 09/10] renames tests, removes duped test --- tests/test_dataframe_checks.py | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 8e25ed699..2aefd1adc 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -77,14 +77,14 @@ def test_check_y_converts_string_to_number(): assert_series_equal(check_y(s, y_numeric=True), s.astype("float")) -def test_check_x_y_returns_pandas(df_vartypes): +def test_check_x_y_returns_pandas_from_pandas(df_vartypes): s = pd.Series([0, 1, 2, 3]) x, y = check_X_y(df_vartypes, s) assert_frame_equal(df_vartypes, x) assert_series_equal(s, y) -def test_check_X_y_pandas_non_typical_index(): +def test_check_X_y_returns_pandas_from_pandas_with_non_typical_index(): df = pd.DataFrame( {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] ) @@ -94,7 +94,7 @@ def test_check_X_y_pandas_non_typical_index(): assert_series_equal(s, y) -def test_check_X_y_pandas_index_dont_match(): +def test_check_X_y_raises_error_when_pandas_index_dont_match(): df = pd.DataFrame( {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] ) @@ -103,7 +103,7 @@ def test_check_X_y_pandas_index_dont_match(): check_X_y(df, s) -def test_check_x_y_reassings_index(): +def test_check_x_y_reassings_index_when_only_one_input_is_pandas(): # case 1: X is dataframe, y is something else df = pd.DataFrame( {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] @@ -137,26 +137,12 @@ def test_check_x_y_converts_numpy_to_pandas(): assert_series_equal(s, y) -def test_check_x_y_inconsistent_length(df_vartypes): +def test_check_x_y_raises_error_when_inconsistent_length(df_vartypes): s = pd.Series([0, 1, 2, 3, 5]) with pytest.raises(ValueError): check_X_y(df_vartypes, s) -def test_check_x_y_raises_index_mismatch(df_vartypes): - s = pd.Series( - [ - 0, - 1, - 2, - 3, - ], - index=[2, 3, 4, 5], - ) - with pytest.raises(ValueError): - check_X_y(df_vartypes, s) - - def test_check_X_matches_training_df(df_vartypes): with pytest.raises(ValueError): assert _check_X_matches_training_df(df_vartypes, 4) From aef03042449987cc9605a52b6fdb6ff4053f0eee Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Thu, 14 Apr 2022 06:58:02 +0200 Subject: [PATCH 10/10] renames tests encoders --- tests/test_encoding/test_check_estimator_encoders.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index 402535e31..996fb59e5 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -111,7 +111,7 @@ def test_check_estimator_from_feature_engine(estimator): ), ], ) -def test_fix_index_mismatch_from_x_numpy_y_pandas(encoder, df_test, df_expected): +def test_encoders_when_x_numpy_y_pandas(encoder, df_test, df_expected): """ Created 2022-03-27 to test fix to issue # 376 Code adapted from: @@ -190,7 +190,7 @@ def test_fix_index_mismatch_from_x_numpy_y_pandas(encoder, df_test, df_expected) ), ], ) -def test_fix_index_mismatch_from_x_pandas_y_numpy(encoder, df_test, df_expected): +def test_encoders_when_x_pandas_y_numpy(encoder, df_test, df_expected): """ Created 2022-03-27 to test fix to issue # 376 Code adapted from: @@ -250,7 +250,7 @@ def test_fix_index_mismatch_from_x_pandas_y_numpy(encoder, df_test, df_expected) ), ], ) -def test_detect_index_mismatch_from_x_pandas_y_pandas(encoder, df_test): +def test_encoders_raise_error_when_x_pandas_y_pandas_index_mismatch(encoder, df_test): """ Created 2022-03-27 to test fix to issue # 376 """