diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index efb210a06..0f9838965 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -2,12 +2,14 @@ transform(). """ -from typing import List, Union +from typing import List, Tuple, Union import numpy as np import pandas as pd from scipy.sparse import issparse +from .numpy_to_pandas import _is_numpy, _numpy_to_dataframe, _numpy_to_series + def _is_dataframe(X: pd.DataFrame) -> pd.DataFrame: """ @@ -33,9 +35,8 @@ def _is_dataframe(X: pd.DataFrame) -> pd.DataFrame: """ # check_estimator uses numpy arrays for its checks. # Thus, we need to allow np arrays - if isinstance(X, (np.generic, np.ndarray)): - col_names = [str(i) for i in range(X.shape[1])] - X = pd.DataFrame(X, columns=col_names) + if _is_numpy(X): + X = _numpy_to_dataframe(X) if issparse(X): raise ValueError("This transformer does not support sparse matrices.") @@ -129,3 +130,86 @@ def _check_contains_inf(X: pd.DataFrame, variables: List[Union[str, int]]) -> No "Some of the variables to transform contain inf values. Check and " "remove those before using this transformer." ) + + +def _check_pd_X_y( + X: Union[pd.DataFrame, np.ndarray], + y: Union[pd.Series, np.ndarray, list, Tuple], +): + """ + Returns X as a DataFrame and y as a Series, converting any numpy + objects to pandas objects as needed. + * If both parameters are numpy objects, they are converted to pandas objects. + * If one parameter is a pandas object and the other is a numpy object, + the former will be converted to a pandas object, with the indexes + of the latter. + * If both parameters are pandas objects, and their indexes are inconsistent, + an exception is raised (i.e. this is the caller's error.) + * If both parameters are pandas objects and their indexes match, they are + copied and returned. + * If X is sparse or X is empty or, after all transforms, is stiil + not a DataFrame, raises an exception + * Raises an exception if either incoming object is None or empty + + Parameters + ---------- + X: Pandas DataFrame or numpy ndarray + y: Pandas Series or numpy ndarray or list or tuple + + Returns + ------- + X: Pandas DataFrame + y: Pandas Series + + Exceptions + ---------- + ValueError: if X and y are dimension-incompatible, X and y are pandas objects + with inconsistent indexes, or if either X or y is None/empty + """ + # * Raises an exception if either incoming object is None or empty + if X is None or len(X) == 0: + raise ValueError("X cannot be None or empty") + if y is None or len(y) == 0: + raise ValueError("y cannot be None or empty") + + # * If both parameters are numpy objects, they are converted to pandas objects. + # * If one parameter is a pandas object and the other is a numpy object, + # the former will be converted to a pandas object, with the indexes + # of the latter. (Lists and tuples are also supported for y) + if _is_numpy(X): + X = _numpy_to_dataframe(X, index=y.index if isinstance(y, pd.Series) else None) + if _is_numpy(y): + y = _numpy_to_series(y, index=X.index if isinstance(X, pd.DataFrame) else None) + if isinstance(y, (list, Tuple)): + y = pd.Series(y) + y.index = X.index if isinstance(X, pd.DataFrame) else None + + # * If both parameters are pandas objects, and their indexes are inconsistent, + # an exception is raised (i.e. this is the caller's error.) + # * If both parameters are pandas objects and their indexes match, they are + # copied and returned + if isinstance(X, pd.DataFrame) and isinstance(y, pd.Series): + if not all(y.index == X.index): + raise ValueError("Index mismatch between DataFrame X and Series y") + else: + return X.copy(), y.copy() + + # * If X is sparse or X is empty or, after all transforms, is stiil + # not a DataFrame, raises an exception + # (This deliberately carries out similar tests in _is_dataframe() above in + # order to support different code paths) + if issparse(X): + raise ValueError("This transformer does not support sparse matrices.") + + if not isinstance(X, pd.DataFrame): + raise TypeError( + "X is not a pandas dataframe. The dataset should be a pandas dataframe." + ) + + if X.empty: + raise ValueError( + "0 feature(s) (shape=%s) while a minimum of %d is " + "required." % (X.shape, 1) + ) + + return X, y diff --git a/feature_engine/encoding/base_encoder.py b/feature_engine/encoding/base_encoder.py index 2645a56f6..2d57ee60e 100644 --- a/feature_engine/encoding/base_encoder.py +++ b/feature_engine/encoding/base_encoder.py @@ -53,7 +53,7 @@ def __init__( self.variables = _check_input_parameter_variables(variables) self.ignore_format = ignore_format - def _check_fit_input_and_variables(self, X: pd.DataFrame) -> pd.DataFrame: + def _check_fit_input_and_variables(self, X: pd.DataFrame): """ Checks that input is a dataframe, finds categorical variables, or alternatively checks that the variables entered by the user are of type object (categorical). @@ -71,18 +71,8 @@ def _check_fit_input_and_variables(self, X: pd.DataFrame) -> pd.DataFrame: ValueError If there are no categorical variables in the df or the df is empty If the variable(s) contain null values - - Returns - ------- - X: Pandas DataFrame - The same dataframe entered as parameter - variables : list - list of categorical variables """ - # check input dataframe - X = _is_dataframe(X) - if not self.ignore_format: # find categorical variables or check variables entered by user are object self.variables_: List[ @@ -101,7 +91,7 @@ def _check_fit_input_and_variables(self, X: pd.DataFrame) -> pd.DataFrame: # save train set shape self.n_features_in_ = X.shape[1] - return X + return self def _check_transform_input_and_state(self, X: pd.DataFrame) -> pd.DataFrame: """ diff --git a/feature_engine/encoding/count_frequency.py b/feature_engine/encoding/count_frequency.py index 420b97b40..31c9d9fcb 100644 --- a/feature_engine/encoding/count_frequency.py +++ b/feature_engine/encoding/count_frequency.py @@ -5,6 +5,7 @@ import pandas as pd +from feature_engine.dataframe_checks import _is_dataframe from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -139,7 +140,8 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): y is not needed in this encoder. You can pass y or None. """ - X = self._check_fit_input_and_variables(X) + X = _is_dataframe(X) + self._check_fit_input_and_variables(X) self.encoder_dict_ = {} diff --git a/feature_engine/encoding/decision_tree.py b/feature_engine/encoding/decision_tree.py index d728a9535..db7d8bf33 100644 --- a/feature_engine/encoding/decision_tree.py +++ b/feature_engine/encoding/decision_tree.py @@ -7,6 +7,7 @@ from sklearn.pipeline import Pipeline from sklearn.utils.multiclass import check_classification_targets, type_of_target +from feature_engine.dataframe_checks import _check_pd_X_y from feature_engine.discretisation import DecisionTreeDiscretiser from feature_engine.docstrings import ( Substitution, @@ -202,7 +203,8 @@ def fit(self, X: pd.DataFrame, y: pd.Series): check_classification_targets(y) # check input dataframe - X = self._check_fit_input_and_variables(X) + X, y = _check_pd_X_y(X, y) + self._check_fit_input_and_variables(X) if self.param_grid: param_grid = self.param_grid diff --git a/feature_engine/encoding/mean_encoding.py b/feature_engine/encoding/mean_encoding.py index 9a69da7ca..741291b41 100644 --- a/feature_engine/encoding/mean_encoding.py +++ b/feature_engine/encoding/mean_encoding.py @@ -5,6 +5,7 @@ import pandas as pd +from feature_engine.dataframe_checks import _check_pd_X_y from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -131,10 +132,8 @@ def fit(self, X: pd.DataFrame, y: pd.Series): The target. """ - X = self._check_fit_input_and_variables(X) - - if not isinstance(y, pd.Series): - y = pd.Series(y) + X, y = _check_pd_X_y(X, y) + self._check_fit_input_and_variables(X) temp = pd.concat([X, y], axis=1) temp.columns = list(X.columns) + ["target"] diff --git a/feature_engine/encoding/one_hot.py b/feature_engine/encoding/one_hot.py index 9da3dde30..71426cba2 100644 --- a/feature_engine/encoding/one_hot.py +++ b/feature_engine/encoding/one_hot.py @@ -7,6 +7,7 @@ import pandas as pd from sklearn.utils.validation import check_is_fitted +from feature_engine.dataframe_checks import _is_dataframe from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -180,7 +181,8 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): None. """ - X = self._check_fit_input_and_variables(X) + X = _is_dataframe(X) + self._check_fit_input_and_variables(X) self.encoder_dict_ = {} diff --git a/feature_engine/encoding/ordinal.py b/feature_engine/encoding/ordinal.py index cca752a8d..36223e407 100644 --- a/feature_engine/encoding/ordinal.py +++ b/feature_engine/encoding/ordinal.py @@ -5,6 +5,7 @@ import pandas as pd +from feature_engine.dataframe_checks import _check_pd_X_y, _is_dataframe from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -146,16 +147,18 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): Otherwise, y needs to be passed when fitting the transformer. """ - X = self._check_fit_input_and_variables(X) + # All dimension, type, etc. checking + if self.encoding_method == "ordered": + X, y = _check_pd_X_y(X, y) + else: + X = _is_dataframe(X) + self._check_fit_input_and_variables(X) # join target to predictor variables if self.encoding_method == "ordered": if y is None: raise ValueError("Please provide a target y for this encoding method") - if not isinstance(y, pd.Series): - y = pd.Series(y) - temp = pd.concat([X, y], axis=1) temp.columns = list(X.columns) + ["target"] diff --git a/feature_engine/encoding/probability_ratio.py b/feature_engine/encoding/probability_ratio.py index 6caaa83ed..a3affb30a 100644 --- a/feature_engine/encoding/probability_ratio.py +++ b/feature_engine/encoding/probability_ratio.py @@ -6,6 +6,7 @@ import numpy as np import pandas as pd +from feature_engine.dataframe_checks import _check_pd_X_y from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -154,10 +155,8 @@ def fit(self, X: pd.DataFrame, y: pd.Series): Target, must be binary. """ - X = self._check_fit_input_and_variables(X) - - if not isinstance(y, pd.Series): - y = pd.Series(y) + X, y = _check_pd_X_y(X, y) + self._check_fit_input_and_variables(X) # check that y is binary if y.nunique() != 2: diff --git a/feature_engine/encoding/rare_label.py b/feature_engine/encoding/rare_label.py index e891398b9..4b14adbff 100644 --- a/feature_engine/encoding/rare_label.py +++ b/feature_engine/encoding/rare_label.py @@ -7,6 +7,7 @@ import numpy as np import pandas as pd +from feature_engine.dataframe_checks import _is_dataframe from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -147,7 +148,8 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): y is not required. You can pass y or None. """ - X = self._check_fit_input_and_variables(X) + X = _is_dataframe(X) + self._check_fit_input_and_variables(X) self.encoder_dict_ = {} diff --git a/feature_engine/encoding/woe.py b/feature_engine/encoding/woe.py index e996bb7d6..d07bd4dbd 100644 --- a/feature_engine/encoding/woe.py +++ b/feature_engine/encoding/woe.py @@ -6,6 +6,7 @@ import numpy as np import pandas as pd +from feature_engine.dataframe_checks import _check_pd_X_y from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -136,10 +137,8 @@ def fit(self, X: pd.DataFrame, y: pd.Series): Target, must be binary. """ - X = self._check_fit_input_and_variables(X) - - if not isinstance(y, pd.Series): - y = pd.Series(y) + X, y = _check_pd_X_y(X, y) + self._check_fit_input_and_variables(X) # check that y is binary if y.nunique() != 2: diff --git a/feature_engine/numpy_to_pandas.py b/feature_engine/numpy_to_pandas.py new file mode 100644 index 000000000..afefd20a9 --- /dev/null +++ b/feature_engine/numpy_to_pandas.py @@ -0,0 +1,62 @@ +"""Functions to detect numpy objects and convert to pandas objects.""" + +from typing import Any, List, Union + +import numpy as np +import pandas as pd + + +def _is_numpy(obj_in: Any) -> bool: + """ + Tests if an object is a numpy object. + If the input is a numpy array, it converts it to a pandas Dataframe. This is mostly + so that we can add the check_estimator checks for compatibility with sklearn. + + Parameters + ---------- + obj_in : the object to test. + + Returns + ------- + True if object is a numpy object, else False + """ + return isinstance(obj_in, (np.generic, np.ndarray)) + + +def _numpy_to_dataframe( + obj_in: Union[np.generic, np.ndarray], index=None +) -> pd.DataFrame: + """ + Converts a numpy object to a pandas DataFrame + + Parameters + ---------- + obj_in : the object to convert + index : array-like (optional); will set index on DataFrame + + Returns + ------- + df_out : the object converted to a pandas DataFrame + """ + col_names: List[str] = [str(i) for i in range(obj_in.shape[1])] + df_out: pd.DataFrame = pd.DataFrame(obj_in, columns=col_names, index=index) + + return df_out + + +def _numpy_to_series(obj_in: Union[np.generic, np.ndarray], index=None) -> pd.Series: + """ + Converts a numpy object to a pandas Series + + Parameters + ---------- + obj_in : the object to convert + index : array-like (optional); will set index on Series + + Returns + ------- + df_out : the object converted to a pandas Series + """ + s_out: pd.Series = pd.Series(obj_in, index=index) + + return s_out diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 396cefcdd..0cc9f3cb5 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -1,9 +1,12 @@ +import numpy as np +import pandas as pd import pytest -from pandas.testing import assert_frame_equal +from pandas.testing import assert_frame_equal, assert_series_equal from feature_engine.dataframe_checks import ( _check_contains_na, _check_input_matches_training_df, + _check_pd_X_y, _is_dataframe, ) @@ -22,3 +25,202 @@ def test_check_input_matches_training_df(df_vartypes): def test_contains_na(df_na): with pytest.raises(ValueError): assert _check_contains_na(df_na, ["Name", "City"]) + + +@pytest.mark.parametrize( + "X_in, y_in, expected_1, expected_2", + [ + # * If both parameters are numpy objects, + # they are converted to pandas objects. + ( + np.array([[1, 2, 3, 4], [5, 6, 7, 8]]).T, + np.array([1, 2, 3, 4]), + pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}), + pd.Series([1, 2, 3, 4]), + ), + ], +) +def test_check_pd_X_y_both_numpy(X_in, y_in, expected_1, expected_2): + # Execute + X_out, y_out = _check_pd_X_y(X_in, y_in) + + # Test X output + if expected_1 is None: + assert X_out is X_in + elif isinstance(expected_1, pd.DataFrame): + assert_frame_equal(X_out, expected_1) + elif isinstance(expected_1, (np.generic, np.ndarray)): + assert all(X_out == expected_1) + + # Test y output + if expected_2 is None: + assert y_out is y_in + elif isinstance(expected_2, pd.Series): + assert_series_equal(y_out, expected_2) + elif isinstance(expected_2, (np.generic, np.ndarray)): + assert all(y_out == expected_2) + + +@pytest.mark.parametrize( + "X_in, y_in, expected_1, expected_2", + [ + # * If both parameters are pandas objects and their indexes match, they are + # copied and returned. + ( + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), + ), + ], +) +def test_check_pd_X_y_both_pandas(X_in, y_in, expected_1, expected_2): + # Execute + X_out, y_out = _check_pd_X_y(X_in, y_in) + + # Test X output + if isinstance(expected_1, pd.DataFrame): + assert_frame_equal(X_out, expected_1) + assert X_out is not expected_1 # make sure copied + elif isinstance(expected_1, (np.generic, np.ndarray)): + assert all(X_out == expected_1) + + # Test y output + if isinstance(expected_2, pd.Series): + assert_series_equal(y_out, expected_2) + assert y_out is not expected_2 # make sure copied + elif isinstance(expected_2, (np.generic, np.ndarray)): + assert all(y_out == expected_2) + + +@pytest.mark.parametrize( + "X_in, y_in, expected_1, expected_2", + [ + # * If one parameter is a numpy object and the + # other is a pandas object, the former will be + # converted to a pandas object, with the indexes + # of the latter. + ( + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + np.array([1, 2, 3, 4]), + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), + ), + ( + np.array([[1, 2, 3, 4], [5, 6, 7, 8]]).T, + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), + ), + # (Lists and tuples are also supported for y) + ( + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + [1, 2, 3, 4], + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), + ), + ( + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + (1, 2, 3, 4), + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), + ), + ], +) +def test_check_pd_X_y_np_to_pd(X_in, y_in, expected_1, expected_2): + # Execute + X_out, y_out = _check_pd_X_y(X_in, y_in) + + # Test X output + if expected_1 is None: + assert X_out is X_in + elif isinstance(expected_1, pd.DataFrame): + assert_frame_equal(X_out, expected_1) + elif isinstance(expected_1, (np.generic, np.ndarray)): + assert all(X_out == expected_1) + + # Test y output + if expected_2 is None: + assert y_out is y_in + elif isinstance(expected_2, pd.Series): + assert_series_equal(y_out, expected_2) + elif isinstance(expected_2, (np.generic, np.ndarray)): + assert all(y_out == expected_2) + + +@pytest.mark.parametrize( + "X_in, y_in, exception_type, exception_match", + [ + # * If both parameters are pandas objects, and their + # indexes are inconsistent, an exception is raised + # (i.e.this is the caller's error.) + ( + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 999]), + ValueError, + ".*Index.*", + ), + # * Raises an exception if either incoming object is None or empty + ( + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + None, + ValueError, + ".*None.*empty.*", + ), + ( + None, + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), + ValueError, + ".*None.*empty.*", + ), + ( + pd.DataFrame(), + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), + ValueError, + ".*None.*empty.*", + ), + ( + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + pd.Series(), + ValueError, + ".*None.*empty.*", + ), + # Show that incompatible dimensions causes same error + ( + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + pd.Series([1, 2, 3], index=[22, 99, 101]), + ValueError, + ".*Lengths.*", + ), + ], +) +def test_check_pd_X_y_errors(X_in, y_in, exception_type, exception_match): + with (pytest.raises(exception_type, match=exception_match)): + # Execute - can throw here (non-null exception_type will expect exception) + X_out, y_out = _check_pd_X_y(X_in, y_in) diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index 9600d36d9..16f70bbfb 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -1,3 +1,5 @@ +import numpy as np +import pandas as pd import pytest from sklearn.utils.estimator_checks import check_estimator @@ -50,3 +52,231 @@ def test_check_estimator_from_sklearn(estimator): @pytest.mark.parametrize("estimator", _estimators) def test_check_estimator_from_feature_engine(estimator): return check_feature_engine_estimator(estimator) + + +@pytest.mark.parametrize( + # Encoders that encode X as a function of y; this is what + # breaks down when X becomes an array and indexes don't + # accidentally match in final concantenation + # All test DataFrames have same data except DecisionTreeEncoder(), + # which needs different y values. + # Key to all: - "non-standard" index that is not the usual + # contiguous range starting a t 0 + "encoder, df_test, df_expected", + [ + ( + DecisionTreeEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [21, 30, 21, 30, 51, 40]}, + index=[101, 105, 42, 76, 88, 92], + ), + pd.DataFrame( + {"0": [25.5, 25.5, 25.5, 25.5, 45.5, 45.5]}, + ), + ), + ( + MeanEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + pd.DataFrame( + {"0": [0.5, 0.5, 0.5, 0.5, 0.5, 0.5]}, + ), + ), + ( + OrdinalEncoder(encoding_method="ordered"), + pd.DataFrame( + { + "x": ["a", "a", "a", "b", "b", "b", "c", "c", "c"], + "y": [3, 3, 3, 2, 2, 2, 1, 1, 1], + }, + index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1], + ), + pd.DataFrame({"0": [2, 2, 2, 1, 1, 1, 0, 0, 0]}), + ), + ( + PRatioEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + pd.DataFrame( + {"0": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, + ), + ), + ( + WoEEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + pd.DataFrame( + {"0": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, + ), + ), + ], +) +def test_fix_index_mismatch_from_x_numpy_y_pandas(encoder, df_test, df_expected): + """ + Created 2022-03-27 to test fix to issue # 376 + Code adapted from: + https://github.com/scikit-learn-contrib/category_encoders/issues/280 + """ + + # Set up for standard pipeline/training etc. + X: pd.DataFrame = df_test[["x"]] + y: pd.Series = df_test["y"] + + # Test issue fix where X becomes array, + # y remains Series with original DataFrame index + X_2: np.ndarray = X.to_numpy() + df_result: pd.DataFrame = encoder.fit_transform(X_2, y) + assert df_result.equals(df_expected) + + +@pytest.mark.parametrize( + # Encoders that encode X as a function of y; this is what + # breaks down when y becomes an array and indexes don't + # accidentally match in final concantenation + # All test DataFrames have same data except DecisionTreeEncoder(), + # which needs different y values. + # Key to all: - "non-standard" index that is not the usual + # contiguous range starting a t 0 + "encoder, df_test, df_expected", + [ + ( + DecisionTreeEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [21, 30, 21, 30, 51, 40]}, + index=[101, 105, 42, 76, 88, 92], + ), + pd.DataFrame( + {"x": [25.5, 25.5, 25.5, 25.5, 45.5, 45.5]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ( + MeanEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + pd.DataFrame( + {"x": [0.5, 0.5, 0.5, 0.5, 0.5, 0.5]}, index=[101, 105, 42, 76, 88, 92] + ), + ), + ( + OrdinalEncoder(encoding_method="ordered"), + pd.DataFrame( + { + "x": ["a", "a", "a", "b", "b", "b", "c", "c", "c"], + "y": [3, 3, 3, 2, 2, 2, 1, 1, 1], + }, + index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1], + ), + pd.DataFrame( + {"x": [2, 2, 2, 1, 1, 1, 0, 0, 0]}, + index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1], + ), + ), + ( + PRatioEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + pd.DataFrame( + {"x": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, index=[101, 105, 42, 76, 88, 92] + ), + ), + ( + WoEEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + pd.DataFrame( + {"x": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, index=[101, 105, 42, 76, 88, 92] + ), + ), + ], +) +def test_fix_index_mismatch_from_x_pandas_y_numpy(encoder, df_test, df_expected): + """ + Created 2022-03-27 to test fix to issue # 376 + Code adapted from: + https://github.com/scikit-learn-contrib/category_encoders/issues/280 + """ + + # Set up for standard pipeline/training etc. + X: pd.DataFrame = df_test[["x"]] + y: pd.Series = df_test["y"] + + # Test issue fix where X becomes array, + # y remains Series with original DataFrame index + y_2: np.ndarray = y.to_numpy() + df_result: pd.DataFrame = encoder.fit_transform(X, y_2) + assert df_result.equals(df_expected) + + +@pytest.mark.parametrize( + "encoder, df_test", + [ + ( + DecisionTreeEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [21, 30, 21, 30, 51, 40]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ( + MeanEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ( + OrdinalEncoder(encoding_method="ordered"), + pd.DataFrame( + { + "x": ["a", "a", "a", "b", "b", "b", "c", "c", "c"], + "y": [3, 3, 3, 2, 2, 2, 1, 1, 1], + }, + index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1], + ), + ), + ( + PRatioEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ( + WoEEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ], +) +def test_detect_index_mismatch_from_x_pandas_y_pandas(encoder, df_test): + """ + Created 2022-03-27 to test fix to issue # 376 + """ + + # Set up for standard pipeline/training etc. + X: pd.DataFrame = df_test[["x"]] + y: pd.Series = df_test["y"] + + # Test issue fix where indexes of pandas objects become mismatched + # y remains Series with original DataFrame index + y = y.reset_index(drop=True) + + e: Exception + with pytest.raises(Exception) as e: + encoder.fit_transform(X, y) + assert "mismatch" in e.value.args[0].lower() diff --git a/tests/test_numpy_to_pandas.py b/tests/test_numpy_to_pandas.py new file mode 100644 index 000000000..02e7959c6 --- /dev/null +++ b/tests/test_numpy_to_pandas.py @@ -0,0 +1,44 @@ +from typing import Any +import numpy as np +import pandas as pd +import pytest +from pandas.testing import assert_frame_equal, assert_series_equal + + +from feature_engine.numpy_to_pandas import ( + _is_numpy, + _numpy_to_series, + _numpy_to_dataframe +) + + +@pytest.mark.parametrize( + "obj, expected", + [ + (np.array([1, 2, 3, 4]), True), + (pd.Series([1, 2, 3, 4]), False), + ("something", False) + ] +) +def test_is_numpy(obj: Any, expected: bool): + assert _is_numpy(obj) == expected + + +def test_numpy_to_dataframe(): + np_array: np.ndarray = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) + expected: pd.DataFrame = pd.DataFrame( + {"0": [1, 4, 7], "1": [2, 5, 8], "2": [3, 6, 9]} + ) + assert_frame_equal(_numpy_to_dataframe(np_array), expected) + + expected.index = ["a", "b", "c"] + assert_frame_equal(_numpy_to_dataframe(np_array, index=["a", "b", "c"]), expected) + + +def test_numpy_to_series(): + np_array: np.ndarray = np.array([1, 2, 3]) + expected: pd.Series = pd.Series([1, 2, 3]) + assert_series_equal(_numpy_to_series(np_array), expected) + + expected.index = ["a", "b", "c"] + assert_series_equal(_numpy_to_series(np_array, index=["a", "b", "c"]), expected)