From 4118156c949e8b34f82c5b3b8aaed461ae365504 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sat, 19 Feb 2022 09:59:50 -0500 Subject: [PATCH 01/55] created unit test reproducing issue 376; all test failing now until fix is done. Made it a separate .py file because it affects multiple encoders; parameterized it for each encoder with the known issue --- ..._fix_index_mismatch_from_upstream_array.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 tests/test_encoding/test_fix_index_mismatch_from_upstream_array.py diff --git a/tests/test_encoding/test_fix_index_mismatch_from_upstream_array.py b/tests/test_encoding/test_fix_index_mismatch_from_upstream_array.py new file mode 100644 index 000000000..60d84dda4 --- /dev/null +++ b/tests/test_encoding/test_fix_index_mismatch_from_upstream_array.py @@ -0,0 +1,56 @@ +import numpy as np +import pandas as pd +import pytest +from sklearn.exceptions import NotFittedError + +from feature_engine.encoding import ( + MeanEncoder, + WoEEncoder, + PRatioEncoder, +) + +from sklearn.impute import SimpleImputer + + +@pytest.mark.parametrize( + # Encoders that encode X as a function of y; this is what + # breaks down when X becomes an array and indexes don't accidentally match in final + # concantenation + "encoder", [MeanEncoder(), WoEEncoder(), PRatioEncoder()] +) +def test_fix_index_mismatch_from_upstream_array(encoder): + """ + Created 2022-02-19 to test fix to issue # 376 + Code adapted from: https://github.com/scikit-learn-contrib/category_encoders/issues/280 + """ + + # test dataframe; setup for a transfromation where + # coded version of 'x' will be a function of target 'y' + df: pd.DataFrame = pd.DataFrame({ + 'x': ['a', 'a', 'b', 'b', 'c', 'c'], + 'y': [1, 0, 1, 0, 1, 0], + }) + # Key - "non-standard" index that is not the usual + # contiguous range starting a t 0 + df.index = [101, 105, 42, 76, 88, 92] + + # Set up for standard pipeline/training etc. + X: pd.DataFrame = df[["x"]] + y: pd.Series = df["y"] + + # Will serve as a no-op whose chief purpose is to turn the + # X into an np.ndarray + si = SimpleImputer(strategy="constant", fill_value="a") + + # Sequence leading to issue: + # 1) X becomes an array + assert type(X) == pd.DataFrame + X_2: np.array = si.fit_transform(X) + assert type(X_2) == np.ndarray + + # 2) Encoder encodes as function of X, y + df_result: pd.DataFrame = encoder.fit_transform(X_2, y) + assert type(df_result) == pd.DataFrame + + # Assertion fails: breakdown in index matches causes results to be all nan + assert all(df_result.iloc[:, 0].notnull()) \ No newline at end of file From 0d5e6257f40c7fde7e74fb54113d0dec8b6e8262 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sat, 19 Feb 2022 11:13:04 -0500 Subject: [PATCH 02/55] added _check_for_X_y_index_mismatch() to dataframe_checks.py. Even though this function addresses issues that so far only pertain to some BaseEncoder subclasses, am putting it here as it may be useful for other situations --- feature_engine/dataframe_checks.py | 37 ++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index efb210a06..4899bc39d 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -129,3 +129,40 @@ def _check_contains_inf(X: pd.DataFrame, variables: List[Union[str, int]]) -> No "Some of the variables to transform contain inf values. Check and " "remove those before using this transformer." ) + + +def _check_for_X_y_index_mismatch( + X: Union[pd.DataFrame, pd.Series, np.ndarray], + y: Union[pd.DataFrame, pd.Series, np.ndarray] +): + """ + Handles the following case: + 1) X and y came from a DataFrame whose index was not the standard contiguous 0-n + 2) Earlier on, X was affected by an sklearn transform, turning it into an array and losing its different index + 3) X enters a feature-engine transformer and becomes a DataFrame again via _is_dataframe() + 4) X and y now have different indexes + This function checks for this case; if it is met, it will return a copy of X with its index replaced with the + correct index from y. It will not make any changes if either X or y is not a pandas object, or of course if there + is no index mismatch. + This case was first detected in issue #376. + + Parameters + ---------- + X: Pandas DataFrame, Series, or numpy ndarray + In all likelihood will be DataFrame, due to this method usually being called with the + output of _is_dataframe(). Will not make any changes if X is not a DataFrame or Series. + y: Pandas DataFrame, Series, or numpy ndarray + In all likelihood will be a Series. Will not make any changes if X is not a DataFrame or Series. + + Returns + ------- + X: same type as parameter X. + If the mismatch conditions described above have been met, returns a copy of + the original parameter, with index set to y's index. Else, returns original parameter value unaffected. + """ + is_pd_X_and_y: bool = all([(type(i) == pd.DataFrame or type(i) == pd.Series) for i in (X, y)]) + if is_pd_X_and_y and any(X.index != y.index): + X = X.copy() + X.index = y.index + + return X \ No newline at end of file From 9ef98e7d1231460669cf5088e4d05329832af8fb Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sat, 19 Feb 2022 11:28:14 -0500 Subject: [PATCH 03/55] calls to _check_for_X_y_index_mismatch(X, y) added to MeanEncoder, WoEEncoder, and PRatioEncoder, fixing the issue. All unit tests pass. --- feature_engine/encoding/mean_encoding.py | 2 ++ feature_engine/encoding/probability_ratio.py | 2 ++ feature_engine/encoding/woe.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/feature_engine/encoding/mean_encoding.py b/feature_engine/encoding/mean_encoding.py index 9a69da7ca..6dad49448 100644 --- a/feature_engine/encoding/mean_encoding.py +++ b/feature_engine/encoding/mean_encoding.py @@ -5,6 +5,7 @@ import pandas as pd +from feature_engine.dataframe_checks import _check_for_X_y_index_mismatch from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -132,6 +133,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): """ X = self._check_fit_input_and_variables(X) + X = _check_for_X_y_index_mismatch(X, y) if not isinstance(y, pd.Series): y = pd.Series(y) diff --git a/feature_engine/encoding/probability_ratio.py b/feature_engine/encoding/probability_ratio.py index 6caaa83ed..305d81b90 100644 --- a/feature_engine/encoding/probability_ratio.py +++ b/feature_engine/encoding/probability_ratio.py @@ -6,6 +6,7 @@ import numpy as np import pandas as pd +from feature_engine.dataframe_checks import _check_for_X_y_index_mismatch from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -155,6 +156,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): """ X = self._check_fit_input_and_variables(X) + X = _check_for_X_y_index_mismatch(X, y) if not isinstance(y, pd.Series): y = pd.Series(y) diff --git a/feature_engine/encoding/woe.py b/feature_engine/encoding/woe.py index e996bb7d6..a944eec9b 100644 --- a/feature_engine/encoding/woe.py +++ b/feature_engine/encoding/woe.py @@ -6,6 +6,7 @@ import numpy as np import pandas as pd +from feature_engine.dataframe_checks import _check_for_X_y_index_mismatch from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -137,6 +138,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): """ X = self._check_fit_input_and_variables(X) + X = _check_for_X_y_index_mismatch(X, y) if not isinstance(y, pd.Series): y = pd.Series(y) From 07cf7c9037aa510bda8603d9c4c20a315aa9e8b5 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sat, 19 Feb 2022 11:52:08 -0500 Subject: [PATCH 04/55] black/isort formatting --- feature_engine/dataframe_checks.py | 10 ++++---- ..._fix_index_mismatch_from_upstream_array.py | 24 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index 4899bc39d..b7977b12f 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -132,8 +132,8 @@ def _check_contains_inf(X: pd.DataFrame, variables: List[Union[str, int]]) -> No def _check_for_X_y_index_mismatch( - X: Union[pd.DataFrame, pd.Series, np.ndarray], - y: Union[pd.DataFrame, pd.Series, np.ndarray] + X: Union[pd.DataFrame, pd.Series, np.ndarray], + y: Union[pd.DataFrame, pd.Series, np.ndarray], ): """ Handles the following case: @@ -160,9 +160,11 @@ def _check_for_X_y_index_mismatch( If the mismatch conditions described above have been met, returns a copy of the original parameter, with index set to y's index. Else, returns original parameter value unaffected. """ - is_pd_X_and_y: bool = all([(type(i) == pd.DataFrame or type(i) == pd.Series) for i in (X, y)]) + is_pd_X_and_y: bool = all( + [(type(i) == pd.DataFrame or type(i) == pd.Series) for i in (X, y)] + ) if is_pd_X_and_y and any(X.index != y.index): X = X.copy() X.index = y.index - return X \ No newline at end of file + return X diff --git a/tests/test_encoding/test_fix_index_mismatch_from_upstream_array.py b/tests/test_encoding/test_fix_index_mismatch_from_upstream_array.py index 60d84dda4..55da9a38c 100644 --- a/tests/test_encoding/test_fix_index_mismatch_from_upstream_array.py +++ b/tests/test_encoding/test_fix_index_mismatch_from_upstream_array.py @@ -2,21 +2,17 @@ import pandas as pd import pytest from sklearn.exceptions import NotFittedError - -from feature_engine.encoding import ( - MeanEncoder, - WoEEncoder, - PRatioEncoder, -) - from sklearn.impute import SimpleImputer +from feature_engine.encoding import MeanEncoder, PRatioEncoder, WoEEncoder + @pytest.mark.parametrize( # Encoders that encode X as a function of y; this is what # breaks down when X becomes an array and indexes don't accidentally match in final # concantenation - "encoder", [MeanEncoder(), WoEEncoder(), PRatioEncoder()] + "encoder", + [MeanEncoder(), WoEEncoder(), PRatioEncoder()], ) def test_fix_index_mismatch_from_upstream_array(encoder): """ @@ -26,10 +22,12 @@ def test_fix_index_mismatch_from_upstream_array(encoder): # test dataframe; setup for a transfromation where # coded version of 'x' will be a function of target 'y' - df: pd.DataFrame = pd.DataFrame({ - 'x': ['a', 'a', 'b', 'b', 'c', 'c'], - 'y': [1, 0, 1, 0, 1, 0], - }) + df: pd.DataFrame = pd.DataFrame( + { + "x": ["a", "a", "b", "b", "c", "c"], + "y": [1, 0, 1, 0, 1, 0], + } + ) # Key - "non-standard" index that is not the usual # contiguous range starting a t 0 df.index = [101, 105, 42, 76, 88, 92] @@ -53,4 +51,4 @@ def test_fix_index_mismatch_from_upstream_array(encoder): assert type(df_result) == pd.DataFrame # Assertion fails: breakdown in index matches causes results to be all nan - assert all(df_result.iloc[:, 0].notnull()) \ No newline at end of file + assert all(df_result.iloc[:, 0].notnull()) From a41b6eea7ac24617bd4bc06f579abb8e7389dd78 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sat, 19 Feb 2022 11:57:50 -0500 Subject: [PATCH 05/55] additional formatting fixes needed for CI --- feature_engine/dataframe_checks.py | 24 ++++++++++++------- ..._fix_index_mismatch_from_upstream_array.py | 4 ++-- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index b7977b12f..fb9683101 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -138,27 +138,33 @@ def _check_for_X_y_index_mismatch( """ Handles the following case: 1) X and y came from a DataFrame whose index was not the standard contiguous 0-n - 2) Earlier on, X was affected by an sklearn transform, turning it into an array and losing its different index - 3) X enters a feature-engine transformer and becomes a DataFrame again via _is_dataframe() + 2) Earlier on, X was affected by an sklearn transform, turning + it into an array and losing its different index + 3) X enters a feature-engine transformer and becomes a + DataFrame again via _is_dataframe() 4) X and y now have different indexes - This function checks for this case; if it is met, it will return a copy of X with its index replaced with the - correct index from y. It will not make any changes if either X or y is not a pandas object, or of course if there - is no index mismatch. + This function checks for this case; if it is met, it will return + a copy of X with its index replaced with the + correct index from y. It will not make any changes if either X or + y is not a pandas object, or of course if there is no index mismatch. This case was first detected in issue #376. Parameters ---------- X: Pandas DataFrame, Series, or numpy ndarray - In all likelihood will be DataFrame, due to this method usually being called with the - output of _is_dataframe(). Will not make any changes if X is not a DataFrame or Series. + In all likelihood will be DataFrame, due to this method usually + being called with the output of _is_dataframe(). + Will not make any changes if X is not a DataFrame or Series. y: Pandas DataFrame, Series, or numpy ndarray - In all likelihood will be a Series. Will not make any changes if X is not a DataFrame or Series. + In all likelihood will be a Series. + Will not make any changes if X is not a DataFrame or Series. Returns ------- X: same type as parameter X. If the mismatch conditions described above have been met, returns a copy of - the original parameter, with index set to y's index. Else, returns original parameter value unaffected. + the original parameter, with index set to y's index. + Else, returns original parameter value unaffected. """ is_pd_X_and_y: bool = all( [(type(i) == pd.DataFrame or type(i) == pd.Series) for i in (X, y)] diff --git a/tests/test_encoding/test_fix_index_mismatch_from_upstream_array.py b/tests/test_encoding/test_fix_index_mismatch_from_upstream_array.py index 55da9a38c..7d5276b14 100644 --- a/tests/test_encoding/test_fix_index_mismatch_from_upstream_array.py +++ b/tests/test_encoding/test_fix_index_mismatch_from_upstream_array.py @@ -1,7 +1,6 @@ import numpy as np import pandas as pd import pytest -from sklearn.exceptions import NotFittedError from sklearn.impute import SimpleImputer from feature_engine.encoding import MeanEncoder, PRatioEncoder, WoEEncoder @@ -17,7 +16,8 @@ def test_fix_index_mismatch_from_upstream_array(encoder): """ Created 2022-02-19 to test fix to issue # 376 - Code adapted from: https://github.com/scikit-learn-contrib/category_encoders/issues/280 + Code adapted from: + https://github.com/scikit-learn-contrib/category_encoders/issues/280 """ # test dataframe; setup for a transfromation where From 75563091f3759bec6a94d6987414f5e2c7d8f89c Mon Sep 17 00:00:00 2001 From: Noah Green Date: Mon, 21 Feb 2022 22:25:57 -0500 Subject: [PATCH 06/55] moved test_fix_index_mismatch_from_upstream_array() out of separate file and into test_check_estimator_encoders.py where there are other tests of multiple encoders --- .../test_check_estimator_encoders.py | 82 +++++++++++++++---- ..._fix_index_mismatch_from_upstream_array.py | 54 ------------ 2 files changed, 66 insertions(+), 70 deletions(-) delete mode 100644 tests/test_encoding/test_fix_index_mismatch_from_upstream_array.py diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index 9600d36d9..096aa3c5d 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -1,4 +1,7 @@ import pytest +import pandas as pd +import numpy as np +from sklearn.impute import SimpleImputer from sklearn.utils.estimator_checks import check_estimator from feature_engine.encoding import ( @@ -29,24 +32,71 @@ PRatioEncoder(ignore_format=True), ] +@pytest.mark.parametrize( + "Estimator", + [ + CountFrequencyEncoder(ignore_format=True), + DecisionTreeEncoder(regression=False, ignore_format=True), + MeanEncoder(ignore_format=True), + OneHotEncoder(ignore_format=True), + OrdinalEncoder(ignore_format=True), + RareLabelEncoder( + tol=0.00000000001, + n_categories=100000000000, + replace_with=10, + ignore_format=True, + ), + WoEEncoder(ignore_format=True), + PRatioEncoder(ignore_format=True), + ], +) +def test_all_transformers(Estimator): + return check_estimator(Estimator) -@pytest.mark.parametrize("estimator", _estimators) -def test_check_estimator_from_sklearn(estimator): - return check_estimator(estimator) +@pytest.mark.parametrize( + # Encoders that encode X as a function of y; this is what + # breaks down when X becomes an array and indexes don't accidentally match in final + # concantenation + "encoder", + [MeanEncoder(), WoEEncoder(), PRatioEncoder()], +) +def test_fix_index_mismatch_from_upstream_array(encoder): + """ + Created 2022-02-19 to test fix to issue # 376 + Code adapted from: + https://github.com/scikit-learn-contrib/category_encoders/issues/280 + """ -_estimators = [ - CountFrequencyEncoder(), - DecisionTreeEncoder(regression=False), - MeanEncoder(), - OneHotEncoder(), - OrdinalEncoder(), - RareLabelEncoder(), - WoEEncoder(), - PRatioEncoder(), -] + # test dataframe; setup for a transfromation where + # coded version of 'x' will be a function of target 'y' + df: pd.DataFrame = pd.DataFrame( + { + "x": ["a", "a", "b", "b", "c", "c"], + "y": [1, 0, 1, 0, 1, 0], + } + ) + # Key - "non-standard" index that is not the usual + # contiguous range starting a t 0 + df.index = [101, 105, 42, 76, 88, 92] + + # Set up for standard pipeline/training etc. + X: pd.DataFrame = df[["x"]] + y: pd.Series = df["y"] + + # Will serve as a no-op whose chief purpose is to turn the + # X into an np.ndarray + si = SimpleImputer(strategy="constant", fill_value="a") + + # Sequence leading to issue: + # 1) X becomes an array + assert type(X) == pd.DataFrame + X_2: np.array = si.fit_transform(X) + assert type(X_2) == np.ndarray + # 2) Encoder encodes as function of X, y + df_result: pd.DataFrame = encoder.fit_transform(X_2, y) + assert type(df_result) == pd.DataFrame -@pytest.mark.parametrize("estimator", _estimators) -def test_check_estimator_from_feature_engine(estimator): - return check_feature_engine_estimator(estimator) + # Assertion fails: breakdown in index matches causes results to be all nan + assert all(df_result.iloc[:, 0].notnull()) diff --git a/tests/test_encoding/test_fix_index_mismatch_from_upstream_array.py b/tests/test_encoding/test_fix_index_mismatch_from_upstream_array.py deleted file mode 100644 index 7d5276b14..000000000 --- a/tests/test_encoding/test_fix_index_mismatch_from_upstream_array.py +++ /dev/null @@ -1,54 +0,0 @@ -import numpy as np -import pandas as pd -import pytest -from sklearn.impute import SimpleImputer - -from feature_engine.encoding import MeanEncoder, PRatioEncoder, WoEEncoder - - -@pytest.mark.parametrize( - # Encoders that encode X as a function of y; this is what - # breaks down when X becomes an array and indexes don't accidentally match in final - # concantenation - "encoder", - [MeanEncoder(), WoEEncoder(), PRatioEncoder()], -) -def test_fix_index_mismatch_from_upstream_array(encoder): - """ - Created 2022-02-19 to test fix to issue # 376 - Code adapted from: - https://github.com/scikit-learn-contrib/category_encoders/issues/280 - """ - - # test dataframe; setup for a transfromation where - # coded version of 'x' will be a function of target 'y' - df: pd.DataFrame = pd.DataFrame( - { - "x": ["a", "a", "b", "b", "c", "c"], - "y": [1, 0, 1, 0, 1, 0], - } - ) - # Key - "non-standard" index that is not the usual - # contiguous range starting a t 0 - df.index = [101, 105, 42, 76, 88, 92] - - # Set up for standard pipeline/training etc. - X: pd.DataFrame = df[["x"]] - y: pd.Series = df["y"] - - # Will serve as a no-op whose chief purpose is to turn the - # X into an np.ndarray - si = SimpleImputer(strategy="constant", fill_value="a") - - # Sequence leading to issue: - # 1) X becomes an array - assert type(X) == pd.DataFrame - X_2: np.array = si.fit_transform(X) - assert type(X_2) == np.ndarray - - # 2) Encoder encodes as function of X, y - df_result: pd.DataFrame = encoder.fit_transform(X_2, y) - assert type(df_result) == pd.DataFrame - - # Assertion fails: breakdown in index matches causes results to be all nan - assert all(df_result.iloc[:, 0].notnull()) From 8916bf05d0909ee33990f465e57e5cce4fd30643 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Mon, 21 Feb 2022 22:46:24 -0500 Subject: [PATCH 07/55] following suggestion in PR feedback to simplify test and remove dependence on SimpleImputer --- .../test_check_estimator_encoders.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index 096aa3c5d..e4ee48056 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -84,19 +84,7 @@ def test_fix_index_mismatch_from_upstream_array(encoder): X: pd.DataFrame = df[["x"]] y: pd.Series = df["y"] - # Will serve as a no-op whose chief purpose is to turn the - # X into an np.ndarray - si = SimpleImputer(strategy="constant", fill_value="a") - - # Sequence leading to issue: - # 1) X becomes an array - assert type(X) == pd.DataFrame - X_2: np.array = si.fit_transform(X) - assert type(X_2) == np.ndarray - - # 2) Encoder encodes as function of X, y + # Test issue fix where X becomes array, y remains Series with original DataFrame index + X_2: np.ndarray = X.to_numpy() df_result: pd.DataFrame = encoder.fit_transform(X_2, y) - assert type(df_result) == pd.DataFrame - - # Assertion fails: breakdown in index matches causes results to be all nan assert all(df_result.iloc[:, 0].notnull()) From db50fe4aa72e60df83ef825d4edba00d254c1d31 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Mon, 21 Feb 2022 22:52:57 -0500 Subject: [PATCH 08/55] forgot to remove import --- tests/test_encoding/test_check_estimator_encoders.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index e4ee48056..2e5465e3a 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -1,7 +1,6 @@ import pytest import pandas as pd import numpy as np -from sklearn.impute import SimpleImputer from sklearn.utils.estimator_checks import check_estimator from feature_engine.encoding import ( From 866769631e8410f571a5d29a16ab888cc0f5bcdb Mon Sep 17 00:00:00 2001 From: Noah Green Date: Mon, 21 Feb 2022 23:01:58 -0500 Subject: [PATCH 09/55] added a few more encoders to unit test that were not having issue but putting them in for coverage --- tests/test_encoding/test_check_estimator_encoders.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index 2e5465e3a..a26601b7e 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -58,7 +58,10 @@ def test_all_transformers(Estimator): # breaks down when X becomes an array and indexes don't accidentally match in final # concantenation "encoder", - [MeanEncoder(), WoEEncoder(), PRatioEncoder()], + [ + MeanEncoder(), WoEEncoder(), PRatioEncoder(), + OrdinalEncoder(encoding_method="ordered"), DecisionTreeEncoder() + ], ) def test_fix_index_mismatch_from_upstream_array(encoder): """ From 28a2edd4990ac9cb4805eb23435732dee65edf7f Mon Sep 17 00:00:00 2001 From: Noah Green Date: Mon, 21 Feb 2022 23:06:36 -0500 Subject: [PATCH 10/55] isort/black and other reformatting --- .../test_encoding/test_check_estimator_encoders.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index a26601b7e..e245318ef 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -1,6 +1,6 @@ -import pytest -import pandas as pd import numpy as np +import pandas as pd +import pytest from sklearn.utils.estimator_checks import check_estimator from feature_engine.encoding import ( @@ -59,8 +59,11 @@ def test_all_transformers(Estimator): # concantenation "encoder", [ - MeanEncoder(), WoEEncoder(), PRatioEncoder(), - OrdinalEncoder(encoding_method="ordered"), DecisionTreeEncoder() + MeanEncoder(), + WoEEncoder(), + PRatioEncoder(), + OrdinalEncoder(encoding_method="ordered"), + DecisionTreeEncoder(), ], ) def test_fix_index_mismatch_from_upstream_array(encoder): @@ -86,7 +89,8 @@ def test_fix_index_mismatch_from_upstream_array(encoder): X: pd.DataFrame = df[["x"]] y: pd.Series = df["y"] - # Test issue fix where X becomes array, y remains Series with original DataFrame index + # Test issue fix where X becomes array, + # y remains Series with original DataFrame index X_2: np.ndarray = X.to_numpy() df_result: pd.DataFrame = encoder.fit_transform(X_2, y) assert all(df_result.iloc[:, 0].notnull()) From 261190dcdd0f1ebe3684e17297cb464544df03cb Mon Sep 17 00:00:00 2001 From: Noah Green Date: Mon, 21 Feb 2022 23:33:14 -0500 Subject: [PATCH 11/55] changed fixture to have different y values for different encoder tests so that DecisionTreeEncoder can be unit tested --- .../test_check_estimator_encoders.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index e245318ef..324c32276 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -55,18 +55,18 @@ def test_all_transformers(Estimator): @pytest.mark.parametrize( # Encoders that encode X as a function of y; this is what - # breaks down when X becomes an array and indexes don't accidentally match in final - # concantenation - "encoder", + # breaks down when X becomes an array and indexes don't + # accidentally match in final concantenation + "encoder, y_vals", [ - MeanEncoder(), - WoEEncoder(), - PRatioEncoder(), - OrdinalEncoder(encoding_method="ordered"), - DecisionTreeEncoder(), + (MeanEncoder(), [1, 0, 1, 0, 1, 0]), + (WoEEncoder(), [1, 0, 1, 0, 1, 0]), + (PRatioEncoder(), [1, 0, 1, 0, 1, 0]), + (OrdinalEncoder(encoding_method="ordered"), [1, 0, 1, 0, 1, 0]), + (DecisionTreeEncoder(), [21, 30, 21, 30, 51, 40]), ], ) -def test_fix_index_mismatch_from_upstream_array(encoder): +def test_fix_index_mismatch_from_upstream_array(encoder, y_vals): """ Created 2022-02-19 to test fix to issue # 376 Code adapted from: @@ -78,7 +78,7 @@ def test_fix_index_mismatch_from_upstream_array(encoder): df: pd.DataFrame = pd.DataFrame( { "x": ["a", "a", "b", "b", "c", "c"], - "y": [1, 0, 1, 0, 1, 0], + "y": y_vals, } ) # Key - "non-standard" index that is not the usual From 553f73dda2d13b33017823f3390211a3526ab6c6 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sun, 27 Mar 2022 11:47:03 -0400 Subject: [PATCH 12/55] change to test_fix_index_mismatch_from_upstream_array() to illustrate additional issue when X has more than 1 column --- .../test_encoding/test_check_estimator_encoders.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index 324c32276..db3e8cbe0 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -59,11 +59,11 @@ def test_all_transformers(Estimator): # accidentally match in final concantenation "encoder, y_vals", [ - (MeanEncoder(), [1, 0, 1, 0, 1, 0]), - (WoEEncoder(), [1, 0, 1, 0, 1, 0]), - (PRatioEncoder(), [1, 0, 1, 0, 1, 0]), - (OrdinalEncoder(encoding_method="ordered"), [1, 0, 1, 0, 1, 0]), - (DecisionTreeEncoder(), [21, 30, 21, 30, 51, 40]), + (MeanEncoder(variables=["x"]), [1, 0, 1, 0, 1, 0]), + (WoEEncoder(variables=["x"]), [1, 0, 1, 0, 1, 0]), + (PRatioEncoder(variables=["x"]), [1, 0, 1, 0, 1, 0]), + (OrdinalEncoder(variables=["x"], encoding_method="ordered"), [1, 0, 1, 0, 1, 0]), + (DecisionTreeEncoder(variables=["x"]), [21, 30, 21, 30, 51, 40]), ], ) def test_fix_index_mismatch_from_upstream_array(encoder, y_vals): @@ -78,6 +78,7 @@ def test_fix_index_mismatch_from_upstream_array(encoder, y_vals): df: pd.DataFrame = pd.DataFrame( { "x": ["a", "a", "b", "b", "c", "c"], + "other": ["g", "w", "d", "f", "l", "m"], "y": y_vals, } ) @@ -86,7 +87,7 @@ def test_fix_index_mismatch_from_upstream_array(encoder, y_vals): df.index = [101, 105, 42, 76, 88, 92] # Set up for standard pipeline/training etc. - X: pd.DataFrame = df[["x"]] + X: pd.DataFrame = df.drop(columns="y") y: pd.Series = df["y"] # Test issue fix where X becomes array, From 23929a88a22f1f647fe847a19d1d1552c6b17d99 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sun, 27 Mar 2022 17:12:50 -0400 Subject: [PATCH 13/55] Revert "change to test_fix_index_mismatch_from_upstream_array() to illustrate additional issue when X has more than 1 column" This reverts commit 9a8d6ceb75280e0e3529fbecfe4b62cf856f2eb5. --- .../test_encoding/test_check_estimator_encoders.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index db3e8cbe0..324c32276 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -59,11 +59,11 @@ def test_all_transformers(Estimator): # accidentally match in final concantenation "encoder, y_vals", [ - (MeanEncoder(variables=["x"]), [1, 0, 1, 0, 1, 0]), - (WoEEncoder(variables=["x"]), [1, 0, 1, 0, 1, 0]), - (PRatioEncoder(variables=["x"]), [1, 0, 1, 0, 1, 0]), - (OrdinalEncoder(variables=["x"], encoding_method="ordered"), [1, 0, 1, 0, 1, 0]), - (DecisionTreeEncoder(variables=["x"]), [21, 30, 21, 30, 51, 40]), + (MeanEncoder(), [1, 0, 1, 0, 1, 0]), + (WoEEncoder(), [1, 0, 1, 0, 1, 0]), + (PRatioEncoder(), [1, 0, 1, 0, 1, 0]), + (OrdinalEncoder(encoding_method="ordered"), [1, 0, 1, 0, 1, 0]), + (DecisionTreeEncoder(), [21, 30, 21, 30, 51, 40]), ], ) def test_fix_index_mismatch_from_upstream_array(encoder, y_vals): @@ -78,7 +78,6 @@ def test_fix_index_mismatch_from_upstream_array(encoder, y_vals): df: pd.DataFrame = pd.DataFrame( { "x": ["a", "a", "b", "b", "c", "c"], - "other": ["g", "w", "d", "f", "l", "m"], "y": y_vals, } ) @@ -87,7 +86,7 @@ def test_fix_index_mismatch_from_upstream_array(encoder, y_vals): df.index = [101, 105, 42, 76, 88, 92] # Set up for standard pipeline/training etc. - X: pd.DataFrame = df.drop(columns="y") + X: pd.DataFrame = df[["x"]] y: pd.Series = df["y"] # Test issue fix where X becomes array, From 5b2d68602724e1ad23fcb44ff71533af750ab221 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sun, 27 Mar 2022 17:34:47 -0400 Subject: [PATCH 14/55] refactored and renamed original unit test to get into form where we can make duplicate using same parameterization only for the y ndarray case --- .../test_check_estimator_encoders.py | 67 ++++++++++++------- 1 file changed, 43 insertions(+), 24 deletions(-) diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index 324c32276..3635f2715 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -57,40 +57,59 @@ def test_all_transformers(Estimator): # Encoders that encode X as a function of y; this is what # breaks down when X becomes an array and indexes don't # accidentally match in final concantenation - "encoder, y_vals", + + # All test DataFrames have same data except DecisionTreeEncoder(), + # which needs different y values. + + # Key to all: - "non-standard" index that is not the usual + # contiguous range starting a t 0 + + "encoder, df_test", [ - (MeanEncoder(), [1, 0, 1, 0, 1, 0]), - (WoEEncoder(), [1, 0, 1, 0, 1, 0]), - (PRatioEncoder(), [1, 0, 1, 0, 1, 0]), - (OrdinalEncoder(encoding_method="ordered"), [1, 0, 1, 0, 1, 0]), - (DecisionTreeEncoder(), [21, 30, 21, 30, 51, 40]), - ], + (MeanEncoder(), + pd.DataFrame({ + "x": ["a", "a", "b", "b", "c", "c"], + "y": [1, 0, 1, 0, 1, 0] + }, index=[101, 105, 42, 76, 88, 92])), + + (WoEEncoder(), + pd.DataFrame({ + "x": ["a", "a", "b", "b", "c", "c"], + "y": [1, 0, 1, 0, 1, 0] + }, index=[101, 105, 42, 76, 88, 92])), + + (PRatioEncoder(), + pd.DataFrame({ + "x": ["a", "a", "b", "b", "c", "c"], + "y": [1, 0, 1, 0, 1, 0] + }, index=[101, 105, 42, 76, 88, 92])), + + (OrdinalEncoder(encoding_method="ordered"), + pd.DataFrame({ + "x": ["a", "a", "b", "b", "c", "c"], + "y": [1, 0, 1, 0, 1, 0] + }, index=[101, 105, 42, 76, 88, 92])), + + (DecisionTreeEncoder(), + pd.DataFrame({ + "x": ["a", "a", "b", "b", "c", "c"], + "y": [21, 30, 21, 30, 51, 40] + }, index=[101, 105, 42, 76, 88, 92])), + ] ) -def test_fix_index_mismatch_from_upstream_array(encoder, y_vals): +def test_fix_index_mismatch_from_x_numpy_y_pandas(encoder, df_test): """ - Created 2022-02-19 to test fix to issue # 376 + Created 2022-03-27 to test fix to issue # 376 Code adapted from: https://github.com/scikit-learn-contrib/category_encoders/issues/280 """ - # test dataframe; setup for a transfromation where - # coded version of 'x' will be a function of target 'y' - df: pd.DataFrame = pd.DataFrame( - { - "x": ["a", "a", "b", "b", "c", "c"], - "y": y_vals, - } - ) - # Key - "non-standard" index that is not the usual - # contiguous range starting a t 0 - df.index = [101, 105, 42, 76, 88, 92] - # Set up for standard pipeline/training etc. - X: pd.DataFrame = df[["x"]] - y: pd.Series = df["y"] + X: pd.DataFrame = df_test[["x"]] + y: pd.Series = df_test["y"] # Test issue fix where X becomes array, # y remains Series with original DataFrame index X_2: np.ndarray = X.to_numpy() df_result: pd.DataFrame = encoder.fit_transform(X_2, y) - assert all(df_result.iloc[:, 0].notnull()) + assert all(df_result.iloc[:, 0].notnull()) \ No newline at end of file From 2f998b73fff53de2ebc53e59ee167e9d948ae9f0 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sun, 27 Mar 2022 20:53:32 -0400 Subject: [PATCH 15/55] added new unit test for case where y is the array and X is the pandas object --- .../test_check_estimator_encoders.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index 3635f2715..dc08db9ed 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -112,4 +112,66 @@ def test_fix_index_mismatch_from_x_numpy_y_pandas(encoder, df_test): # y remains Series with original DataFrame index X_2: np.ndarray = X.to_numpy() df_result: pd.DataFrame = encoder.fit_transform(X_2, y) + assert all(df_result.iloc[:, 0].notnull()) + + +@pytest.mark.parametrize( + # Encoders that encode X as a function of y; this is what + # breaks down when y becomes an array and indexes don't + # accidentally match in final concantenation + + # All test DataFrames have same data except DecisionTreeEncoder(), + # which needs different y values. + + # Key to all: - "non-standard" index that is not the usual + # contiguous range starting a t 0 + + "encoder, df_test", + [ + (MeanEncoder(), + pd.DataFrame({ + "x": ["a", "a", "b", "b", "c", "c"], + "y": [1, 0, 1, 0, 1, 0] + }, index=[101, 105, 42, 76, 88, 92])), + + (WoEEncoder(), + pd.DataFrame({ + "x": ["a", "a", "b", "b", "c", "c"], + "y": [1, 0, 1, 0, 1, 0] + }, index=[101, 105, 42, 76, 88, 92])), + + (PRatioEncoder(), + pd.DataFrame({ + "x": ["a", "a", "b", "b", "c", "c"], + "y": [1, 0, 1, 0, 1, 0] + }, index=[101, 105, 42, 76, 88, 92])), + + (OrdinalEncoder(encoding_method="ordered"), + pd.DataFrame({ + "x": ["a", "a", "b", "b", "c", "c"], + "y": [1, 0, 1, 0, 1, 0] + }, index=[101, 105, 42, 76, 88, 92])), + + (DecisionTreeEncoder(), + pd.DataFrame({ + "x": ["a", "a", "b", "b", "c", "c"], + "y": [21, 30, 21, 30, 51, 40] + }, index=[101, 105, 42, 76, 88, 92])), + ] +) +def test_fix_index_mismatch_from_x_pandas_y_numpy(encoder, df_test): + """ + Created 2022-03-27 to test fix to issue # 376 + Code adapted from: + https://github.com/scikit-learn-contrib/category_encoders/issues/280 + """ + + # Set up for standard pipeline/training etc. + X: pd.DataFrame = df_test[["x"]] + y: pd.Series = df_test["y"] + + # Test issue fix where X becomes array, + # y remains Series with original DataFrame index + y_2: np.ndarray = y.to_numpy() + df_result: pd.DataFrame = encoder.fit_transform(X, y_2) assert all(df_result.iloc[:, 0].notnull()) \ No newline at end of file From d74d14343849607c2d6e1dd9c94f44ec276bf070 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sun, 27 Mar 2022 21:21:42 -0400 Subject: [PATCH 16/55] renamed _check_for_X_y_index_mismatch() to _check_X_y_pd_np_mismatch(); function now handles case where y is the ndarray, and is just more generalized to support normalizing to pandas objects with consistent indexes. Changed calls to new function in all relevant encoders --- feature_engine/encoding/mean_encoding.py | 4 ++-- feature_engine/encoding/probability_ratio.py | 4 ++-- feature_engine/encoding/woe.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/feature_engine/encoding/mean_encoding.py b/feature_engine/encoding/mean_encoding.py index 6dad49448..de4d5aace 100644 --- a/feature_engine/encoding/mean_encoding.py +++ b/feature_engine/encoding/mean_encoding.py @@ -5,7 +5,7 @@ import pandas as pd -from feature_engine.dataframe_checks import _check_for_X_y_index_mismatch +from feature_engine.dataframe_checks import _check_X_y_pd_np_mismatch from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -132,8 +132,8 @@ def fit(self, X: pd.DataFrame, y: pd.Series): The target. """ + X, y = _check_X_y_pd_np_mismatch(X, y) X = self._check_fit_input_and_variables(X) - X = _check_for_X_y_index_mismatch(X, y) if not isinstance(y, pd.Series): y = pd.Series(y) diff --git a/feature_engine/encoding/probability_ratio.py b/feature_engine/encoding/probability_ratio.py index 305d81b90..d5acc7c24 100644 --- a/feature_engine/encoding/probability_ratio.py +++ b/feature_engine/encoding/probability_ratio.py @@ -6,7 +6,7 @@ import numpy as np import pandas as pd -from feature_engine.dataframe_checks import _check_for_X_y_index_mismatch +from feature_engine.dataframe_checks import _check_X_y_pd_np_mismatch from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -155,8 +155,8 @@ def fit(self, X: pd.DataFrame, y: pd.Series): Target, must be binary. """ + X, y = _check_X_y_pd_np_mismatch(X, y) X = self._check_fit_input_and_variables(X) - X = _check_for_X_y_index_mismatch(X, y) if not isinstance(y, pd.Series): y = pd.Series(y) diff --git a/feature_engine/encoding/woe.py b/feature_engine/encoding/woe.py index a944eec9b..af180a3c8 100644 --- a/feature_engine/encoding/woe.py +++ b/feature_engine/encoding/woe.py @@ -6,7 +6,7 @@ import numpy as np import pandas as pd -from feature_engine.dataframe_checks import _check_for_X_y_index_mismatch +from feature_engine.dataframe_checks import _check_X_y_pd_np_mismatch from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -137,8 +137,8 @@ def fit(self, X: pd.DataFrame, y: pd.Series): Target, must be binary. """ + X, y = _check_X_y_pd_np_mismatch(X, y) X = self._check_fit_input_and_variables(X) - X = _check_for_X_y_index_mismatch(X, y) if not isinstance(y, pd.Series): y = pd.Series(y) From 7c274dd96479a3d9132bfb8370a6efb4b19f298c Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sun, 27 Mar 2022 21:58:45 -0400 Subject: [PATCH 17/55] black/isort/flake8 etc. fixes --- feature_engine/dataframe_checks.py | 45 +++--- .../test_check_estimator_encoders.py | 140 +++++++++--------- 2 files changed, 90 insertions(+), 95 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index fb9683101..247e09bbd 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -131,46 +131,35 @@ def _check_contains_inf(X: pd.DataFrame, variables: List[Union[str, int]]) -> No ) -def _check_for_X_y_index_mismatch( +def _check_X_y_pd_np_mismatch( X: Union[pd.DataFrame, pd.Series, np.ndarray], y: Union[pd.DataFrame, pd.Series, np.ndarray], ): """ - Handles the following case: - 1) X and y came from a DataFrame whose index was not the standard contiguous 0-n - 2) Earlier on, X was affected by an sklearn transform, turning - it into an array and losing its different index - 3) X enters a feature-engine transformer and becomes a - DataFrame again via _is_dataframe() - 4) X and y now have different indexes - This function checks for this case; if it is met, it will return - a copy of X with its index replaced with the - correct index from y. It will not make any changes if either X or - y is not a pandas object, or of course if there is no index mismatch. - This case was first detected in issue #376. + Handles case where X is an ndarray and y is a Series, + or when X is a DataFrame but y is in an ndarray. + In both cases, the non-pandas object will be converted + to a pandas object, and take on :wqthe index of the other (pandas) object. + If both are ndarray objects, they are returned unchanged. + If both are not pandas objects, method returns objects unchanged. Parameters ---------- X: Pandas DataFrame, Series, or numpy ndarray - In all likelihood will be DataFrame, due to this method usually - being called with the output of _is_dataframe(). - Will not make any changes if X is not a DataFrame or Series. y: Pandas DataFrame, Series, or numpy ndarray - In all likelihood will be a Series. - Will not make any changes if X is not a DataFrame or Series. Returns ------- - X: same type as parameter X. - If the mismatch conditions described above have been met, returns a copy of - the original parameter, with index set to y's index. - Else, returns original parameter value unaffected. + X: changed as per description above + y: changed as per description above """ - is_pd_X_and_y: bool = all( - [(type(i) == pd.DataFrame or type(i) == pd.Series) for i in (X, y)] - ) - if is_pd_X_and_y and any(X.index != y.index): - X = X.copy() + if isinstance(X, np.ndarray) and isinstance(y, (pd.DataFrame, pd.Series)): + # already know X is not a DataFrame, use machinery in _is_dataframe() + # to correctly convert X to DataFrame + X = _is_dataframe(X) X.index = y.index + elif isinstance(X, (pd.DataFrame, pd.Series)) and isinstance(y, np.ndarray): + y = pd.Series(y) + y.index = X.index - return X + return X, y diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index dc08db9ed..a4385b5c9 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -57,45 +57,48 @@ def test_all_transformers(Estimator): # Encoders that encode X as a function of y; this is what # breaks down when X becomes an array and indexes don't # accidentally match in final concantenation - # All test DataFrames have same data except DecisionTreeEncoder(), # which needs different y values. - # Key to all: - "non-standard" index that is not the usual # contiguous range starting a t 0 - "encoder, df_test", [ - (MeanEncoder(), - pd.DataFrame({ - "x": ["a", "a", "b", "b", "c", "c"], - "y": [1, 0, 1, 0, 1, 0] - }, index=[101, 105, 42, 76, 88, 92])), - - (WoEEncoder(), - pd.DataFrame({ - "x": ["a", "a", "b", "b", "c", "c"], - "y": [1, 0, 1, 0, 1, 0] - }, index=[101, 105, 42, 76, 88, 92])), - - (PRatioEncoder(), - pd.DataFrame({ - "x": ["a", "a", "b", "b", "c", "c"], - "y": [1, 0, 1, 0, 1, 0] - }, index=[101, 105, 42, 76, 88, 92])), - - (OrdinalEncoder(encoding_method="ordered"), - pd.DataFrame({ - "x": ["a", "a", "b", "b", "c", "c"], - "y": [1, 0, 1, 0, 1, 0] - }, index=[101, 105, 42, 76, 88, 92])), - - (DecisionTreeEncoder(), - pd.DataFrame({ - "x": ["a", "a", "b", "b", "c", "c"], - "y": [21, 30, 21, 30, 51, 40] - }, index=[101, 105, 42, 76, 88, 92])), - ] + ( + MeanEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ( + WoEEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ( + PRatioEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ( + OrdinalEncoder(encoding_method="ordered"), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ( + DecisionTreeEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [21, 30, 21, 30, 51, 40]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ], ) def test_fix_index_mismatch_from_x_numpy_y_pandas(encoder, df_test): """ @@ -119,45 +122,48 @@ def test_fix_index_mismatch_from_x_numpy_y_pandas(encoder, df_test): # Encoders that encode X as a function of y; this is what # breaks down when y becomes an array and indexes don't # accidentally match in final concantenation - # All test DataFrames have same data except DecisionTreeEncoder(), # which needs different y values. - # Key to all: - "non-standard" index that is not the usual # contiguous range starting a t 0 - "encoder, df_test", [ - (MeanEncoder(), - pd.DataFrame({ - "x": ["a", "a", "b", "b", "c", "c"], - "y": [1, 0, 1, 0, 1, 0] - }, index=[101, 105, 42, 76, 88, 92])), - - (WoEEncoder(), - pd.DataFrame({ - "x": ["a", "a", "b", "b", "c", "c"], - "y": [1, 0, 1, 0, 1, 0] - }, index=[101, 105, 42, 76, 88, 92])), - - (PRatioEncoder(), - pd.DataFrame({ - "x": ["a", "a", "b", "b", "c", "c"], - "y": [1, 0, 1, 0, 1, 0] - }, index=[101, 105, 42, 76, 88, 92])), - - (OrdinalEncoder(encoding_method="ordered"), - pd.DataFrame({ - "x": ["a", "a", "b", "b", "c", "c"], - "y": [1, 0, 1, 0, 1, 0] - }, index=[101, 105, 42, 76, 88, 92])), - - (DecisionTreeEncoder(), - pd.DataFrame({ - "x": ["a", "a", "b", "b", "c", "c"], - "y": [21, 30, 21, 30, 51, 40] - }, index=[101, 105, 42, 76, 88, 92])), - ] + ( + MeanEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ( + WoEEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ( + PRatioEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ( + OrdinalEncoder(encoding_method="ordered"), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ( + DecisionTreeEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [21, 30, 21, 30, 51, 40]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ], ) def test_fix_index_mismatch_from_x_pandas_y_numpy(encoder, df_test): """ @@ -174,4 +180,4 @@ def test_fix_index_mismatch_from_x_pandas_y_numpy(encoder, df_test): # y remains Series with original DataFrame index y_2: np.ndarray = y.to_numpy() df_result: pd.DataFrame = encoder.fit_transform(X, y_2) - assert all(df_result.iloc[:, 0].notnull()) \ No newline at end of file + assert all(df_result.iloc[:, 0].notnull()) From dd901f523cefa9ddf326fe041c0a5da7174d445d Mon Sep 17 00:00:00 2001 From: Noah Green Date: Mon, 28 Mar 2022 07:55:28 -0400 Subject: [PATCH 18/55] changed unit tests to assert on expected values, rather than all non-NaN; OrdinalEncoder test now failing as expected --- .../test_check_estimator_encoders.py | 64 +++++++++++++++---- 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index a4385b5c9..aea902d94 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -61,7 +61,7 @@ def test_all_transformers(Estimator): # which needs different y values. # Key to all: - "non-standard" index that is not the usual # contiguous range starting a t 0 - "encoder, df_test", + "encoder, df_test, df_expected", [ ( MeanEncoder(), @@ -69,6 +69,9 @@ def test_all_transformers(Estimator): {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, index=[101, 105, 42, 76, 88, 92], ), + pd.DataFrame( + {"0": [0.5, 0.5, 0.5, 0.5, 0.5, 0.5]}, + ) ), ( WoEEncoder(), @@ -76,6 +79,9 @@ def test_all_transformers(Estimator): {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, index=[101, 105, 42, 76, 88, 92], ), + pd.DataFrame( + {"0": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, + ) ), ( PRatioEncoder(), @@ -83,13 +89,19 @@ def test_all_transformers(Estimator): {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, index=[101, 105, 42, 76, 88, 92], ), + pd.DataFrame( + {"0": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, + ) ), ( OrdinalEncoder(encoding_method="ordered"), pd.DataFrame( - {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, - index=[101, 105, 42, 76, 88, 92], + {"x": ["a", "a", "a", "b", "b", "b", "c", "c", "c"], "y":[3, 3, 3, 2, 2, 2, 1, 1, 1]}, + index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1] ), + pd.DataFrame( + {"0": [2, 2, 2, 1, 1, 1, 0, 0, 0]} + ) ), ( DecisionTreeEncoder(), @@ -97,10 +109,13 @@ def test_all_transformers(Estimator): {"x": ["a", "a", "b", "b", "c", "c"], "y": [21, 30, 21, 30, 51, 40]}, index=[101, 105, 42, 76, 88, 92], ), + pd.DataFrame( + {"0": [25.5, 25.5, 25.5, 25.5, 45.5, 45.5]}, + ) ), ], ) -def test_fix_index_mismatch_from_x_numpy_y_pandas(encoder, df_test): +def test_fix_index_mismatch_from_x_numpy_y_pandas(encoder, df_test, df_expected): """ Created 2022-03-27 to test fix to issue # 376 Code adapted from: @@ -115,7 +130,7 @@ def test_fix_index_mismatch_from_x_numpy_y_pandas(encoder, df_test): # y remains Series with original DataFrame index X_2: np.ndarray = X.to_numpy() df_result: pd.DataFrame = encoder.fit_transform(X_2, y) - assert all(df_result.iloc[:, 0].notnull()) + assert df_result.equals(df_expected) @pytest.mark.parametrize( @@ -126,46 +141,66 @@ def test_fix_index_mismatch_from_x_numpy_y_pandas(encoder, df_test): # which needs different y values. # Key to all: - "non-standard" index that is not the usual # contiguous range starting a t 0 - "encoder, df_test", + "encoder, df_test, df_expected", [ ( MeanEncoder(), pd.DataFrame( {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, - index=[101, 105, 42, 76, 88, 92], + index=[101, 105, 42, 76, 88, 92] ), + pd.DataFrame( + {"x": [0.5, 0.5, 0.5, 0.5, 0.5, 0.5]}, + index=[101, 105, 42, 76, 88, 92] + ) ), ( WoEEncoder(), pd.DataFrame( {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, - index=[101, 105, 42, 76, 88, 92], + index=[101, 105, 42, 76, 88, 92] ), + pd.DataFrame( + {"x": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, + index=[101, 105, 42, 76, 88, 92] + ) ), ( PRatioEncoder(), pd.DataFrame( {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, - index=[101, 105, 42, 76, 88, 92], + index=[101, 105, 42, 76, 88, 92] ), + pd.DataFrame( + {"x": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, + index=[101, 105, 42, 76, 88, 92] + ) ), ( OrdinalEncoder(encoding_method="ordered"), pd.DataFrame( - {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, - index=[101, 105, 42, 76, 88, 92], + {"x": ["a", "a", "a", "b", "b", "b", "c", "c", "c"], "y":[3, 3, 3, 2, 2, 2, 1, 1, 1]}, + index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1] ), + pd.DataFrame( + {"x": [2, 2, 2, 1, 1, 1, 0, 0, 0]}, + index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1] + ) ), ( DecisionTreeEncoder(), pd.DataFrame( {"x": ["a", "a", "b", "b", "c", "c"], "y": [21, 30, 21, 30, 51, 40]}, - index=[101, 105, 42, 76, 88, 92], + index=[101, 105, 42, 76, 88, 92] ), + pd.DataFrame( + {"x": [25.5, 25.5, 25.5, 25.5, 45.5, 45.5]}, + index=[101, 105, 42, 76, 88, 92] + ) ), ], ) -def test_fix_index_mismatch_from_x_pandas_y_numpy(encoder, df_test): +def test_fix_index_mismatch_from_x_pandas_y_numpy(encoder, df_test, df_expected): """ Created 2022-03-27 to test fix to issue # 376 Code adapted from: @@ -180,4 +215,5 @@ def test_fix_index_mismatch_from_x_pandas_y_numpy(encoder, df_test): # y remains Series with original DataFrame index y_2: np.ndarray = y.to_numpy() df_result: pd.DataFrame = encoder.fit_transform(X, y_2) - assert all(df_result.iloc[:, 0].notnull()) + assert df_result.equals(df_expected) + From 248edd9aa8d9e6bed88bf2f8b79f2fe1b02186aa Mon Sep 17 00:00:00 2001 From: Noah Green Date: Mon, 28 Mar 2022 08:18:35 -0400 Subject: [PATCH 19/55] added fix to OrdinalEncoder; all unit tests now passing --- feature_engine/encoding/ordinal.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/feature_engine/encoding/ordinal.py b/feature_engine/encoding/ordinal.py index cca752a8d..f31881738 100644 --- a/feature_engine/encoding/ordinal.py +++ b/feature_engine/encoding/ordinal.py @@ -5,6 +5,7 @@ import pandas as pd +from feature_engine.dataframe_checks import _check_X_y_pd_np_mismatch from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -146,6 +147,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): Otherwise, y needs to be passed when fitting the transformer. """ + X, y = _check_X_y_pd_np_mismatch(X, y) X = self._check_fit_input_and_variables(X) # join target to predictor variables From 39990ac1ea26e7785a934da923eb03d758d4eefe Mon Sep 17 00:00:00 2001 From: Noah Green Date: Mon, 28 Mar 2022 08:21:10 -0400 Subject: [PATCH 20/55] standardized ordering of test parameterizations for both new unit tests --- .../test_check_estimator_encoders.py | 68 +++++++++---------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index aea902d94..e65bf38d3 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -64,53 +64,53 @@ def test_all_transformers(Estimator): "encoder, df_test, df_expected", [ ( - MeanEncoder(), + DecisionTreeEncoder(), pd.DataFrame( - {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + {"x": ["a", "a", "b", "b", "c", "c"], "y": [21, 30, 21, 30, 51, 40]}, index=[101, 105, 42, 76, 88, 92], ), pd.DataFrame( - {"0": [0.5, 0.5, 0.5, 0.5, 0.5, 0.5]}, + {"0": [25.5, 25.5, 25.5, 25.5, 45.5, 45.5]}, ) ), ( - WoEEncoder(), + MeanEncoder(), pd.DataFrame( {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, index=[101, 105, 42, 76, 88, 92], ), pd.DataFrame( - {"0": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, + {"0": [0.5, 0.5, 0.5, 0.5, 0.5, 0.5]}, ) ), ( - PRatioEncoder(), + OrdinalEncoder(encoding_method="ordered"), pd.DataFrame( - {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, - index=[101, 105, 42, 76, 88, 92], + {"x": ["a", "a", "a", "b", "b", "b", "c", "c", "c"], "y": [3, 3, 3, 2, 2, 2, 1, 1, 1]}, + index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1] ), pd.DataFrame( - {"0": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, + {"0": [2, 2, 2, 1, 1, 1, 0, 0, 0]} ) ), ( - OrdinalEncoder(encoding_method="ordered"), + PRatioEncoder(), pd.DataFrame( - {"x": ["a", "a", "a", "b", "b", "b", "c", "c", "c"], "y":[3, 3, 3, 2, 2, 2, 1, 1, 1]}, - index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1] + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], ), pd.DataFrame( - {"0": [2, 2, 2, 1, 1, 1, 0, 0, 0]} + {"0": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, ) ), ( - DecisionTreeEncoder(), + WoEEncoder(), pd.DataFrame( - {"x": ["a", "a", "b", "b", "c", "c"], "y": [21, 30, 21, 30, 51, 40]}, + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, index=[101, 105, 42, 76, 88, 92], ), pd.DataFrame( - {"0": [25.5, 25.5, 25.5, 25.5, 45.5, 45.5]}, + {"0": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, ) ), ], @@ -144,57 +144,57 @@ def test_fix_index_mismatch_from_x_numpy_y_pandas(encoder, df_test, df_expected) "encoder, df_test, df_expected", [ ( - MeanEncoder(), + DecisionTreeEncoder(), pd.DataFrame( - {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + {"x": ["a", "a", "b", "b", "c", "c"], "y": [21, 30, 21, 30, 51, 40]}, index=[101, 105, 42, 76, 88, 92] ), pd.DataFrame( - {"x": [0.5, 0.5, 0.5, 0.5, 0.5, 0.5]}, + {"x": [25.5, 25.5, 25.5, 25.5, 45.5, 45.5]}, index=[101, 105, 42, 76, 88, 92] ) ), ( - WoEEncoder(), + MeanEncoder(), pd.DataFrame( {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, index=[101, 105, 42, 76, 88, 92] ), pd.DataFrame( - {"x": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, + {"x": [0.5, 0.5, 0.5, 0.5, 0.5, 0.5]}, index=[101, 105, 42, 76, 88, 92] ) ), ( - PRatioEncoder(), + OrdinalEncoder(encoding_method="ordered"), pd.DataFrame( - {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, - index=[101, 105, 42, 76, 88, 92] + {"x": ["a", "a", "a", "b", "b", "b", "c", "c", "c"], "y": [3, 3, 3, 2, 2, 2, 1, 1, 1]}, + index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1] ), pd.DataFrame( - {"x": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, - index=[101, 105, 42, 76, 88, 92] + {"x": [2, 2, 2, 1, 1, 1, 0, 0, 0]}, + index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1] ) ), ( - OrdinalEncoder(encoding_method="ordered"), + PRatioEncoder(), pd.DataFrame( - {"x": ["a", "a", "a", "b", "b", "b", "c", "c", "c"], "y":[3, 3, 3, 2, 2, 2, 1, 1, 1]}, - index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1] + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92] ), pd.DataFrame( - {"x": [2, 2, 2, 1, 1, 1, 0, 0, 0]}, - index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1] + {"x": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, + index=[101, 105, 42, 76, 88, 92] ) ), ( - DecisionTreeEncoder(), + WoEEncoder(), pd.DataFrame( - {"x": ["a", "a", "b", "b", "c", "c"], "y": [21, 30, 21, 30, 51, 40]}, + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, index=[101, 105, 42, 76, 88, 92] ), pd.DataFrame( - {"x": [25.5, 25.5, 25.5, 25.5, 45.5, 45.5]}, + {"x": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, index=[101, 105, 42, 76, 88, 92] ) ), From 8377b3b60cf33afd615902c0f5d18de042b5ffa4 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Mon, 28 Mar 2022 08:22:39 -0400 Subject: [PATCH 21/55] black/isort/flake8 changes --- .../test_check_estimator_encoders.py | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index e65bf38d3..c228487dc 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -71,7 +71,7 @@ def test_all_transformers(Estimator): ), pd.DataFrame( {"0": [25.5, 25.5, 25.5, 25.5, 45.5, 45.5]}, - ) + ), ), ( MeanEncoder(), @@ -81,17 +81,18 @@ def test_all_transformers(Estimator): ), pd.DataFrame( {"0": [0.5, 0.5, 0.5, 0.5, 0.5, 0.5]}, - ) + ), ), ( OrdinalEncoder(encoding_method="ordered"), pd.DataFrame( - {"x": ["a", "a", "a", "b", "b", "b", "c", "c", "c"], "y": [3, 3, 3, 2, 2, 2, 1, 1, 1]}, - index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1] + { + "x": ["a", "a", "a", "b", "b", "b", "c", "c", "c"], + "y": [3, 3, 3, 2, 2, 2, 1, 1, 1], + }, + index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1], ), - pd.DataFrame( - {"0": [2, 2, 2, 1, 1, 1, 0, 0, 0]} - ) + pd.DataFrame({"0": [2, 2, 2, 1, 1, 1, 0, 0, 0]}), ), ( PRatioEncoder(), @@ -101,7 +102,7 @@ def test_all_transformers(Estimator): ), pd.DataFrame( {"0": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, - ) + ), ), ( WoEEncoder(), @@ -111,7 +112,7 @@ def test_all_transformers(Estimator): ), pd.DataFrame( {"0": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, - ) + ), ), ], ) @@ -147,56 +148,56 @@ def test_fix_index_mismatch_from_x_numpy_y_pandas(encoder, df_test, df_expected) DecisionTreeEncoder(), pd.DataFrame( {"x": ["a", "a", "b", "b", "c", "c"], "y": [21, 30, 21, 30, 51, 40]}, - index=[101, 105, 42, 76, 88, 92] + index=[101, 105, 42, 76, 88, 92], ), pd.DataFrame( {"x": [25.5, 25.5, 25.5, 25.5, 45.5, 45.5]}, - index=[101, 105, 42, 76, 88, 92] - ) + index=[101, 105, 42, 76, 88, 92], + ), ), ( MeanEncoder(), pd.DataFrame( {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, - index=[101, 105, 42, 76, 88, 92] + index=[101, 105, 42, 76, 88, 92], ), pd.DataFrame( - {"x": [0.5, 0.5, 0.5, 0.5, 0.5, 0.5]}, - index=[101, 105, 42, 76, 88, 92] - ) + {"x": [0.5, 0.5, 0.5, 0.5, 0.5, 0.5]}, index=[101, 105, 42, 76, 88, 92] + ), ), ( OrdinalEncoder(encoding_method="ordered"), pd.DataFrame( - {"x": ["a", "a", "a", "b", "b", "b", "c", "c", "c"], "y": [3, 3, 3, 2, 2, 2, 1, 1, 1]}, - index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1] + { + "x": ["a", "a", "a", "b", "b", "b", "c", "c", "c"], + "y": [3, 3, 3, 2, 2, 2, 1, 1, 1], + }, + index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1], ), pd.DataFrame( {"x": [2, 2, 2, 1, 1, 1, 0, 0, 0]}, - index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1] - ) + index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1], + ), ), ( PRatioEncoder(), pd.DataFrame( {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, - index=[101, 105, 42, 76, 88, 92] + index=[101, 105, 42, 76, 88, 92], ), pd.DataFrame( - {"x": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, - index=[101, 105, 42, 76, 88, 92] - ) + {"x": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, index=[101, 105, 42, 76, 88, 92] + ), ), ( WoEEncoder(), pd.DataFrame( {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, - index=[101, 105, 42, 76, 88, 92] + index=[101, 105, 42, 76, 88, 92], ), pd.DataFrame( - {"x": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, - index=[101, 105, 42, 76, 88, 92] - ) + {"x": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, index=[101, 105, 42, 76, 88, 92] + ), ), ], ) @@ -216,4 +217,3 @@ def test_fix_index_mismatch_from_x_pandas_y_numpy(encoder, df_test, df_expected) y_2: np.ndarray = y.to_numpy() df_result: pd.DataFrame = encoder.fit_transform(X, y_2) assert df_result.equals(df_expected) - From b298711baf01d0144db549daa5cb93d5bbec0806 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Mon, 28 Mar 2022 19:41:50 -0400 Subject: [PATCH 22/55] did away with logic in _check_X_y_pd_np_mismatch() where X could be a Series --- feature_engine/dataframe_checks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index 247e09bbd..c4be46fd6 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -132,21 +132,21 @@ def _check_contains_inf(X: pd.DataFrame, variables: List[Union[str, int]]) -> No def _check_X_y_pd_np_mismatch( - X: Union[pd.DataFrame, pd.Series, np.ndarray], - y: Union[pd.DataFrame, pd.Series, np.ndarray], + X: Union[pd.DataFrame, np.ndarray], + y: Union[pd.Series, np.ndarray], ): """ Handles case where X is an ndarray and y is a Series, or when X is a DataFrame but y is in an ndarray. In both cases, the non-pandas object will be converted - to a pandas object, and take on :wqthe index of the other (pandas) object. + to a pandas object, and take on the index of the other (pandas) object. If both are ndarray objects, they are returned unchanged. If both are not pandas objects, method returns objects unchanged. Parameters ---------- - X: Pandas DataFrame, Series, or numpy ndarray - y: Pandas DataFrame, Series, or numpy ndarray + X: Pandas DataFrame or numpy ndarray + y: Pandas Series, or numpy ndarray Returns ------- @@ -158,7 +158,7 @@ def _check_X_y_pd_np_mismatch( # to correctly convert X to DataFrame X = _is_dataframe(X) X.index = y.index - elif isinstance(X, (pd.DataFrame, pd.Series)) and isinstance(y, np.ndarray): + elif isinstance(X, pd.DataFrame) and isinstance(y, np.ndarray): y = pd.Series(y) y.index = X.index From ccb390ddfd457df2d2b9a344290bdea77d49b485 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Mon, 28 Mar 2022 20:37:05 -0400 Subject: [PATCH 23/55] changed _check_X_y_pd_np_mismatch() to raise error when DataFrame X and Series y have mismatched indexes --- feature_engine/dataframe_checks.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index c4be46fd6..d64d9e75d 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -136,17 +136,17 @@ def _check_X_y_pd_np_mismatch( y: Union[pd.Series, np.ndarray], ): """ - Handles case where X is an ndarray and y is a Series, - or when X is a DataFrame but y is in an ndarray. - In both cases, the non-pandas object will be converted - to a pandas object, and take on the index of the other (pandas) object. + Handles 3 cases + 1. X is an ndarray and y is a Series - converts X to DataFrame with y's index + 2. X is a DataFrame and y is an ndarray - converts y to Series with X's index + 3. X is a DataFrame and y is a Series, but their indexes don't match - raises an error + If both are ndarray objects, they are returned unchanged. - If both are not pandas objects, method returns objects unchanged. Parameters ---------- X: Pandas DataFrame or numpy ndarray - y: Pandas Series, or numpy ndarray + y: Pandas Series or numpy ndarray Returns ------- @@ -161,5 +161,8 @@ def _check_X_y_pd_np_mismatch( elif isinstance(X, pd.DataFrame) and isinstance(y, np.ndarray): y = pd.Series(y) y.index = X.index + elif isinstance(X, pd.DataFrame) and isinstance(y, pd.Series): + if not all(y.index == X.index): + raise Exception("Index mismatch between DataFrame X and Series y") return X, y From e4eafce1d4330967ed3e19042f778f61920c5889 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Mon, 28 Mar 2022 20:38:05 -0400 Subject: [PATCH 24/55] added call to _check_X_y_pd_np_mismatch() in fit because tests were failing due to X/y mismatch in DecisionTree's leveraging of OrdinalEncoder --- feature_engine/encoding/decision_tree.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/feature_engine/encoding/decision_tree.py b/feature_engine/encoding/decision_tree.py index d728a9535..6e388fb61 100644 --- a/feature_engine/encoding/decision_tree.py +++ b/feature_engine/encoding/decision_tree.py @@ -8,6 +8,7 @@ from sklearn.utils.multiclass import check_classification_targets, type_of_target from feature_engine.discretisation import DecisionTreeDiscretiser +from feature_engine.dataframe_checks import _check_X_y_pd_np_mismatch from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -202,6 +203,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): check_classification_targets(y) # check input dataframe + X, y = _check_X_y_pd_np_mismatch(X, y) X = self._check_fit_input_and_variables(X) if self.param_grid: From 7dd69e9dc4ff94850f5fa0bf0c152bde778cdebf Mon Sep 17 00:00:00 2001 From: Noah Green Date: Mon, 28 Mar 2022 20:55:00 -0400 Subject: [PATCH 25/55] added unit test to assert exception in case where DataFrame X and Series y indexes do not match --- .../test_check_estimator_encoders.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index c228487dc..062c699fe 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -217,3 +217,65 @@ def test_fix_index_mismatch_from_x_pandas_y_numpy(encoder, df_test, df_expected) y_2: np.ndarray = y.to_numpy() df_result: pd.DataFrame = encoder.fit_transform(X, y_2) assert df_result.equals(df_expected) + + +@pytest.mark.parametrize( + "encoder, df_test", + [ + ( + DecisionTreeEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [21, 30, 21, 30, 51, 40]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ( + MeanEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ( + OrdinalEncoder(encoding_method="ordered"), + pd.DataFrame( + { + "x": ["a", "a", "a", "b", "b", "b", "c", "c", "c"], + "y": [3, 3, 3, 2, 2, 2, 1, 1, 1], + }, + index=[33, 5412, 66, 99, 334, 1212, 22, 555, 1], + ), + ), + ( + PRatioEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ( + WoEEncoder(), + pd.DataFrame( + {"x": ["a", "a", "b", "b", "c", "c"], "y": [1, 0, 1, 0, 1, 0]}, + index=[101, 105, 42, 76, 88, 92], + ), + ), + ], +) +def test_detect_index_mismatch_from_x_pandas_y_pandas(encoder, df_test): + """ + Created 2022-03-27 to test fix to issue # 376 + """ + + # Set up for standard pipeline/training etc. + X: pd.DataFrame = df_test[["x"]] + y: pd.Series = df_test["y"] + + # Test issue fix where indexes of pandas objects become mismatched + # y remains Series with original DataFrame index + y = y.reset_index(drop=True) + + e: Exception + with pytest.raises(Exception) as e: + encoder.fit_transform(X, y) + assert "mismatch" in e.value.args[0].lower() From fa4f14fbe27fb930852c7c93dcf22c6878837d17 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Mon, 28 Mar 2022 20:56:41 -0400 Subject: [PATCH 26/55] black/isort/flake8 --- feature_engine/encoding/decision_tree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feature_engine/encoding/decision_tree.py b/feature_engine/encoding/decision_tree.py index 6e388fb61..8046b30cb 100644 --- a/feature_engine/encoding/decision_tree.py +++ b/feature_engine/encoding/decision_tree.py @@ -7,8 +7,8 @@ from sklearn.pipeline import Pipeline from sklearn.utils.multiclass import check_classification_targets, type_of_target -from feature_engine.discretisation import DecisionTreeDiscretiser from feature_engine.dataframe_checks import _check_X_y_pd_np_mismatch +from feature_engine.discretisation import DecisionTreeDiscretiser from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, From c121ca6d703adef48340203e4a29c2a186605dd5 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Mon, 28 Mar 2022 20:58:23 -0400 Subject: [PATCH 27/55] minor flake8 --- feature_engine/dataframe_checks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index d64d9e75d..5d48eb4d7 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -139,7 +139,8 @@ def _check_X_y_pd_np_mismatch( Handles 3 cases 1. X is an ndarray and y is a Series - converts X to DataFrame with y's index 2. X is a DataFrame and y is an ndarray - converts y to Series with X's index - 3. X is a DataFrame and y is a Series, but their indexes don't match - raises an error + 3. X is a DataFrame and y is a Series, but their indexes don't match + - raises an error If both are ndarray objects, they are returned unchanged. From a02a1da2f03e643ce73961783630bd78151cf22d Mon Sep 17 00:00:00 2001 From: Noah Green Date: Tue, 29 Mar 2022 23:23:09 -0400 Subject: [PATCH 28/55] renamed _check_X_y_pd_np_mismatch() to _check_X_y() as per feedback and in preparation for new role --- feature_engine/dataframe_checks.py | 2 +- feature_engine/encoding/decision_tree.py | 4 ++-- feature_engine/encoding/mean_encoding.py | 4 ++-- feature_engine/encoding/ordinal.py | 4 ++-- feature_engine/encoding/probability_ratio.py | 4 ++-- feature_engine/encoding/woe.py | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index 5d48eb4d7..34d15ab31 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -131,7 +131,7 @@ def _check_contains_inf(X: pd.DataFrame, variables: List[Union[str, int]]) -> No ) -def _check_X_y_pd_np_mismatch( +def _check_X_y( X: Union[pd.DataFrame, np.ndarray], y: Union[pd.Series, np.ndarray], ): diff --git a/feature_engine/encoding/decision_tree.py b/feature_engine/encoding/decision_tree.py index 8046b30cb..d862a9bdd 100644 --- a/feature_engine/encoding/decision_tree.py +++ b/feature_engine/encoding/decision_tree.py @@ -7,7 +7,7 @@ from sklearn.pipeline import Pipeline from sklearn.utils.multiclass import check_classification_targets, type_of_target -from feature_engine.dataframe_checks import _check_X_y_pd_np_mismatch +from feature_engine.dataframe_checks import _check_X_y from feature_engine.discretisation import DecisionTreeDiscretiser from feature_engine.docstrings import ( Substitution, @@ -203,7 +203,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): check_classification_targets(y) # check input dataframe - X, y = _check_X_y_pd_np_mismatch(X, y) + X, y = _check_X_y(X, y) X = self._check_fit_input_and_variables(X) if self.param_grid: diff --git a/feature_engine/encoding/mean_encoding.py b/feature_engine/encoding/mean_encoding.py index de4d5aace..27c8f0e37 100644 --- a/feature_engine/encoding/mean_encoding.py +++ b/feature_engine/encoding/mean_encoding.py @@ -5,7 +5,7 @@ import pandas as pd -from feature_engine.dataframe_checks import _check_X_y_pd_np_mismatch +from feature_engine.dataframe_checks import _check_X_y from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -132,7 +132,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): The target. """ - X, y = _check_X_y_pd_np_mismatch(X, y) + X, y = _check_X_y(X, y) X = self._check_fit_input_and_variables(X) if not isinstance(y, pd.Series): diff --git a/feature_engine/encoding/ordinal.py b/feature_engine/encoding/ordinal.py index f31881738..da0e0725f 100644 --- a/feature_engine/encoding/ordinal.py +++ b/feature_engine/encoding/ordinal.py @@ -5,7 +5,7 @@ import pandas as pd -from feature_engine.dataframe_checks import _check_X_y_pd_np_mismatch +from feature_engine.dataframe_checks import _check_X_y from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -147,7 +147,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): Otherwise, y needs to be passed when fitting the transformer. """ - X, y = _check_X_y_pd_np_mismatch(X, y) + X, y = _check_X_y(X, y) X = self._check_fit_input_and_variables(X) # join target to predictor variables diff --git a/feature_engine/encoding/probability_ratio.py b/feature_engine/encoding/probability_ratio.py index d5acc7c24..cee573a84 100644 --- a/feature_engine/encoding/probability_ratio.py +++ b/feature_engine/encoding/probability_ratio.py @@ -6,7 +6,7 @@ import numpy as np import pandas as pd -from feature_engine.dataframe_checks import _check_X_y_pd_np_mismatch +from feature_engine.dataframe_checks import _check_X_y from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -155,7 +155,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): Target, must be binary. """ - X, y = _check_X_y_pd_np_mismatch(X, y) + X, y = _check_X_y(X, y) X = self._check_fit_input_and_variables(X) if not isinstance(y, pd.Series): diff --git a/feature_engine/encoding/woe.py b/feature_engine/encoding/woe.py index af180a3c8..d6951aa7a 100644 --- a/feature_engine/encoding/woe.py +++ b/feature_engine/encoding/woe.py @@ -6,7 +6,7 @@ import numpy as np import pandas as pd -from feature_engine.dataframe_checks import _check_X_y_pd_np_mismatch +from feature_engine.dataframe_checks import _check_X_y from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -137,7 +137,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): Target, must be binary. """ - X, y = _check_X_y_pd_np_mismatch(X, y) + X, y = _check_X_y(X, y) X = self._check_fit_input_and_variables(X) if not isinstance(y, pd.Series): From f90c63f93ee1a254843ec8087555264be859878c Mon Sep 17 00:00:00 2001 From: Noah Green Date: Wed, 30 Mar 2022 22:45:10 -0400 Subject: [PATCH 29/55] added numpy_to_pandas functionality, subroutines to be used in several other places. Includes unit tests. --- feature_engine/numpy_to_pandas.py | 62 +++++++++++++++++++++++++++++++ tests/test_numpy_to_pandas.py | 42 +++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 feature_engine/numpy_to_pandas.py create mode 100644 tests/test_numpy_to_pandas.py diff --git a/feature_engine/numpy_to_pandas.py b/feature_engine/numpy_to_pandas.py new file mode 100644 index 000000000..6dc21bca7 --- /dev/null +++ b/feature_engine/numpy_to_pandas.py @@ -0,0 +1,62 @@ +"""Functions to detect numpy objects and convert to pandas objects.""" + +from typing import Any, Union, List +import numpy as np +import pandas as pd + +def _is_numpy(obj_in: Any) -> bool: + """ + Tests if an object is a numpy object. + If the input is a numpy array, it converts it to a pandas Dataframe. This is mostly + so that we can add the check_estimator checks for compatibility with sklearn. + + Parameters + ---------- + obj_in : the object to test. + + Returns + ------- + True if object is a numpy object, else False + """ + return isinstance(obj_in, (np.generic, np.ndarray)) + + +def _numpy_to_dataframe(obj_in: Union[np.generic, np.ndarray], index=None) -> pd.DataFrame: + """ + Converts a numpy object to a pandas DataFrame + + Parameters + ---------- + obj_in : the object to convert + index : array-like (optional); will set index on DataFrame + + Returns + ------- + df_out : the object converted to a pandas DataFrame + """ + col_names: List[str] = [str(i) for i in range(obj_in.shape[1])] + df_out: pd.DataFrame = pd.DataFrame(obj_in, columns=col_names) + if index is not None: + df_out.index = index + + return df_out + + +def _numpy_to_series(obj_in: Union[np.generic, np.ndarray], index=None) -> pd.Series: + """ + Converts a numpy object to a pandas Series + + Parameters + ---------- + obj_in : the object to convert + index : array-like (optional); will set index on Series + + Returns + ------- + df_out : the object converted to a pandas Series + """ + s_out: pd.Series = pd.Series(obj_in) + if index is not None: + s_out.index = index + + return s_out \ No newline at end of file diff --git a/tests/test_numpy_to_pandas.py b/tests/test_numpy_to_pandas.py new file mode 100644 index 000000000..d4e07b020 --- /dev/null +++ b/tests/test_numpy_to_pandas.py @@ -0,0 +1,42 @@ +from typing import Any +import numpy as np +import pandas as pd +import pytest +from pandas.testing import assert_frame_equal, assert_series_equal + + +from feature_engine.numpy_to_pandas import ( + _is_numpy, + _numpy_to_series, + _numpy_to_dataframe +) + + +@pytest.mark.parametrize( + "obj, expected", + [ + (np.array([1, 2, 3, 4]), True), + (pd.Series([1, 2, 3, 4]), False), + ("something", False) + ] +) +def test_is_numpy(obj: Any, expected: bool): + assert _is_numpy(obj) == expected + + +def test_numpy_to_dataframe(): + np_array: np.ndarray = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) + expected: pd.DataFrame = pd.DataFrame({"0": [1, 4, 7], "1": [2, 5, 8], "2": [3, 6, 9]}) + assert_frame_equal(_numpy_to_dataframe(np_array), expected) + + expected.index = ["a", "b", "c"] + assert_frame_equal(_numpy_to_dataframe(np_array, index=["a", "b", "c"]), expected) + + +def test_numpy_to_series(): + np_array: np.ndarray = np.array([1, 2, 3]) + expected: pd.Series = pd.Series([1, 2, 3]) + assert_series_equal(_numpy_to_series(np_array), expected) + + expected.index = ["a", "b", "c"] + assert_series_equal(_numpy_to_series(np_array, index=["a", "b", "c"]), expected) \ No newline at end of file From 030c3ebf1da84fa236ed7c1f4a709b4b7bf8e70c Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sat, 2 Apr 2022 11:31:37 -0400 Subject: [PATCH 30/55] rewrote _check_X_y() to handle all specified cases; new unit test suite added for all cases and errors for _check_X_y() --- feature_engine/dataframe_checks.py | 67 ++++++++++++++++------ tests/test_dataframe_checks.py | 90 +++++++++++++++++++++++++++++- 2 files changed, 138 insertions(+), 19 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index 34d15ab31..4bd499283 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -8,6 +8,8 @@ import pandas as pd from scipy.sparse import issparse +from .numpy_to_pandas import (_is_numpy, _numpy_to_series, _numpy_to_dataframe) + def _is_dataframe(X: pd.DataFrame) -> pd.DataFrame: """ @@ -136,13 +138,17 @@ def _check_X_y( y: Union[pd.Series, np.ndarray], ): """ - Handles 3 cases - 1. X is an ndarray and y is a Series - converts X to DataFrame with y's index - 2. X is a DataFrame and y is an ndarray - converts y to Series with X's index - 3. X is a DataFrame and y is a Series, but their indexes don't match - - raises an error + Returns X as a DataFrame and y as a Series, converting any numpy + objects to pandas objects as needed. + * If both parameters are numpy objects, they are converted to pandas objects. + * If one parameter is a pandas object and the other is a numpy object, + the former will be converted to a pandas object, with the indexes + of the latter. + * If both parameters are pandas objects, and their indexes are inconsistent, + an exception is raised (i.e. this is the caller's error.) + * If both parameters are pandas objects and their indexes match, they are + returned unchanged. - If both are ndarray objects, they are returned unchanged. Parameters ---------- @@ -151,19 +157,44 @@ def _check_X_y( Returns ------- - X: changed as per description above - y: changed as per description above + X: Pandas DataFrame + y: Pandas Series + + Exceptions + ---------- + ValueError: if X and y are dimension-incompatible, X and y are pandas objects + with inconsistent indexes """ - if isinstance(X, np.ndarray) and isinstance(y, (pd.DataFrame, pd.Series)): - # already know X is not a DataFrame, use machinery in _is_dataframe() - # to correctly convert X to DataFrame - X = _is_dataframe(X) - X.index = y.index - elif isinstance(X, pd.DataFrame) and isinstance(y, np.ndarray): - y = pd.Series(y) - y.index = X.index - elif isinstance(X, pd.DataFrame) and isinstance(y, pd.Series): + + # * If both parameters are numpy objects, they are converted to pandas objects. + # * If one parameter is a pandas object and the other is a numpy object, + # the former will be converted to a pandas object, with the indexes + # of the latter. + if _is_numpy(X): + X = _numpy_to_dataframe(X, index=y.index if isinstance(y, pd.Series) else None) + if _is_numpy(y): + y = _numpy_to_series(y, index=X.index if isinstance(X, pd.DataFrame) else None) + + # * If both parameters are pandas objects, and their indexes are inconsistent, + # an exception is raised (i.e. this is the caller's error.) + # * If both parameters are pandas objects and their indexes match, they are + # returned unchanged. + if isinstance(X, pd.DataFrame) and isinstance(y, pd.Series): if not all(y.index == X.index): - raise Exception("Index mismatch between DataFrame X and Series y") + raise ValueError("Index mismatch between DataFrame X and Series y") + else: + pass # deliberately highlighting the no-op case + + # * If X is sparse or X is empty, raises an exception + # (This deliberately carries out similar tests in _is_dataframe() above in + # order to support different code paths) + if issparse(X): + raise ValueError("This transformer does not support sparse matrices.") + + if X.empty: + raise ValueError( + "0 feature(s) (shape=%s) while a minimum of %d is " + "required." % (X.shape, 1) + ) return X, y diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 396cefcdd..6c92b76c1 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -1,10 +1,14 @@ import pytest -from pandas.testing import assert_frame_equal +import contextlib +import numpy as np +import pandas as pd +from pandas.testing import assert_frame_equal, assert_series_equal from feature_engine.dataframe_checks import ( _check_contains_na, _check_input_matches_training_df, _is_dataframe, + _check_X_y ) @@ -22,3 +26,87 @@ def test_check_input_matches_training_df(df_vartypes): def test_contains_na(df_na): with pytest.raises(ValueError): assert _check_contains_na(df_na, ["Name", "City"]) + + +@pytest.mark.parametrize( + "X_in, y_in, expected_1, expected_2, exception_type, exception_match", + [ + # * If both parameters are numpy objects, + # they are converted to pandas objects. + ( + np.array([[1, 2, 3, 4], [5, 6, 7, 8]]).T, + np.array([1, 2, 3, 4]), + pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}), + pd.Series([1, 2, 3, 4]), + None, + None + ), + + # * If one parameter is a numpy object and the + # other is a pandas object, the former will be + # converted to a pandas object, with the indexes + # of the latter. + ( + pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]), + np.array([1, 2, 3, 4]), + pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]), + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), + None, + None + ), + ( + np.array([[1, 2, 3, 4], [5, 6, 7, 8]]).T, + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), + pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]), + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), + None, + None + ), + + # * If both parameters are pandas objects, and their + # indexes are inconsistent, an exception is raised + # (i.e.this is the caller's error.) + ( + pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]), + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 999]), + None, + None, + ValueError, + ".*Index.*" + ), + + # * If both parameters are pandas objects and their indexes match, they are + # returned unchanged. + ( + pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]), + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), + None, + None, + None, + None + ), + ] +) +def test_check_X_y(X_in, y_in, expected_1, expected_2, exception_type, exception_match): + with ( + contextlib.nullcontext() if not exception_type + else pytest.raises(exception_type, match=exception_match) + ): + # Execute - can throw here (non-null exception_type will expect exception) + X_out, y_out = _check_X_y(X_in, y_in) + + # Test X output + if expected_1 is None: + assert X_out is X_in + elif isinstance(expected_1, pd.DataFrame): + assert_frame_equal(X_out, expected_1) + elif isinstance(expected_1, (np.generic, np.ndarray)): + assert all(X_out == expected_1) + + # Test y output + if expected_2 is None: + assert y_out is y_in + elif isinstance(expected_2, pd.Series): + assert_series_equal(y_out, expected_2) + elif isinstance(expected_2, (np.generic, np.ndarray)): + assert all(y_out == expected_2) \ No newline at end of file From e1abfd882d0bea65249223415eda6dbc6629a72a Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sat, 2 Apr 2022 11:33:38 -0400 Subject: [PATCH 31/55] base encoder: remove is_dataframe from _check_fit_input_and_variables, which now returns self --- feature_engine/encoding/base_encoder.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/feature_engine/encoding/base_encoder.py b/feature_engine/encoding/base_encoder.py index 2645a56f6..666813dc2 100644 --- a/feature_engine/encoding/base_encoder.py +++ b/feature_engine/encoding/base_encoder.py @@ -80,9 +80,6 @@ def _check_fit_input_and_variables(self, X: pd.DataFrame) -> pd.DataFrame: list of categorical variables """ - # check input dataframe - X = _is_dataframe(X) - if not self.ignore_format: # find categorical variables or check variables entered by user are object self.variables_: List[ @@ -101,7 +98,7 @@ def _check_fit_input_and_variables(self, X: pd.DataFrame) -> pd.DataFrame: # save train set shape self.n_features_in_ = X.shape[1] - return X + return self def _check_transform_input_and_state(self, X: pd.DataFrame) -> pd.DataFrame: """ From d0c440196085d8743da64567131325bc113cf586 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sat, 2 Apr 2022 11:38:01 -0400 Subject: [PATCH 32/55] for the encoders that do not require y, add is_dataframe() when needed --- feature_engine/encoding/count_frequency.py | 4 +++- feature_engine/encoding/one_hot.py | 4 +++- feature_engine/encoding/rare_label.py | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/feature_engine/encoding/count_frequency.py b/feature_engine/encoding/count_frequency.py index 420b97b40..31c9d9fcb 100644 --- a/feature_engine/encoding/count_frequency.py +++ b/feature_engine/encoding/count_frequency.py @@ -5,6 +5,7 @@ import pandas as pd +from feature_engine.dataframe_checks import _is_dataframe from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -139,7 +140,8 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): y is not needed in this encoder. You can pass y or None. """ - X = self._check_fit_input_and_variables(X) + X = _is_dataframe(X) + self._check_fit_input_and_variables(X) self.encoder_dict_ = {} diff --git a/feature_engine/encoding/one_hot.py b/feature_engine/encoding/one_hot.py index 9da3dde30..71426cba2 100644 --- a/feature_engine/encoding/one_hot.py +++ b/feature_engine/encoding/one_hot.py @@ -7,6 +7,7 @@ import pandas as pd from sklearn.utils.validation import check_is_fitted +from feature_engine.dataframe_checks import _is_dataframe from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -180,7 +181,8 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): None. """ - X = self._check_fit_input_and_variables(X) + X = _is_dataframe(X) + self._check_fit_input_and_variables(X) self.encoder_dict_ = {} diff --git a/feature_engine/encoding/rare_label.py b/feature_engine/encoding/rare_label.py index e891398b9..4b14adbff 100644 --- a/feature_engine/encoding/rare_label.py +++ b/feature_engine/encoding/rare_label.py @@ -7,6 +7,7 @@ import numpy as np import pandas as pd +from feature_engine.dataframe_checks import _is_dataframe from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -147,7 +148,8 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): y is not required. You can pass y or None. """ - X = self._check_fit_input_and_variables(X) + X = _is_dataframe(X) + self._check_fit_input_and_variables(X) self.encoder_dict_ = {} From 81613b615175e656fc828a5832943e960553be7c Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sat, 2 Apr 2022 11:39:59 -0400 Subject: [PATCH 33/55] for the encoders that do require y, check_X_y() handles everything, including turning everything into pandas objects --- feature_engine/encoding/decision_tree.py | 2 +- feature_engine/encoding/mean_encoding.py | 2 +- feature_engine/encoding/probability_ratio.py | 2 +- feature_engine/encoding/woe.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/feature_engine/encoding/decision_tree.py b/feature_engine/encoding/decision_tree.py index d862a9bdd..f6b069292 100644 --- a/feature_engine/encoding/decision_tree.py +++ b/feature_engine/encoding/decision_tree.py @@ -204,7 +204,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): # check input dataframe X, y = _check_X_y(X, y) - X = self._check_fit_input_and_variables(X) + self._check_fit_input_and_variables(X) if self.param_grid: param_grid = self.param_grid diff --git a/feature_engine/encoding/mean_encoding.py b/feature_engine/encoding/mean_encoding.py index 27c8f0e37..5af25dffc 100644 --- a/feature_engine/encoding/mean_encoding.py +++ b/feature_engine/encoding/mean_encoding.py @@ -133,7 +133,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): """ X, y = _check_X_y(X, y) - X = self._check_fit_input_and_variables(X) + self._check_fit_input_and_variables(X) if not isinstance(y, pd.Series): y = pd.Series(y) diff --git a/feature_engine/encoding/probability_ratio.py b/feature_engine/encoding/probability_ratio.py index cee573a84..48ee027ef 100644 --- a/feature_engine/encoding/probability_ratio.py +++ b/feature_engine/encoding/probability_ratio.py @@ -156,7 +156,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): """ X, y = _check_X_y(X, y) - X = self._check_fit_input_and_variables(X) + self._check_fit_input_and_variables(X) if not isinstance(y, pd.Series): y = pd.Series(y) diff --git a/feature_engine/encoding/woe.py b/feature_engine/encoding/woe.py index d6951aa7a..b18366bf8 100644 --- a/feature_engine/encoding/woe.py +++ b/feature_engine/encoding/woe.py @@ -138,7 +138,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): """ X, y = _check_X_y(X, y) - X = self._check_fit_input_and_variables(X) + self._check_fit_input_and_variables(X) if not isinstance(y, pd.Series): y = pd.Series(y) From 3788b23399c9862f3e699d4c2927fe02fdc5aed6 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sat, 2 Apr 2022 11:44:26 -0400 Subject: [PATCH 34/55] OrdinalEncoder is special, because it should work with and without y --- feature_engine/encoding/ordinal.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/feature_engine/encoding/ordinal.py b/feature_engine/encoding/ordinal.py index da0e0725f..845d26a9e 100644 --- a/feature_engine/encoding/ordinal.py +++ b/feature_engine/encoding/ordinal.py @@ -4,8 +4,9 @@ from typing import List, Optional, Union import pandas as pd +from sklearn.utils import check_X_y -from feature_engine.dataframe_checks import _check_X_y +from feature_engine.dataframe_checks import _check_X_y, _is_dataframe from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -23,6 +24,9 @@ from feature_engine.encoding.base_encoder import BaseCategorical + + + @Substitution( ignore_format=_ignore_format_docstring, variables=_variables_docstring, @@ -147,8 +151,12 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): Otherwise, y needs to be passed when fitting the transformer. """ - X, y = _check_X_y(X, y) - X = self._check_fit_input_and_variables(X) + # All dimension, type, etc. checking + if self.encoding_method == "ordered": + X, y = _check_X_y(X, y) + else: + X = _is_dataframe(X) + self._check_fit_input_and_variables(X) # join target to predictor variables if self.encoding_method == "ordered": From a2dc4946716a8bffc7ea6b06569b8bb028e44951 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sat, 2 Apr 2022 12:24:33 -0400 Subject: [PATCH 35/55] removed unneccessary checking/converting of y to Series now handled by _check_X_y() --- feature_engine/encoding/mean_encoding.py | 3 --- feature_engine/encoding/ordinal.py | 3 --- feature_engine/encoding/probability_ratio.py | 3 --- feature_engine/encoding/woe.py | 3 --- 4 files changed, 12 deletions(-) diff --git a/feature_engine/encoding/mean_encoding.py b/feature_engine/encoding/mean_encoding.py index 5af25dffc..ed7f3f1b9 100644 --- a/feature_engine/encoding/mean_encoding.py +++ b/feature_engine/encoding/mean_encoding.py @@ -135,9 +135,6 @@ def fit(self, X: pd.DataFrame, y: pd.Series): X, y = _check_X_y(X, y) self._check_fit_input_and_variables(X) - if not isinstance(y, pd.Series): - y = pd.Series(y) - temp = pd.concat([X, y], axis=1) temp.columns = list(X.columns) + ["target"] diff --git a/feature_engine/encoding/ordinal.py b/feature_engine/encoding/ordinal.py index 845d26a9e..4f5619807 100644 --- a/feature_engine/encoding/ordinal.py +++ b/feature_engine/encoding/ordinal.py @@ -163,9 +163,6 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): if y is None: raise ValueError("Please provide a target y for this encoding method") - if not isinstance(y, pd.Series): - y = pd.Series(y) - temp = pd.concat([X, y], axis=1) temp.columns = list(X.columns) + ["target"] diff --git a/feature_engine/encoding/probability_ratio.py b/feature_engine/encoding/probability_ratio.py index 48ee027ef..3ba437185 100644 --- a/feature_engine/encoding/probability_ratio.py +++ b/feature_engine/encoding/probability_ratio.py @@ -158,9 +158,6 @@ def fit(self, X: pd.DataFrame, y: pd.Series): X, y = _check_X_y(X, y) self._check_fit_input_and_variables(X) - if not isinstance(y, pd.Series): - y = pd.Series(y) - # check that y is binary if y.nunique() != 2: raise ValueError( diff --git a/feature_engine/encoding/woe.py b/feature_engine/encoding/woe.py index b18366bf8..2724170c0 100644 --- a/feature_engine/encoding/woe.py +++ b/feature_engine/encoding/woe.py @@ -140,9 +140,6 @@ def fit(self, X: pd.DataFrame, y: pd.Series): X, y = _check_X_y(X, y) self._check_fit_input_and_variables(X) - if not isinstance(y, pd.Series): - y = pd.Series(y) - # check that y is binary if y.nunique() != 2: raise ValueError( From 7e00ef5883cc96e6f05bec7d1e10af1c21dde207 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sat, 2 Apr 2022 19:11:42 -0400 Subject: [PATCH 36/55] _is_dataframe() now uses functionality/subroutines from numpy_to_pandas.py for convenience and standardization --- feature_engine/dataframe_checks.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index 4bd499283..a336a2cda 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -35,9 +35,8 @@ def _is_dataframe(X: pd.DataFrame) -> pd.DataFrame: """ # check_estimator uses numpy arrays for its checks. # Thus, we need to allow np arrays - if isinstance(X, (np.generic, np.ndarray)): - col_names = [str(i) for i in range(X.shape[1])] - X = pd.DataFrame(X, columns=col_names) + if _is_numpy(X): + X = _numpy_to_dataframe(X) if issparse(X): raise ValueError("This transformer does not support sparse matrices.") From ccfea670999f3e4efcbb81857c273ac5f84d8f0c Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sat, 2 Apr 2022 19:15:35 -0400 Subject: [PATCH 37/55] renamed the new _check_X_y() to _check_pd_X_y(), in order to avoid name confusion with check_X_y() from sklearn.utils --- feature_engine/dataframe_checks.py | 2 +- feature_engine/encoding/decision_tree.py | 4 ++-- feature_engine/encoding/mean_encoding.py | 4 ++-- feature_engine/encoding/ordinal.py | 4 ++-- feature_engine/encoding/probability_ratio.py | 4 ++-- feature_engine/encoding/woe.py | 4 ++-- tests/test_dataframe_checks.py | 6 +++--- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index a336a2cda..da26feeeb 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -132,7 +132,7 @@ def _check_contains_inf(X: pd.DataFrame, variables: List[Union[str, int]]) -> No ) -def _check_X_y( +def _check_pd_X_y( X: Union[pd.DataFrame, np.ndarray], y: Union[pd.Series, np.ndarray], ): diff --git a/feature_engine/encoding/decision_tree.py b/feature_engine/encoding/decision_tree.py index f6b069292..db7d8bf33 100644 --- a/feature_engine/encoding/decision_tree.py +++ b/feature_engine/encoding/decision_tree.py @@ -7,7 +7,7 @@ from sklearn.pipeline import Pipeline from sklearn.utils.multiclass import check_classification_targets, type_of_target -from feature_engine.dataframe_checks import _check_X_y +from feature_engine.dataframe_checks import _check_pd_X_y from feature_engine.discretisation import DecisionTreeDiscretiser from feature_engine.docstrings import ( Substitution, @@ -203,7 +203,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): check_classification_targets(y) # check input dataframe - X, y = _check_X_y(X, y) + X, y = _check_pd_X_y(X, y) self._check_fit_input_and_variables(X) if self.param_grid: diff --git a/feature_engine/encoding/mean_encoding.py b/feature_engine/encoding/mean_encoding.py index ed7f3f1b9..741291b41 100644 --- a/feature_engine/encoding/mean_encoding.py +++ b/feature_engine/encoding/mean_encoding.py @@ -5,7 +5,7 @@ import pandas as pd -from feature_engine.dataframe_checks import _check_X_y +from feature_engine.dataframe_checks import _check_pd_X_y from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -132,7 +132,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): The target. """ - X, y = _check_X_y(X, y) + X, y = _check_pd_X_y(X, y) self._check_fit_input_and_variables(X) temp = pd.concat([X, y], axis=1) diff --git a/feature_engine/encoding/ordinal.py b/feature_engine/encoding/ordinal.py index 4f5619807..768286974 100644 --- a/feature_engine/encoding/ordinal.py +++ b/feature_engine/encoding/ordinal.py @@ -6,7 +6,7 @@ import pandas as pd from sklearn.utils import check_X_y -from feature_engine.dataframe_checks import _check_X_y, _is_dataframe +from feature_engine.dataframe_checks import _check_pd_X_y, _is_dataframe from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -153,7 +153,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # All dimension, type, etc. checking if self.encoding_method == "ordered": - X, y = _check_X_y(X, y) + X, y = _check_pd_X_y(X, y) else: X = _is_dataframe(X) self._check_fit_input_and_variables(X) diff --git a/feature_engine/encoding/probability_ratio.py b/feature_engine/encoding/probability_ratio.py index 3ba437185..a3affb30a 100644 --- a/feature_engine/encoding/probability_ratio.py +++ b/feature_engine/encoding/probability_ratio.py @@ -6,7 +6,7 @@ import numpy as np import pandas as pd -from feature_engine.dataframe_checks import _check_X_y +from feature_engine.dataframe_checks import _check_pd_X_y from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -155,7 +155,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): Target, must be binary. """ - X, y = _check_X_y(X, y) + X, y = _check_pd_X_y(X, y) self._check_fit_input_and_variables(X) # check that y is binary diff --git a/feature_engine/encoding/woe.py b/feature_engine/encoding/woe.py index 2724170c0..d07bd4dbd 100644 --- a/feature_engine/encoding/woe.py +++ b/feature_engine/encoding/woe.py @@ -6,7 +6,7 @@ import numpy as np import pandas as pd -from feature_engine.dataframe_checks import _check_X_y +from feature_engine.dataframe_checks import _check_pd_X_y from feature_engine.docstrings import ( Substitution, _feature_names_in_docstring, @@ -137,7 +137,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): Target, must be binary. """ - X, y = _check_X_y(X, y) + X, y = _check_pd_X_y(X, y) self._check_fit_input_and_variables(X) # check that y is binary diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 6c92b76c1..36f0279dd 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -8,7 +8,7 @@ _check_contains_na, _check_input_matches_training_df, _is_dataframe, - _check_X_y + _check_pd_X_y ) @@ -87,13 +87,13 @@ def test_contains_na(df_na): ), ] ) -def test_check_X_y(X_in, y_in, expected_1, expected_2, exception_type, exception_match): +def test_check_pd_X_y(X_in, y_in, expected_1, expected_2, exception_type, exception_match): with ( contextlib.nullcontext() if not exception_type else pytest.raises(exception_type, match=exception_match) ): # Execute - can throw here (non-null exception_type will expect exception) - X_out, y_out = _check_X_y(X_in, y_in) + X_out, y_out = _check_pd_X_y(X_in, y_in) # Test X output if expected_1 is None: From 784dd21b25f741baf4a80db6ac8bd23beb2c4403 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sat, 2 Apr 2022 19:25:41 -0400 Subject: [PATCH 38/55] black/isort/flake8 --- feature_engine/dataframe_checks.py | 2 +- feature_engine/encoding/ordinal.py | 4 --- feature_engine/numpy_to_pandas.py | 10 ++++-- tests/test_dataframe_checks.py | 49 ++++++++++++++++++------------ 4 files changed, 38 insertions(+), 27 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index da26feeeb..8128c76ce 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -8,7 +8,7 @@ import pandas as pd from scipy.sparse import issparse -from .numpy_to_pandas import (_is_numpy, _numpy_to_series, _numpy_to_dataframe) +from .numpy_to_pandas import _is_numpy, _numpy_to_dataframe, _numpy_to_series def _is_dataframe(X: pd.DataFrame) -> pd.DataFrame: diff --git a/feature_engine/encoding/ordinal.py b/feature_engine/encoding/ordinal.py index 768286974..36223e407 100644 --- a/feature_engine/encoding/ordinal.py +++ b/feature_engine/encoding/ordinal.py @@ -4,7 +4,6 @@ from typing import List, Optional, Union import pandas as pd -from sklearn.utils import check_X_y from feature_engine.dataframe_checks import _check_pd_X_y, _is_dataframe from feature_engine.docstrings import ( @@ -24,9 +23,6 @@ from feature_engine.encoding.base_encoder import BaseCategorical - - - @Substitution( ignore_format=_ignore_format_docstring, variables=_variables_docstring, diff --git a/feature_engine/numpy_to_pandas.py b/feature_engine/numpy_to_pandas.py index 6dc21bca7..bb9fbd381 100644 --- a/feature_engine/numpy_to_pandas.py +++ b/feature_engine/numpy_to_pandas.py @@ -1,9 +1,11 @@ """Functions to detect numpy objects and convert to pandas objects.""" -from typing import Any, Union, List +from typing import Any, List, Union + import numpy as np import pandas as pd + def _is_numpy(obj_in: Any) -> bool: """ Tests if an object is a numpy object. @@ -21,7 +23,9 @@ def _is_numpy(obj_in: Any) -> bool: return isinstance(obj_in, (np.generic, np.ndarray)) -def _numpy_to_dataframe(obj_in: Union[np.generic, np.ndarray], index=None) -> pd.DataFrame: +def _numpy_to_dataframe( + obj_in: Union[np.generic, np.ndarray], index=None +) -> pd.DataFrame: """ Converts a numpy object to a pandas DataFrame @@ -59,4 +63,4 @@ def _numpy_to_series(obj_in: Union[np.generic, np.ndarray], index=None) -> pd.Se if index is not None: s_out.index = index - return s_out \ No newline at end of file + return s_out diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 36f0279dd..2978b1c7f 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -1,14 +1,15 @@ -import pytest import contextlib + import numpy as np import pandas as pd +import pytest from pandas.testing import assert_frame_equal, assert_series_equal from feature_engine.dataframe_checks import ( _check_contains_na, _check_input_matches_training_df, + _check_pd_X_y, _is_dataframe, - _check_pd_X_y ) @@ -39,57 +40,67 @@ def test_contains_na(df_na): pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}), pd.Series([1, 2, 3, 4]), None, - None + None, ), - # * If one parameter is a numpy object and the # other is a pandas object, the former will be # converted to a pandas object, with the indexes # of the latter. ( - pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]), + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), np.array([1, 2, 3, 4]), - pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]), + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), None, - None + None, ), ( np.array([[1, 2, 3, 4], [5, 6, 7, 8]]).T, pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), - pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]), + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), None, - None + None, ), - # * If both parameters are pandas objects, and their # indexes are inconsistent, an exception is raised # (i.e.this is the caller's error.) ( - pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]), + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), pd.Series([1, 2, 3, 4], index=[22, 99, 101, 999]), None, None, ValueError, - ".*Index.*" + ".*Index.*", ), - # * If both parameters are pandas objects and their indexes match, they are # returned unchanged. ( - pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212]), + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), None, None, None, - None + None, ), - ] + ], ) -def test_check_pd_X_y(X_in, y_in, expected_1, expected_2, exception_type, exception_match): +def test_check_pd_X_y( + X_in, y_in, expected_1, expected_2, exception_type, exception_match +): with ( - contextlib.nullcontext() if not exception_type + contextlib.nullcontext() + if not exception_type else pytest.raises(exception_type, match=exception_match) ): # Execute - can throw here (non-null exception_type will expect exception) @@ -109,4 +120,4 @@ def test_check_pd_X_y(X_in, y_in, expected_1, expected_2, exception_type, except elif isinstance(expected_2, pd.Series): assert_series_equal(y_out, expected_2) elif isinstance(expected_2, (np.generic, np.ndarray)): - assert all(y_out == expected_2) \ No newline at end of file + assert all(y_out == expected_2) From d5ea00c3ba9b4b90e077934cb0aee560c80ef9c2 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sat, 2 Apr 2022 19:27:33 -0400 Subject: [PATCH 39/55] additional flake8 --- tests/test_numpy_to_pandas.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_numpy_to_pandas.py b/tests/test_numpy_to_pandas.py index d4e07b020..743f0cd78 100644 --- a/tests/test_numpy_to_pandas.py +++ b/tests/test_numpy_to_pandas.py @@ -26,7 +26,9 @@ def test_is_numpy(obj: Any, expected: bool): def test_numpy_to_dataframe(): np_array: np.ndarray = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) - expected: pd.DataFrame = pd.DataFrame({"0": [1, 4, 7], "1": [2, 5, 8], "2": [3, 6, 9]}) + expected: pd.DataFrame = pd.DataFrame( + {"0": [1, 4, 7], "1": [2, 5, 8], "2": [3, 6, 9]} + ) assert_frame_equal(_numpy_to_dataframe(np_array), expected) expected.index = ["a", "b", "c"] @@ -39,4 +41,5 @@ def test_numpy_to_series(): assert_series_equal(_numpy_to_series(np_array), expected) expected.index = ["a", "b", "c"] - assert_series_equal(_numpy_to_series(np_array, index=["a", "b", "c"]), expected) \ No newline at end of file + assert_series_equal(_numpy_to_series(np_array, index=["a", "b", "c"]), expected) + From c3e8fd839303a0dd50cefe787ddb206e6dfc63a5 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sat, 2 Apr 2022 19:29:17 -0400 Subject: [PATCH 40/55] minor flake8 --- tests/test_numpy_to_pandas.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_numpy_to_pandas.py b/tests/test_numpy_to_pandas.py index 743f0cd78..02e7959c6 100644 --- a/tests/test_numpy_to_pandas.py +++ b/tests/test_numpy_to_pandas.py @@ -42,4 +42,3 @@ def test_numpy_to_series(): expected.index = ["a", "b", "c"] assert_series_equal(_numpy_to_series(np_array, index=["a", "b", "c"]), expected) - From ad2321503b12b6b3103399860329bb53f0cb3c6e Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sat, 2 Apr 2022 19:31:12 -0400 Subject: [PATCH 41/55] cleaned up some cruftiness in numpy_to_pandas --- feature_engine/numpy_to_pandas.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/feature_engine/numpy_to_pandas.py b/feature_engine/numpy_to_pandas.py index bb9fbd381..c7328b742 100644 --- a/feature_engine/numpy_to_pandas.py +++ b/feature_engine/numpy_to_pandas.py @@ -39,9 +39,11 @@ def _numpy_to_dataframe( df_out : the object converted to a pandas DataFrame """ col_names: List[str] = [str(i) for i in range(obj_in.shape[1])] - df_out: pd.DataFrame = pd.DataFrame(obj_in, columns=col_names) - if index is not None: - df_out.index = index + df_out: pd.DataFrame = pd.DataFrame( + obj_in, + columns=col_names, + index=index + ) return df_out @@ -59,8 +61,6 @@ def _numpy_to_series(obj_in: Union[np.generic, np.ndarray], index=None) -> pd.Se ------- df_out : the object converted to a pandas Series """ - s_out: pd.Series = pd.Series(obj_in) - if index is not None: - s_out.index = index + s_out: pd.Series = pd.Series(obj_in, index=index) return s_out From 2337ca1920249502ce9134fe656012db6767e935 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sat, 2 Apr 2022 19:32:00 -0400 Subject: [PATCH 42/55] black/isort/flake8 --- feature_engine/numpy_to_pandas.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/feature_engine/numpy_to_pandas.py b/feature_engine/numpy_to_pandas.py index c7328b742..afefd20a9 100644 --- a/feature_engine/numpy_to_pandas.py +++ b/feature_engine/numpy_to_pandas.py @@ -39,11 +39,7 @@ def _numpy_to_dataframe( df_out : the object converted to a pandas DataFrame """ col_names: List[str] = [str(i) for i in range(obj_in.shape[1])] - df_out: pd.DataFrame = pd.DataFrame( - obj_in, - columns=col_names, - index=index - ) + df_out: pd.DataFrame = pd.DataFrame(obj_in, columns=col_names, index=index) return df_out From 5156bd1f6105faaa21b968eb4ffdb33a2e1f7f89 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sat, 2 Apr 2022 20:59:48 -0400 Subject: [PATCH 43/55] fixed incorrect merge that I had done after git pull upstream --- .../test_check_estimator_encoders.py | 45 ++++++++++--------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index 062c699fe..a744e0fd0 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -1,6 +1,8 @@ +import pytest + import numpy as np import pandas as pd -import pytest + from sklearn.utils.estimator_checks import check_estimator from feature_engine.encoding import ( @@ -31,26 +33,27 @@ PRatioEncoder(ignore_format=True), ] -@pytest.mark.parametrize( - "Estimator", - [ - CountFrequencyEncoder(ignore_format=True), - DecisionTreeEncoder(regression=False, ignore_format=True), - MeanEncoder(ignore_format=True), - OneHotEncoder(ignore_format=True), - OrdinalEncoder(ignore_format=True), - RareLabelEncoder( - tol=0.00000000001, - n_categories=100000000000, - replace_with=10, - ignore_format=True, - ), - WoEEncoder(ignore_format=True), - PRatioEncoder(ignore_format=True), - ], -) -def test_all_transformers(Estimator): - return check_estimator(Estimator) + +@pytest.mark.parametrize("estimator", _estimators) +def test_check_estimator_from_sklearn(estimator): + return check_estimator(estimator) + + +_estimators = [ + CountFrequencyEncoder(), + DecisionTreeEncoder(regression=False), + MeanEncoder(), + OneHotEncoder(), + OrdinalEncoder(), + RareLabelEncoder(), + WoEEncoder(), + PRatioEncoder(), +] + + +@pytest.mark.parametrize("estimator", _estimators) +def test_check_estimator_from_feature_engine(estimator): + return check_feature_engine_estimator(estimator) @pytest.mark.parametrize( From 869d114356ceee59596827abd3a560c53e25efcc Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sat, 2 Apr 2022 21:03:51 -0400 Subject: [PATCH 44/55] black/isort/flake8 --- tests/test_encoding/test_check_estimator_encoders.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_encoding/test_check_estimator_encoders.py b/tests/test_encoding/test_check_estimator_encoders.py index a744e0fd0..16f70bbfb 100644 --- a/tests/test_encoding/test_check_estimator_encoders.py +++ b/tests/test_encoding/test_check_estimator_encoders.py @@ -1,8 +1,6 @@ -import pytest - import numpy as np import pandas as pd - +import pytest from sklearn.utils.estimator_checks import check_estimator from feature_engine.encoding import ( From 1d05f95979256d0b267342e3b32c65febc7b659f Mon Sep 17 00:00:00 2001 From: Noah Green Date: Sat, 2 Apr 2022 21:14:23 -0400 Subject: [PATCH 45/55] fixes to _check_pd_X_y() to fix unit test after git pull upstream and rebase --- feature_engine/dataframe_checks.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index 8128c76ce..1a4a6ef1c 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -147,7 +147,8 @@ def _check_pd_X_y( an exception is raised (i.e. this is the caller's error.) * If both parameters are pandas objects and their indexes match, they are returned unchanged. - + * If X is sparse or X is empty or, after all transforms, is stiil + not a DataFrame, raises an exception Parameters ---------- @@ -184,12 +185,18 @@ def _check_pd_X_y( else: pass # deliberately highlighting the no-op case - # * If X is sparse or X is empty, raises an exception + # * If X is sparse or X is empty or, after all transforms, is stiil + # not a DataFrame, raises an exception # (This deliberately carries out similar tests in _is_dataframe() above in # order to support different code paths) if issparse(X): raise ValueError("This transformer does not support sparse matrices.") + if not isinstance(X, pd.DataFrame): + raise TypeError( + "X is not a pandas dataframe. The dataset should be a pandas dataframe." + ) + if X.empty: raise ValueError( "0 feature(s) (shape=%s) while a minimum of %d is " From 4f3c343e8e04641de842178aefdc453e9ab299f5 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Mon, 4 Apr 2022 12:18:37 -0400 Subject: [PATCH 46/55] broke up test_check_pd_X_y() into 3 separate test functions: test_check_pd_X_y_both_same_type(), test_check_pd_X_y_np_to_pd(), and test_check_pd_X_y_errors() --- tests/test_dataframe_checks.py | 111 ++++++++++++++++++++------------- 1 file changed, 68 insertions(+), 43 deletions(-) diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 2978b1c7f..0947b30aa 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -30,7 +30,7 @@ def test_contains_na(df_na): @pytest.mark.parametrize( - "X_in, y_in, expected_1, expected_2, exception_type, exception_match", + "X_in, y_in, expected_1, expected_2", [ # * If both parameters are numpy objects, # they are converted to pandas objects. @@ -39,9 +39,44 @@ def test_contains_na(df_na): np.array([1, 2, 3, 4]), pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}), pd.Series([1, 2, 3, 4]), + ), + # * If both parameters are pandas objects and their indexes match, they are + # returned unchanged. + ( + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), None, None, ), + ], +) +def test_check_pd_X_y_both_same_type( + X_in, y_in, expected_1, expected_2 +): + # Execute + X_out, y_out = _check_pd_X_y(X_in, y_in) + + # Test X output + if expected_1 is None: + assert X_out is X_in + elif isinstance(expected_1, pd.DataFrame): + assert_frame_equal(X_out, expected_1) + elif isinstance(expected_1, (np.generic, np.ndarray)): + assert all(X_out == expected_1) + + # Test y output + if expected_2 is None: + assert y_out is y_in + elif isinstance(expected_2, pd.Series): + assert_series_equal(y_out, expected_2) + elif isinstance(expected_2, (np.generic, np.ndarray)): + assert all(y_out == expected_2) + +@pytest.mark.parametrize( + "X_in, y_in, expected_1, expected_2", + [ # * If one parameter is a numpy object and the # other is a pandas object, the former will be # converted to a pandas object, with the indexes @@ -55,8 +90,6 @@ def test_contains_na(df_na): {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] ), pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), - None, - None, ), ( np.array([[1, 2, 3, 4], [5, 6, 7, 8]]).T, @@ -65,9 +98,35 @@ def test_contains_na(df_na): {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] ), pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), - None, - None, ), + ], +) +def test_check_pd_X_y_np_to_pd( + X_in, y_in, expected_1, expected_2 +): + # Execute + X_out, y_out = _check_pd_X_y(X_in, y_in) + + # Test X output + if expected_1 is None: + assert X_out is X_in + elif isinstance(expected_1, pd.DataFrame): + assert_frame_equal(X_out, expected_1) + elif isinstance(expected_1, (np.generic, np.ndarray)): + assert all(X_out == expected_1) + + # Test y output + if expected_2 is None: + assert y_out is y_in + elif isinstance(expected_2, pd.Series): + assert_series_equal(y_out, expected_2) + elif isinstance(expected_2, (np.generic, np.ndarray)): + assert all(y_out == expected_2) + + +@pytest.mark.parametrize( + "X_in, y_in, exception_type, exception_match", + [ # * If both parameters are pandas objects, and their # indexes are inconsistent, an exception is raised # (i.e.this is the caller's error.) @@ -76,48 +135,14 @@ def test_contains_na(df_na): {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] ), pd.Series([1, 2, 3, 4], index=[22, 99, 101, 999]), - None, - None, ValueError, ".*Index.*", ), - # * If both parameters are pandas objects and their indexes match, they are - # returned unchanged. - ( - pd.DataFrame( - {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] - ), - pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), - None, - None, - None, - None, - ), ], ) -def test_check_pd_X_y( - X_in, y_in, expected_1, expected_2, exception_type, exception_match +def test_check_pd_X_y_errors( + X_in, y_in, exception_type, exception_match ): - with ( - contextlib.nullcontext() - if not exception_type - else pytest.raises(exception_type, match=exception_match) - ): + with (pytest.raises(exception_type, match=exception_match)): # Execute - can throw here (non-null exception_type will expect exception) - X_out, y_out = _check_pd_X_y(X_in, y_in) - - # Test X output - if expected_1 is None: - assert X_out is X_in - elif isinstance(expected_1, pd.DataFrame): - assert_frame_equal(X_out, expected_1) - elif isinstance(expected_1, (np.generic, np.ndarray)): - assert all(X_out == expected_1) - - # Test y output - if expected_2 is None: - assert y_out is y_in - elif isinstance(expected_2, pd.Series): - assert_series_equal(y_out, expected_2) - elif isinstance(expected_2, (np.generic, np.ndarray)): - assert all(y_out == expected_2) + X_out, y_out = _check_pd_X_y(X_in, y_in) \ No newline at end of file From 07e7a184971217d0c4303c5e9c022fcf49fac5c4 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Mon, 4 Apr 2022 12:19:36 -0400 Subject: [PATCH 47/55] black/isort/flake8 --- tests/test_dataframe_checks.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 0947b30aa..24a489953 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -1,5 +1,3 @@ -import contextlib - import numpy as np import pandas as pd import pytest @@ -52,9 +50,7 @@ def test_contains_na(df_na): ), ], ) -def test_check_pd_X_y_both_same_type( - X_in, y_in, expected_1, expected_2 -): +def test_check_pd_X_y_both_same_type(X_in, y_in, expected_1, expected_2): # Execute X_out, y_out = _check_pd_X_y(X_in, y_in) @@ -74,6 +70,7 @@ def test_check_pd_X_y_both_same_type( elif isinstance(expected_2, (np.generic, np.ndarray)): assert all(y_out == expected_2) + @pytest.mark.parametrize( "X_in, y_in, expected_1, expected_2", [ @@ -101,9 +98,7 @@ def test_check_pd_X_y_both_same_type( ), ], ) -def test_check_pd_X_y_np_to_pd( - X_in, y_in, expected_1, expected_2 -): +def test_check_pd_X_y_np_to_pd(X_in, y_in, expected_1, expected_2): # Execute X_out, y_out = _check_pd_X_y(X_in, y_in) @@ -140,9 +135,7 @@ def test_check_pd_X_y_np_to_pd( ), ], ) -def test_check_pd_X_y_errors( - X_in, y_in, exception_type, exception_match -): +def test_check_pd_X_y_errors(X_in, y_in, exception_type, exception_match): with (pytest.raises(exception_type, match=exception_match)): # Execute - can throw here (non-null exception_type will expect exception) - X_out, y_out = _check_pd_X_y(X_in, y_in) \ No newline at end of file + X_out, y_out = _check_pd_X_y(X_in, y_in) From e96af8633b0047c40da7119ee41ba9024365c09e Mon Sep 17 00:00:00 2001 From: Noah Green Date: Mon, 4 Apr 2022 13:00:15 -0400 Subject: [PATCH 48/55] broke up further... both_same_type now both_numpy and both_pandas --- tests/test_dataframe_checks.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 24a489953..47aa3237c 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -38,6 +38,31 @@ def test_contains_na(df_na): pd.DataFrame({"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}), pd.Series([1, 2, 3, 4]), ), + ], +) +def test_check_pd_X_y_both_numpy(X_in, y_in, expected_1, expected_2): + # Execute + X_out, y_out = _check_pd_X_y(X_in, y_in) + + # Test X output + if expected_1 is None: + assert X_out is X_in + elif isinstance(expected_1, pd.DataFrame): + assert_frame_equal(X_out, expected_1) + elif isinstance(expected_1, (np.generic, np.ndarray)): + assert all(X_out == expected_1) + + # Test y output + if expected_2 is None: + assert y_out is y_in + elif isinstance(expected_2, pd.Series): + assert_series_equal(y_out, expected_2) + elif isinstance(expected_2, (np.generic, np.ndarray)): + assert all(y_out == expected_2) + +@pytest.mark.parametrize( + "X_in, y_in, expected_1, expected_2", + [ # * If both parameters are pandas objects and their indexes match, they are # returned unchanged. ( @@ -50,7 +75,7 @@ def test_contains_na(df_na): ), ], ) -def test_check_pd_X_y_both_same_type(X_in, y_in, expected_1, expected_2): +def test_check_pd_X_y_both_pandas(X_in, y_in, expected_1, expected_2): # Execute X_out, y_out = _check_pd_X_y(X_in, y_in) From fafbea0f6ea4eb4a99653dd17b413ff3780fa87e Mon Sep 17 00:00:00 2001 From: Noah Green Date: Mon, 4 Apr 2022 13:00:51 -0400 Subject: [PATCH 49/55] black/isort/flake8 --- tests/test_dataframe_checks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 47aa3237c..c90246469 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -60,6 +60,7 @@ def test_check_pd_X_y_both_numpy(X_in, y_in, expected_1, expected_2): elif isinstance(expected_2, (np.generic, np.ndarray)): assert all(y_out == expected_2) + @pytest.mark.parametrize( "X_in, y_in, expected_1, expected_2", [ From 12cc894eadd64c0817502b350b09202b601bbe4c Mon Sep 17 00:00:00 2001 From: Noah Green Date: Tue, 5 Apr 2022 05:58:36 -0400 Subject: [PATCH 50/55] fixed type hint and docstring on _check_fit_input_and_variables() --- feature_engine/encoding/base_encoder.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/feature_engine/encoding/base_encoder.py b/feature_engine/encoding/base_encoder.py index 666813dc2..2d57ee60e 100644 --- a/feature_engine/encoding/base_encoder.py +++ b/feature_engine/encoding/base_encoder.py @@ -53,7 +53,7 @@ def __init__( self.variables = _check_input_parameter_variables(variables) self.ignore_format = ignore_format - def _check_fit_input_and_variables(self, X: pd.DataFrame) -> pd.DataFrame: + def _check_fit_input_and_variables(self, X: pd.DataFrame): """ Checks that input is a dataframe, finds categorical variables, or alternatively checks that the variables entered by the user are of type object (categorical). @@ -71,13 +71,6 @@ def _check_fit_input_and_variables(self, X: pd.DataFrame) -> pd.DataFrame: ValueError If there are no categorical variables in the df or the df is empty If the variable(s) contain null values - - Returns - ------- - X: Pandas DataFrame - The same dataframe entered as parameter - variables : list - list of categorical variables """ if not self.ignore_format: From f953e91dc45e96ffeeee1c0bf7bb55c55fd8e71d Mon Sep 17 00:00:00 2001 From: Noah Green Date: Wed, 6 Apr 2022 10:35:46 -0400 Subject: [PATCH 51/55] confirmed that incompatible lengths also raise error in _check_pd_X_y --- tests/test_dataframe_checks.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index c90246469..54ac8d302 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -159,6 +159,16 @@ def test_check_pd_X_y_np_to_pd(X_in, y_in, expected_1, expected_2): ValueError, ".*Index.*", ), + + # Show that incompatible dimensions causes same error + ( + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + pd.Series([1, 2, 3], index=[22, 99, 101]), + ValueError, + ".*Lengths.*", + ), ], ) def test_check_pd_X_y_errors(X_in, y_in, exception_type, exception_match): From c24514b1ca7cd432a8e6b70e31f0e6ffba7b7ef0 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Wed, 6 Apr 2022 10:39:47 -0400 Subject: [PATCH 52/55] _check_pd_X_y() now copies any incoming pandas objects; fixed unit tests to confirm --- feature_engine/dataframe_checks.py | 6 +++--- tests/test_dataframe_checks.py | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index 1a4a6ef1c..0ae609cef 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -146,7 +146,7 @@ def _check_pd_X_y( * If both parameters are pandas objects, and their indexes are inconsistent, an exception is raised (i.e. this is the caller's error.) * If both parameters are pandas objects and their indexes match, they are - returned unchanged. + copied and returned. * If X is sparse or X is empty or, after all transforms, is stiil not a DataFrame, raises an exception @@ -178,12 +178,12 @@ def _check_pd_X_y( # * If both parameters are pandas objects, and their indexes are inconsistent, # an exception is raised (i.e. this is the caller's error.) # * If both parameters are pandas objects and their indexes match, they are - # returned unchanged. + # copied and returned if isinstance(X, pd.DataFrame) and isinstance(y, pd.Series): if not all(y.index == X.index): raise ValueError("Index mismatch between DataFrame X and Series y") else: - pass # deliberately highlighting the no-op case + return X.copy(), y.copy() # * If X is sparse or X is empty or, after all transforms, is stiil # not a DataFrame, raises an exception diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 54ac8d302..0ac88e86a 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -65,14 +65,16 @@ def test_check_pd_X_y_both_numpy(X_in, y_in, expected_1, expected_2): "X_in, y_in, expected_1, expected_2", [ # * If both parameters are pandas objects and their indexes match, they are - # returned unchanged. + # copied and returned. ( pd.DataFrame( {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] ), pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), - None, - None, + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), ), ], ) @@ -81,18 +83,16 @@ def test_check_pd_X_y_both_pandas(X_in, y_in, expected_1, expected_2): X_out, y_out = _check_pd_X_y(X_in, y_in) # Test X output - if expected_1 is None: - assert X_out is X_in - elif isinstance(expected_1, pd.DataFrame): + if isinstance(expected_1, pd.DataFrame): assert_frame_equal(X_out, expected_1) + assert X_out is not expected_1 # make sure copied elif isinstance(expected_1, (np.generic, np.ndarray)): assert all(X_out == expected_1) # Test y output - if expected_2 is None: - assert y_out is y_in - elif isinstance(expected_2, pd.Series): + if isinstance(expected_2, pd.Series): assert_series_equal(y_out, expected_2) + assert y_out is not expected_2 # make sure copied elif isinstance(expected_2, (np.generic, np.ndarray)): assert all(y_out == expected_2) From a92d4b00cd3393f2cbf70d714e3253187c56628b Mon Sep 17 00:00:00 2001 From: Noah Green Date: Wed, 6 Apr 2022 10:45:28 -0400 Subject: [PATCH 53/55] _check_pd_X_y() raises exception if either incoming object is None or empty; unit tests to confirm --- feature_engine/dataframe_checks.py | 8 +++++++- tests/test_dataframe_checks.py | 31 ++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index 0ae609cef..9eeed7ff8 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -149,6 +149,7 @@ def _check_pd_X_y( copied and returned. * If X is sparse or X is empty or, after all transforms, is stiil not a DataFrame, raises an exception + * Raises an exception if either incoming object is None or empty Parameters ---------- @@ -163,8 +164,13 @@ def _check_pd_X_y( Exceptions ---------- ValueError: if X and y are dimension-incompatible, X and y are pandas objects - with inconsistent indexes + with inconsistent indexes, or if either X or y is None/empty """ + # * Raises an exception if either incoming object is None or empty + if X is None or X.shape[0] == 0: + raise ValueError("X cannot be None or empty") + if y is None or y.shape[0] == 0: + raise ValueError("y cannot be None or empty") # * If both parameters are numpy objects, they are converted to pandas objects. # * If one parameter is a pandas object and the other is a numpy object, diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 0ac88e86a..02bd9047f 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -160,6 +160,37 @@ def test_check_pd_X_y_np_to_pd(X_in, y_in, expected_1, expected_2): ".*Index.*", ), + # * Raises an exception if either incoming object is None or empty + ( + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + None, + ValueError, + ".*None.*empty.*", + ), + ( + None, + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), + ValueError, + ".*None.*empty.*", + ), + ( + pd.DataFrame(), + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), + ValueError, + ".*None.*empty.*", + ), + ( + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + pd.Series(), + ValueError, + ".*None.*empty.*", + ), + + # Show that incompatible dimensions causes same error ( pd.DataFrame( From 709ac247d7a282488106fe70b3ea8bd276f51348 Mon Sep 17 00:00:00 2001 From: Noah Green Date: Wed, 6 Apr 2022 10:50:52 -0400 Subject: [PATCH 54/55] _check_pd_X_y() now supports list or Tuple for y; unit tests to confirm --- feature_engine/dataframe_checks.py | 15 +++++++++------ tests/test_dataframe_checks.py | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index 9eeed7ff8..4f39ac197 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -2,7 +2,7 @@ transform(). """ -from typing import List, Union +from typing import List, Union, Tuple import numpy as np import pandas as pd @@ -134,7 +134,7 @@ def _check_contains_inf(X: pd.DataFrame, variables: List[Union[str, int]]) -> No def _check_pd_X_y( X: Union[pd.DataFrame, np.ndarray], - y: Union[pd.Series, np.ndarray], + y: Union[pd.Series, np.ndarray, list, Tuple], ): """ Returns X as a DataFrame and y as a Series, converting any numpy @@ -154,7 +154,7 @@ def _check_pd_X_y( Parameters ---------- X: Pandas DataFrame or numpy ndarray - y: Pandas Series or numpy ndarray + y: Pandas Series or numpy ndarray or list or tuple Returns ------- @@ -167,19 +167,22 @@ def _check_pd_X_y( with inconsistent indexes, or if either X or y is None/empty """ # * Raises an exception if either incoming object is None or empty - if X is None or X.shape[0] == 0: + if X is None or len(X) == 0: raise ValueError("X cannot be None or empty") - if y is None or y.shape[0] == 0: + if y is None or len(y) == 0: raise ValueError("y cannot be None or empty") # * If both parameters are numpy objects, they are converted to pandas objects. # * If one parameter is a pandas object and the other is a numpy object, # the former will be converted to a pandas object, with the indexes - # of the latter. + # of the latter. (Lists and tuples are also supported for y) if _is_numpy(X): X = _numpy_to_dataframe(X, index=y.index if isinstance(y, pd.Series) else None) if _is_numpy(y): y = _numpy_to_series(y, index=X.index if isinstance(X, pd.DataFrame) else None) + if isinstance(y, (list, Tuple)): + y = pd.Series(y) + y.index = X.index if isinstance(X, pd.DataFrame) else None # * If both parameters are pandas objects, and their indexes are inconsistent, # an exception is raised (i.e. this is the caller's error.) diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 02bd9047f..66a9dda4d 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -122,6 +122,27 @@ def test_check_pd_X_y_both_pandas(X_in, y_in, expected_1, expected_2): ), pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), ), + # (Lists and tuples are also supported for y) + ( + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + [1, 2, 3, 4], + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), + ), + ( + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + (1, 2, 3, 4), + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), + ), ], ) def test_check_pd_X_y_np_to_pd(X_in, y_in, expected_1, expected_2): From f2b5ad980850316d88e0985d5336b25d2c1c553d Mon Sep 17 00:00:00 2001 From: Noah Green Date: Wed, 6 Apr 2022 10:51:46 -0400 Subject: [PATCH 55/55] black/isort/flake8 --- feature_engine/dataframe_checks.py | 2 +- tests/test_dataframe_checks.py | 87 +++++++++++++++--------------- 2 files changed, 43 insertions(+), 46 deletions(-) diff --git a/feature_engine/dataframe_checks.py b/feature_engine/dataframe_checks.py index 4f39ac197..0f9838965 100644 --- a/feature_engine/dataframe_checks.py +++ b/feature_engine/dataframe_checks.py @@ -2,7 +2,7 @@ transform(). """ -from typing import List, Union, Tuple +from typing import List, Tuple, Union import numpy as np import pandas as pd diff --git a/tests/test_dataframe_checks.py b/tests/test_dataframe_checks.py index 66a9dda4d..0cc9f3cb5 100644 --- a/tests/test_dataframe_checks.py +++ b/tests/test_dataframe_checks.py @@ -124,24 +124,24 @@ def test_check_pd_X_y_both_pandas(X_in, y_in, expected_1, expected_2): ), # (Lists and tuples are also supported for y) ( - pd.DataFrame( - {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] - ), - [1, 2, 3, 4], - pd.DataFrame( - {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] - ), - pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + [1, 2, 3, 4], + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), ), ( - pd.DataFrame( - {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] - ), - (1, 2, 3, 4), - pd.DataFrame( - {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] - ), - pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + (1, 2, 3, 4), + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), ), ], ) @@ -180,46 +180,43 @@ def test_check_pd_X_y_np_to_pd(X_in, y_in, expected_1, expected_2): ValueError, ".*Index.*", ), - # * Raises an exception if either incoming object is None or empty ( - pd.DataFrame( - {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] - ), - None, - ValueError, - ".*None.*empty.*", + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + None, + ValueError, + ".*None.*empty.*", ), ( - None, - pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), - ValueError, - ".*None.*empty.*", + None, + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), + ValueError, + ".*None.*empty.*", ), ( - pd.DataFrame(), - pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), - ValueError, - ".*None.*empty.*", + pd.DataFrame(), + pd.Series([1, 2, 3, 4], index=[22, 99, 101, 212]), + ValueError, + ".*None.*empty.*", ), ( - pd.DataFrame( - {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] - ), - pd.Series(), - ValueError, - ".*None.*empty.*", + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + pd.Series(), + ValueError, + ".*None.*empty.*", ), - - # Show that incompatible dimensions causes same error ( - pd.DataFrame( - {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] - ), - pd.Series([1, 2, 3], index=[22, 99, 101]), - ValueError, - ".*Lengths.*", + pd.DataFrame( + {"0": [1, 2, 3, 4], "1": [5, 6, 7, 8]}, index=[22, 99, 101, 212] + ), + pd.Series([1, 2, 3], index=[22, 99, 101]), + ValueError, + ".*Lengths.*", ), ], )