From a8a4fa978f1a7c2cae7c07fb046254931dde759d Mon Sep 17 00:00:00 2001 From: Luis Seabra Date: Sun, 5 Mar 2023 11:05:35 +0000 Subject: [PATCH 1/6] any_kwargs --- tests/test_encoding/test_encoders/test_similarity_encoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_encoding/test_encoders/test_similarity_encoder.py b/tests/test_encoding/test_encoders/test_similarity_encoder.py index 8e2336131..3e74b3717 100644 --- a/tests/test_encoding/test_encoders/test_similarity_encoder.py +++ b/tests/test_encoding/test_encoders/test_similarity_encoder.py @@ -142,7 +142,7 @@ def test_nan_behaviour_impute(df_enc_big_na): def test_nan_behaviour_ignore(df_enc_big_na): encoder = StringSimilarityEncoder(missing_values="ignore") X = encoder.fit_transform(df_enc_big_na) - assert (X.isna().any(1) == df_enc_big_na.isna().any(1)).all() + assert (X.isna().any(axis=1) == df_enc_big_na.isna().any(axis=1)).all() assert encoder.encoder_dict_ == { "var_A": ["B", "D", "G", "A", "C", "E", "F"], "var_B": ["A", "D", "B", "G", "C", "E", "F"], From 0b7d7d9e6402e55936683604a73a2e68d80a81fa Mon Sep 17 00:00:00 2001 From: Luis Seabra Date: Sun, 5 Mar 2023 11:05:51 +0000 Subject: [PATCH 2/6] missing_cats --- feature_engine/imputation/base_imputer.py | 4 +--- feature_engine/imputation/categorical.py | 7 +++---- tests/test_imputation/test_categorical_imputer.py | 3 +-- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/feature_engine/imputation/base_imputer.py b/feature_engine/imputation/base_imputer.py index 60dee5df9..0ab657a1d 100644 --- a/feature_engine/imputation/base_imputer.py +++ b/feature_engine/imputation/base_imputer.py @@ -60,9 +60,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: X = self._transform(X) # Replace missing data with learned parameters - X.fillna(value=self.imputer_dict_, inplace=True) - - return X + return X.fillna(value=self.imputer_dict_) def _get_feature_names_in(self, X): """Get the names and number of features in the train set (the dataframe diff --git a/feature_engine/imputation/categorical.py b/feature_engine/imputation/categorical.py index 16942f598..610f0ffdc 100644 --- a/feature_engine/imputation/categorical.py +++ b/feature_engine/imputation/categorical.py @@ -230,13 +230,12 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # if variable is of type category, we need to add the new # category, before filling in the nan + add_cats = {} for variable in self.variables_: if pd.api.types.is_categorical_dtype(X[variable]): - X[variable].cat.add_categories( - self.imputer_dict_[variable], inplace=True - ) + add_cats.update({variable: X[variable].cat.add_categories(self.imputer_dict_[variable])}) - X.fillna(self.imputer_dict_, inplace=True) + X = X.assign(**add_cats).fillna(self.imputer_dict_) # add additional step to return variables cast as object if self.return_object: diff --git a/tests/test_imputation/test_categorical_imputer.py b/tests/test_imputation/test_categorical_imputer.py index d10640b2c..6a0c55cc1 100644 --- a/tests/test_imputation/test_categorical_imputer.py +++ b/tests/test_imputation/test_categorical_imputer.py @@ -245,8 +245,7 @@ def test_variables_cast_as_category_missing(df_na): X_reference["Name"] = X_reference["Name"].fillna("Missing") X_reference["Studies"] = X_reference["Studies"].fillna("Missing") - X_reference["City"].cat.add_categories("Missing", inplace=True) - X_reference["City"] = X_reference["City"].fillna("Missing") + X_reference["City"] = X_reference["City"].cat.add_categories("Missing").fillna("Missing") # test fit attributes assert imputer.variables_ == ["Name", "City", "Studies"] From c4d3cca591d46c26ed0152c34668ef661447f86c Mon Sep 17 00:00:00 2001 From: Luis Seabra Date: Sun, 5 Mar 2023 11:11:13 +0000 Subject: [PATCH 3/6] mixed iloc in test drop high psi --- tests/test_selection/test_drop_high_psi_features.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_selection/test_drop_high_psi_features.py b/tests/test_selection/test_drop_high_psi_features.py index 2764ea716..004a93baa 100644 --- a/tests/test_selection/test_drop_high_psi_features.py +++ b/tests/test_selection/test_drop_high_psi_features.py @@ -266,7 +266,7 @@ def test_split_col_not_included_in_variables(df): def test_error_if_na_in_split_col(df): """Test an error is raised if the split column contains missing values.""" data = df.copy() - data["var_3"].iloc[15] = np.nan + data.iloc[15, data.columns.get_loc("var_3")] = np.nan transformer = DropHighPSIFeatures(split_col="var_3") @@ -277,7 +277,7 @@ def test_error_if_na_in_split_col(df): def test_raise_error_if_na_in_df(df): """Test an error is raised when missing values is set to raise.""" data = df.copy() - data["var_3"].iloc[15] = np.nan + data.iloc[15, data.columns.get_loc("var_3")] = np.nan transformer = DropHighPSIFeatures(missing_values="raise") @@ -288,7 +288,7 @@ def test_raise_error_if_na_in_df(df): def test_missing_value_ignored(df): """Test if PSI are computed when missing values are present in the dataframe.""" data = df.copy() - data["var_3"].iloc[15] = np.nan + data.iloc[15, data.columns.get_loc("var_3")] = np.nan var_col = [col for col in data if "var" in col] @@ -301,7 +301,7 @@ def test_missing_value_ignored(df): def test_raise_error_if_inf_in_df(df): """Test an error is raised for inf when missing values is set to raise.""" data = df.copy() - data["var_3"].iloc[15] = np.inf + data.iloc[15, data.columns.get_loc("var_3")] = np.nan transformer = DropHighPSIFeatures(missing_values="raise") From de22bb19baa18dea2e920fadba5f9e8686716f1c Mon Sep 17 00:00:00 2001 From: Luis Seabra Date: Sun, 5 Mar 2023 11:53:49 +0000 Subject: [PATCH 4/6] sklearn >= 1.2 OneHotEncoder --- tests/test_wrappers/test_sklearn_wrapper.py | 28 +++++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/test_wrappers/test_sklearn_wrapper.py b/tests/test_wrappers/test_sklearn_wrapper.py index 81a425a98..42c6b3c4f 100644 --- a/tests/test_wrappers/test_sklearn_wrapper.py +++ b/tests/test_wrappers/test_sklearn_wrapper.py @@ -1,6 +1,7 @@ import numpy as np import pandas as pd import pytest +from sklearn import __version__ as skl_version from sklearn.base import clone from sklearn.datasets import fetch_california_housing from sklearn.decomposition import PCA @@ -50,11 +51,22 @@ ] +def _OneHotEncoder(sparse, drop=None, dtype=np.float64) -> OneHotEncoder: + """OneHotEncoder kwarg sparse has been renamed as sparse_output in scikitlearn >=1.2""" + + if skl_version.split('.')[0] == '1' and int(skl_version.split('.')[1]) >= 2: + return OneHotEncoder(sparse_output=sparse, drop=drop, dtype=dtype) + else: + return OneHotEncoder(sparse=sparse, drop=drop, dtype=dtype) + + + + @pytest.mark.parametrize( "transformer", [ SimpleImputer(), - OneHotEncoder(sparse=False), + _OneHotEncoder(sparse=False), StandardScaler(), SelectKBest(), ], @@ -319,7 +331,7 @@ def test_sklearn_ohe_object_one_feature(df_vartypes): variables_to_encode = ["Name"] transformer = SklearnTransformerWrapper( - transformer=OneHotEncoder(sparse=False, dtype=np.int64), + transformer=_OneHotEncoder(sparse=False, dtype=np.int64), variables=variables_to_encode, ) @@ -341,7 +353,7 @@ def test_sklearn_ohe_object_many_features(df_vartypes): variables_to_encode = ["Name", "City"] transformer = SklearnTransformerWrapper( - transformer=OneHotEncoder(sparse=False, dtype=np.int64), + transformer=_OneHotEncoder(sparse=False, dtype=np.int64), variables=variables_to_encode, ) @@ -367,7 +379,7 @@ def test_sklearn_ohe_numeric(df_vartypes): variables_to_encode = ["Age"] transformer = SklearnTransformerWrapper( - transformer=OneHotEncoder(sparse=False, dtype=np.int64), + transformer=_OneHotEncoder(sparse=False, dtype=np.int64), variables=variables_to_encode, ) @@ -387,7 +399,7 @@ def test_sklearn_ohe_numeric(df_vartypes): def test_sklearn_ohe_all_features(df_vartypes): transformer = SklearnTransformerWrapper( - transformer=OneHotEncoder(sparse=False, dtype=np.int64) + transformer=_OneHotEncoder(sparse=False, dtype=np.int64) ) ref = pd.DataFrame( @@ -443,7 +455,7 @@ def test_sklearn_ohe_with_crossvalidation(): ( "encode_cat", SklearnTransformerWrapper( - transformer=OneHotEncoder(drop="first", sparse=False), + transformer=_OneHotEncoder(drop="first", sparse=False), variables=["AveBedrms_cat"], ), ), @@ -459,7 +471,7 @@ def test_sklearn_ohe_with_crossvalidation(): def test_wrap_one_hot_encoder_get_features_name_out(df_vartypes): - ohe_wrap = SklearnTransformerWrapper(transformer=OneHotEncoder(sparse=False)) + ohe_wrap = SklearnTransformerWrapper(transformer=_OneHotEncoder(sparse=False)) ohe_wrap.fit(df_vartypes) expected_features_all = [ @@ -602,7 +614,7 @@ def test_get_feature_names_out_polynomialfeatures(varlist): def test_get_feature_names_out_ohe(varlist, df_vartypes): transformer = SklearnTransformerWrapper( - transformer=OneHotEncoder(sparse=False, dtype=np.int64), + transformer=_OneHotEncoder(sparse=False, dtype=np.int64), variables=varlist, ) From e488e5dcea837c711b8f2d39199df00b60f7e0f8 Mon Sep 17 00:00:00 2001 From: Luis Seabra Date: Sun, 5 Mar 2023 15:13:41 +0000 Subject: [PATCH 5/6] formatting --- feature_engine.code-workspace | 8 + feature_engine/imputation/categorical.py | 11 +- test_run_cut.txt | 166 ++++++++++++++++++ .../test_categorical_imputer.py | 6 +- tests/test_wrappers/test_sklearn_wrapper.py | 9 +- 5 files changed, 187 insertions(+), 13 deletions(-) create mode 100644 feature_engine.code-workspace create mode 100644 test_run_cut.txt diff --git a/feature_engine.code-workspace b/feature_engine.code-workspace new file mode 100644 index 000000000..876a1499c --- /dev/null +++ b/feature_engine.code-workspace @@ -0,0 +1,8 @@ +{ + "folders": [ + { + "path": "." + } + ], + "settings": {} +} \ No newline at end of file diff --git a/feature_engine/imputation/categorical.py b/feature_engine/imputation/categorical.py index 610f0ffdc..096dfa22e 100644 --- a/feature_engine/imputation/categorical.py +++ b/feature_engine/imputation/categorical.py @@ -134,7 +134,6 @@ def __init__( return_object: bool = False, ignore_format: bool = False, ) -> None: - if imputation_method not in ["missing", "frequent"]: raise ValueError( "imputation_method takes only values 'missing' or 'frequent'" @@ -180,7 +179,6 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): self.imputer_dict_ = {var: self.fill_value for var in self.variables_} elif self.imputation_method == "frequent": - # if imputing only 1 variable: if len(self.variables_) == 1: var = self.variables_[0] @@ -219,7 +217,6 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): return self def transform(self, X: pd.DataFrame) -> pd.DataFrame: - # Frequent category imputation if self.imputation_method == "frequent": X = super().transform(X) @@ -233,7 +230,13 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: add_cats = {} for variable in self.variables_: if pd.api.types.is_categorical_dtype(X[variable]): - add_cats.update({variable: X[variable].cat.add_categories(self.imputer_dict_[variable])}) + add_cats.update( + { + variable: X[variable].cat.add_categories( + self.imputer_dict_[variable] + ) + } + ) X = X.assign(**add_cats).fillna(self.imputer_dict_) diff --git a/test_run_cut.txt b/test_run_cut.txt new file mode 100644 index 000000000..8644329b5 --- /dev/null +++ b/test_run_cut.txt @@ -0,0 +1,166 @@ +==================== solved ===================== + +tests/test_encoding/test_encoders/test_similarity_encoder.py::test_nan_behaviour_ignore + /Users/luis/code/feature_engine/tests/test_encoding/test_encoders/test_similarity_encoder.py:145: FutureWarning: In a future version of pandas all arguments of DataFrame.any and Series.any will be keyword-only. + assert (X.isna().any(1) == df_enc_big_na.isna().any(1)).all() + + +tests/test_imputation/test_categorical_imputer.py::test_variables_cast_as_category_missing + /Users/luis/code/feature_engine/feature_engine/imputation/categorical.py:235: FutureWarning: The `inplace` parameter in pandas.Categorical.add_categories is deprecated and will be removed in a future version. Removing unused categories will always return a new Categorical object. + X[variable].cat.add_categories( + +tests/test_imputation/test_categorical_imputer.py::test_variables_cast_as_category_missing + /Users/luis/code/feature_engine/feature_engine/imputation/categorical.py:239: DeprecationWarning: In a future version, `df.iloc[:, i] = newvals` will attempt to set the values inplace instead of always setting a new array. To retain the old behavior, use either `df[df.columns[i]] = newvals` or, if columns are non-unique, `df.isetitem(i, newvals)` + X.fillna(self.imputer_dict_, inplace=True) + +tests/test_imputation/test_categorical_imputer.py::test_variables_cast_as_category_missing + /Users/luis/code/feature_engine/tests/test_imputation/test_categorical_imputer.py:248: FutureWarning: The `inplace` parameter in pandas.Categorical.add_categories is deprecated and will be removed in a future version. Removing unused categories will always return a new Categorical object. + X_reference["City"].cat.add_categories("Missing", inplace=True) + +tests/test_imputation/test_categorical_imputer.py::test_variables_cast_as_category_frequent + /Users/luis/code/feature_engine/feature_engine/imputation/base_imputer.py:63: DeprecationWarning: In a future version, `df.iloc[:, i] = newvals` will attempt to set the values inplace instead of always setting a new array. To retain the old behavior, use either `df[df.columns[i]] = newvals` or, if columns are non-unique, `df.isetitem(i, newvals)` + X.fillna(value=self.imputer_dict_, inplace=True) + + + +tests/test_selection/test_drop_high_psi_features.py::test_error_if_na_in_split_col + /Users/luis/code/feature_engine/tests/test_selection/test_drop_high_psi_features.py:269: SettingWithCopyWarning: + A value is trying to be set on a copy of a slice from a DataFrame + + See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy + data["var_3"].iloc[15] = np.nan + +tests/test_selection/test_drop_high_psi_features.py::test_raise_error_if_na_in_df + /Users/luis/code/feature_engine/tests/test_selection/test_drop_high_psi_features.py:280: SettingWithCopyWarning: + A value is trying to be set on a copy of a slice from a DataFrame + + See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy + data["var_3"].iloc[15] = np.nan + +tests/test_selection/test_drop_high_psi_features.py::test_missing_value_ignored + /Users/luis/code/feature_engine/tests/test_selection/test_drop_high_psi_features.py:291: SettingWithCopyWarning: + A value is trying to be set on a copy of a slice from a DataFrame + + See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy + data["var_3"].iloc[15] = np.nan + +tests/test_selection/test_drop_high_psi_features.py::test_raise_error_if_inf_in_df + /Users/luis/code/feature_engine/tests/test_selection/test_drop_high_psi_features.py:304: SettingWithCopyWarning: + A value is trying to be set on a copy of a slice from a DataFrame + + See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy + data["var_3"].iloc[15] = np.inf + + + ### harmless, we want to use the new behaviour------------------------- +tests/test_transformation/test_power_transformer.py::test_inverse_transform_exp_no_default[4] + /Users/luis/code/feature_engine/feature_engine/transformation/power.py:152: DeprecationWarning: In a future version, `df.iloc[:, i] = newvals` will attempt to set the values inplace instead of always setting a new array. To retain the old behavior, use either `df[df.columns[i]] = newvals` or, if columns are non-unique, `df.isetitem(i, newvals)` + X.loc[:, self.variables_] = np.power(X.loc[:, self.variables_], 1 / self.exp) + +tests/test_transformation/test_reciprocal_transformer.py::test_automatically_find_variables + /Users/luis/code/feature_engine/feature_engine/transformation/reciprocal.py:137: DeprecationWarning: In a future version, `df.iloc[:, i] = newvals` will attempt to set the values inplace instead of always setting a new array. To retain the old behavior, use either `df[df.columns[i]] = newvals` or, if columns are non-unique, `df.isetitem(i, newvals)` + X.loc[:, self.variables_] = X.loc[:, self.variables_].astype("float") + + + +tests/test_wrappers/test_sklearn_wrapper.py: 10 warnings + /Users/luis/miniforge3/envs/py310/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:828: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value. + warnings.warn( + + +====================unsolved =================== + +tests/test_encoding/test_encoders/test_ordinal_encoder.py::test_inverse_transform_when_ignore_unseen + /Users/luis/code/feature_engine/feature_engine/encoding/base_encoder.py:257: UserWarning: During the encoding, NaN values were introduced in the feature(s) words. + warnings.warn( + +tests/test_encoding/test_encoders/test_check_estimator_encoders.py::test_df + /Users/luis/miniforge3/envs/py310/lib/python3.10/site-packages/_pytest/python.py:199: PytestReturnNotNoneWarning: Expected None, but tests/test_encoding/test_encoders/test_check_estimator_encoders.py::test_df returned ( var_0 var_1 var_2 ... var_9 var_10 var_11 + 0 1.471061 -2.376400 -0.247208 ... 3.528094 2.070526 -1.989335 + 1 1.819196 1.969326 -0.126894 ... 3.304213 1.184820 -1.309524 + 2 1.625024 1.499174 0.334123 ... 3.717297 -0.066448 -0.852703 + 3 1.939212 0.075341 1.627132 ... 5.131589 0.713558 0.484649 + 4 1.579307 0.372213 0.338141 ... 3.512742 0.398790 -0.186530 + .. ... ... ... ... ... ... ... + 995 1.803380 -1.363868 -0.048464 ... 0.213561 -0.144619 0.555170 + 996 2.073749 0.263967 1.269868 ... 0.564732 -0.182916 1.193456 + 997 2.442529 0.528868 0.230641 ... 0.787021 0.021500 -2.710394 + 998 1.451356 -0.871014 -0.117844 ... 2.985371 0.340473 0.149377 + 999 1.721321 -0.534328 -0.331306 ... 4.000561 -0.416476 -0.111816 + + [1000 rows x 12 columns], 0 1 + 1 1 + 2 1 + 3 1 + 4 1 + .. + 995 0 + 996 0 + 997 0 + 998 1 + 999 1 + Length: 1000, dtype: int64), which will be an error in a future version of pytest. Did you mean to use `assert` instead of `return`? + warnings.warn( + + +tests/test_prediction/test_check_estimator_prediction.py::test_df + /Users/luis/miniforge3/envs/py310/lib/python3.10/site-packages/_pytest/python.py:199: PytestReturnNotNoneWarning: Expected None, but tests/test_prediction/test_check_estimator_prediction.py::test_df returned ( var_0 var_1 var_2 ... var_9 var_10 var_11 + 0 1.471061 -2.376400 -0.247208 ... 3.528094 2.070526 -1.989335 + 1 1.819196 1.969326 -0.126894 ... 3.304213 1.184820 -1.309524 + 2 1.625024 1.499174 0.334123 ... 3.717297 -0.066448 -0.852703 + 3 1.939212 0.075341 1.627132 ... 5.131589 0.713558 0.484649 + 4 1.579307 0.372213 0.338141 ... 3.512742 0.398790 -0.186530 + .. ... ... ... ... ... ... ... + 995 1.803380 -1.363868 -0.048464 ... 0.213561 -0.144619 0.555170 + 996 2.073749 0.263967 1.269868 ... 0.564732 -0.182916 1.193456 + 997 2.442529 0.528868 0.230641 ... 0.787021 0.021500 -2.710394 + 998 1.451356 -0.871014 -0.117844 ... 2.985371 0.340473 0.149377 + 999 1.721321 -0.534328 -0.331306 ... 4.000561 -0.416476 -0.111816 + + [1000 rows x 12 columns], 0 1 + 1 1 + 2 1 + 3 1 + 4 1 + .. + 995 0 + 996 0 + 997 0 + 998 1 + 999 1 + Length: 1000, dtype: int64), which will be an error in a future version of pytest. Did you mean to use `assert` instead of `return`? + warnings.warn( + + +tests/test_preprocessing/test_check_estimator_preprocessing.py::test_df + /Users/luis/miniforge3/envs/py310/lib/python3.10/site-packages/_pytest/python.py:199: PytestReturnNotNoneWarning: Expected None, but tests/test_preprocessing/test_check_estimator_preprocessing.py::test_df returned ( var_0 var_1 var_2 ... var_9 var_10 var_11 + 0 1.471061 -2.376400 -0.247208 ... 3.528094 2.070526 -1.989335 + 1 1.819196 1.969326 -0.126894 ... 3.304213 1.184820 -1.309524 + 2 1.625024 1.499174 0.334123 ... 3.717297 -0.066448 -0.852703 + 3 1.939212 0.075341 1.627132 ... 5.131589 0.713558 0.484649 + 4 1.579307 0.372213 0.338141 ... 3.512742 0.398790 -0.186530 + .. ... ... ... ... ... ... ... + 995 1.803380 -1.363868 -0.048464 ... 0.213561 -0.144619 0.555170 + 996 2.073749 0.263967 1.269868 ... 0.564732 -0.182916 1.193456 + 997 2.442529 0.528868 0.230641 ... 0.787021 0.021500 -2.710394 + 998 1.451356 -0.871014 -0.117844 ... 2.985371 0.340473 0.149377 + 999 1.721321 -0.534328 -0.331306 ... 4.000561 -0.416476 -0.111816 + + [1000 rows x 12 columns], 0 1 + 1 1 + 2 1 + 3 1 + 4 1 + .. + 995 0 + 996 0 + 997 0 + 998 1 + 999 1 + Length: 1000, dtype: int64), which will be an error in a future version of pytest. Did you mean to use `assert` instead of `return`? + warnings.warn( + + +tests/test_wrappers/test_sklearn_wrapper.py::test_get_feature_names_out_transformers[transformer6-None] + /Users/luis/miniforge3/envs/py310/lib/python3.10/site-packages/sklearn/preprocessing/_function_transformer.py:307: RuntimeWarning: invalid value encountered in log + return func(X, **(kw_args if kw_args else {})) diff --git a/tests/test_imputation/test_categorical_imputer.py b/tests/test_imputation/test_categorical_imputer.py index 6a0c55cc1..95de56a44 100644 --- a/tests/test_imputation/test_categorical_imputer.py +++ b/tests/test_imputation/test_categorical_imputer.py @@ -150,7 +150,6 @@ def test_error_when_imputation_method_not_frequent_or_missing(): def test_error_when_variable_contains_multiple_modes(df_na): - msg = "The variable Name contains multiple frequent categories." imputer = CategoricalImputer(imputation_method="frequent", variables="Name") with pytest.raises(ValueError) as record: @@ -245,7 +244,9 @@ def test_variables_cast_as_category_missing(df_na): X_reference["Name"] = X_reference["Name"].fillna("Missing") X_reference["Studies"] = X_reference["Studies"].fillna("Missing") - X_reference["City"] = X_reference["City"].cat.add_categories("Missing").fillna("Missing") + X_reference["City"] = ( + X_reference["City"].cat.add_categories("Missing").fillna("Missing") + ) # test fit attributes assert imputer.variables_ == ["Name", "City", "Studies"] @@ -264,7 +265,6 @@ def test_variables_cast_as_category_missing(df_na): def test_variables_cast_as_category_frequent(df_na): - df_na = df_na.copy() df_na["City"] = df_na["City"].astype("category") diff --git a/tests/test_wrappers/test_sklearn_wrapper.py b/tests/test_wrappers/test_sklearn_wrapper.py index 42c6b3c4f..d600e2ae7 100644 --- a/tests/test_wrappers/test_sklearn_wrapper.py +++ b/tests/test_wrappers/test_sklearn_wrapper.py @@ -52,16 +52,15 @@ def _OneHotEncoder(sparse, drop=None, dtype=np.float64) -> OneHotEncoder: - """OneHotEncoder kwarg sparse has been renamed as sparse_output in scikitlearn >=1.2""" + """OneHotEncoder sparse argument has been renamed as sparse_output + in scikitlearn >=1.2""" - if skl_version.split('.')[0] == '1' and int(skl_version.split('.')[1]) >= 2: + if skl_version.split(".")[0] == "1" and int(skl_version.split(".")[1]) >= 2: return OneHotEncoder(sparse_output=sparse, drop=drop, dtype=dtype) else: return OneHotEncoder(sparse=sparse, drop=drop, dtype=dtype) - - @pytest.mark.parametrize( "transformer", [ @@ -553,7 +552,6 @@ def test_error_when_inverse_transform_not_implemented(transformer): ) @pytest.mark.parametrize("transformer", _transformers) def test_get_feature_names_out_transformers(varlist, transformer): - X = fetch_california_housing(as_frame=True).frame tr_wrap = SklearnTransformerWrapper(transformer=transformer, variables=varlist) Xw = tr_wrap.fit_transform(X) @@ -612,7 +610,6 @@ def test_get_feature_names_out_polynomialfeatures(varlist): @pytest.mark.parametrize("varlist", [["Name", "City"], None]) def test_get_feature_names_out_ohe(varlist, df_vartypes): - transformer = SklearnTransformerWrapper( transformer=_OneHotEncoder(sparse=False, dtype=np.int64), variables=varlist, From bb8557e6e61f6b0e49b92f09e38a89a487247c28 Mon Sep 17 00:00:00 2001 From: Luis Seabra Date: Fri, 10 Mar 2023 08:46:11 +0000 Subject: [PATCH 6/6] cleanup --- feature_engine.code-workspace | 8 -- test_run_cut.txt | 166 ---------------------------------- 2 files changed, 174 deletions(-) delete mode 100644 feature_engine.code-workspace delete mode 100644 test_run_cut.txt diff --git a/feature_engine.code-workspace b/feature_engine.code-workspace deleted file mode 100644 index 876a1499c..000000000 --- a/feature_engine.code-workspace +++ /dev/null @@ -1,8 +0,0 @@ -{ - "folders": [ - { - "path": "." - } - ], - "settings": {} -} \ No newline at end of file diff --git a/test_run_cut.txt b/test_run_cut.txt deleted file mode 100644 index 8644329b5..000000000 --- a/test_run_cut.txt +++ /dev/null @@ -1,166 +0,0 @@ -==================== solved ===================== - -tests/test_encoding/test_encoders/test_similarity_encoder.py::test_nan_behaviour_ignore - /Users/luis/code/feature_engine/tests/test_encoding/test_encoders/test_similarity_encoder.py:145: FutureWarning: In a future version of pandas all arguments of DataFrame.any and Series.any will be keyword-only. - assert (X.isna().any(1) == df_enc_big_na.isna().any(1)).all() - - -tests/test_imputation/test_categorical_imputer.py::test_variables_cast_as_category_missing - /Users/luis/code/feature_engine/feature_engine/imputation/categorical.py:235: FutureWarning: The `inplace` parameter in pandas.Categorical.add_categories is deprecated and will be removed in a future version. Removing unused categories will always return a new Categorical object. - X[variable].cat.add_categories( - -tests/test_imputation/test_categorical_imputer.py::test_variables_cast_as_category_missing - /Users/luis/code/feature_engine/feature_engine/imputation/categorical.py:239: DeprecationWarning: In a future version, `df.iloc[:, i] = newvals` will attempt to set the values inplace instead of always setting a new array. To retain the old behavior, use either `df[df.columns[i]] = newvals` or, if columns are non-unique, `df.isetitem(i, newvals)` - X.fillna(self.imputer_dict_, inplace=True) - -tests/test_imputation/test_categorical_imputer.py::test_variables_cast_as_category_missing - /Users/luis/code/feature_engine/tests/test_imputation/test_categorical_imputer.py:248: FutureWarning: The `inplace` parameter in pandas.Categorical.add_categories is deprecated and will be removed in a future version. Removing unused categories will always return a new Categorical object. - X_reference["City"].cat.add_categories("Missing", inplace=True) - -tests/test_imputation/test_categorical_imputer.py::test_variables_cast_as_category_frequent - /Users/luis/code/feature_engine/feature_engine/imputation/base_imputer.py:63: DeprecationWarning: In a future version, `df.iloc[:, i] = newvals` will attempt to set the values inplace instead of always setting a new array. To retain the old behavior, use either `df[df.columns[i]] = newvals` or, if columns are non-unique, `df.isetitem(i, newvals)` - X.fillna(value=self.imputer_dict_, inplace=True) - - - -tests/test_selection/test_drop_high_psi_features.py::test_error_if_na_in_split_col - /Users/luis/code/feature_engine/tests/test_selection/test_drop_high_psi_features.py:269: SettingWithCopyWarning: - A value is trying to be set on a copy of a slice from a DataFrame - - See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy - data["var_3"].iloc[15] = np.nan - -tests/test_selection/test_drop_high_psi_features.py::test_raise_error_if_na_in_df - /Users/luis/code/feature_engine/tests/test_selection/test_drop_high_psi_features.py:280: SettingWithCopyWarning: - A value is trying to be set on a copy of a slice from a DataFrame - - See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy - data["var_3"].iloc[15] = np.nan - -tests/test_selection/test_drop_high_psi_features.py::test_missing_value_ignored - /Users/luis/code/feature_engine/tests/test_selection/test_drop_high_psi_features.py:291: SettingWithCopyWarning: - A value is trying to be set on a copy of a slice from a DataFrame - - See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy - data["var_3"].iloc[15] = np.nan - -tests/test_selection/test_drop_high_psi_features.py::test_raise_error_if_inf_in_df - /Users/luis/code/feature_engine/tests/test_selection/test_drop_high_psi_features.py:304: SettingWithCopyWarning: - A value is trying to be set on a copy of a slice from a DataFrame - - See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy - data["var_3"].iloc[15] = np.inf - - - ### harmless, we want to use the new behaviour------------------------- -tests/test_transformation/test_power_transformer.py::test_inverse_transform_exp_no_default[4] - /Users/luis/code/feature_engine/feature_engine/transformation/power.py:152: DeprecationWarning: In a future version, `df.iloc[:, i] = newvals` will attempt to set the values inplace instead of always setting a new array. To retain the old behavior, use either `df[df.columns[i]] = newvals` or, if columns are non-unique, `df.isetitem(i, newvals)` - X.loc[:, self.variables_] = np.power(X.loc[:, self.variables_], 1 / self.exp) - -tests/test_transformation/test_reciprocal_transformer.py::test_automatically_find_variables - /Users/luis/code/feature_engine/feature_engine/transformation/reciprocal.py:137: DeprecationWarning: In a future version, `df.iloc[:, i] = newvals` will attempt to set the values inplace instead of always setting a new array. To retain the old behavior, use either `df[df.columns[i]] = newvals` or, if columns are non-unique, `df.isetitem(i, newvals)` - X.loc[:, self.variables_] = X.loc[:, self.variables_].astype("float") - - - -tests/test_wrappers/test_sklearn_wrapper.py: 10 warnings - /Users/luis/miniforge3/envs/py310/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:828: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value. - warnings.warn( - - -====================unsolved =================== - -tests/test_encoding/test_encoders/test_ordinal_encoder.py::test_inverse_transform_when_ignore_unseen - /Users/luis/code/feature_engine/feature_engine/encoding/base_encoder.py:257: UserWarning: During the encoding, NaN values were introduced in the feature(s) words. - warnings.warn( - -tests/test_encoding/test_encoders/test_check_estimator_encoders.py::test_df - /Users/luis/miniforge3/envs/py310/lib/python3.10/site-packages/_pytest/python.py:199: PytestReturnNotNoneWarning: Expected None, but tests/test_encoding/test_encoders/test_check_estimator_encoders.py::test_df returned ( var_0 var_1 var_2 ... var_9 var_10 var_11 - 0 1.471061 -2.376400 -0.247208 ... 3.528094 2.070526 -1.989335 - 1 1.819196 1.969326 -0.126894 ... 3.304213 1.184820 -1.309524 - 2 1.625024 1.499174 0.334123 ... 3.717297 -0.066448 -0.852703 - 3 1.939212 0.075341 1.627132 ... 5.131589 0.713558 0.484649 - 4 1.579307 0.372213 0.338141 ... 3.512742 0.398790 -0.186530 - .. ... ... ... ... ... ... ... - 995 1.803380 -1.363868 -0.048464 ... 0.213561 -0.144619 0.555170 - 996 2.073749 0.263967 1.269868 ... 0.564732 -0.182916 1.193456 - 997 2.442529 0.528868 0.230641 ... 0.787021 0.021500 -2.710394 - 998 1.451356 -0.871014 -0.117844 ... 2.985371 0.340473 0.149377 - 999 1.721321 -0.534328 -0.331306 ... 4.000561 -0.416476 -0.111816 - - [1000 rows x 12 columns], 0 1 - 1 1 - 2 1 - 3 1 - 4 1 - .. - 995 0 - 996 0 - 997 0 - 998 1 - 999 1 - Length: 1000, dtype: int64), which will be an error in a future version of pytest. Did you mean to use `assert` instead of `return`? - warnings.warn( - - -tests/test_prediction/test_check_estimator_prediction.py::test_df - /Users/luis/miniforge3/envs/py310/lib/python3.10/site-packages/_pytest/python.py:199: PytestReturnNotNoneWarning: Expected None, but tests/test_prediction/test_check_estimator_prediction.py::test_df returned ( var_0 var_1 var_2 ... var_9 var_10 var_11 - 0 1.471061 -2.376400 -0.247208 ... 3.528094 2.070526 -1.989335 - 1 1.819196 1.969326 -0.126894 ... 3.304213 1.184820 -1.309524 - 2 1.625024 1.499174 0.334123 ... 3.717297 -0.066448 -0.852703 - 3 1.939212 0.075341 1.627132 ... 5.131589 0.713558 0.484649 - 4 1.579307 0.372213 0.338141 ... 3.512742 0.398790 -0.186530 - .. ... ... ... ... ... ... ... - 995 1.803380 -1.363868 -0.048464 ... 0.213561 -0.144619 0.555170 - 996 2.073749 0.263967 1.269868 ... 0.564732 -0.182916 1.193456 - 997 2.442529 0.528868 0.230641 ... 0.787021 0.021500 -2.710394 - 998 1.451356 -0.871014 -0.117844 ... 2.985371 0.340473 0.149377 - 999 1.721321 -0.534328 -0.331306 ... 4.000561 -0.416476 -0.111816 - - [1000 rows x 12 columns], 0 1 - 1 1 - 2 1 - 3 1 - 4 1 - .. - 995 0 - 996 0 - 997 0 - 998 1 - 999 1 - Length: 1000, dtype: int64), which will be an error in a future version of pytest. Did you mean to use `assert` instead of `return`? - warnings.warn( - - -tests/test_preprocessing/test_check_estimator_preprocessing.py::test_df - /Users/luis/miniforge3/envs/py310/lib/python3.10/site-packages/_pytest/python.py:199: PytestReturnNotNoneWarning: Expected None, but tests/test_preprocessing/test_check_estimator_preprocessing.py::test_df returned ( var_0 var_1 var_2 ... var_9 var_10 var_11 - 0 1.471061 -2.376400 -0.247208 ... 3.528094 2.070526 -1.989335 - 1 1.819196 1.969326 -0.126894 ... 3.304213 1.184820 -1.309524 - 2 1.625024 1.499174 0.334123 ... 3.717297 -0.066448 -0.852703 - 3 1.939212 0.075341 1.627132 ... 5.131589 0.713558 0.484649 - 4 1.579307 0.372213 0.338141 ... 3.512742 0.398790 -0.186530 - .. ... ... ... ... ... ... ... - 995 1.803380 -1.363868 -0.048464 ... 0.213561 -0.144619 0.555170 - 996 2.073749 0.263967 1.269868 ... 0.564732 -0.182916 1.193456 - 997 2.442529 0.528868 0.230641 ... 0.787021 0.021500 -2.710394 - 998 1.451356 -0.871014 -0.117844 ... 2.985371 0.340473 0.149377 - 999 1.721321 -0.534328 -0.331306 ... 4.000561 -0.416476 -0.111816 - - [1000 rows x 12 columns], 0 1 - 1 1 - 2 1 - 3 1 - 4 1 - .. - 995 0 - 996 0 - 997 0 - 998 1 - 999 1 - Length: 1000, dtype: int64), which will be an error in a future version of pytest. Did you mean to use `assert` instead of `return`? - warnings.warn( - - -tests/test_wrappers/test_sklearn_wrapper.py::test_get_feature_names_out_transformers[transformer6-None] - /Users/luis/miniforge3/envs/py310/lib/python3.10/site-packages/sklearn/preprocessing/_function_transformer.py:307: RuntimeWarning: invalid value encountered in log - return func(X, **(kw_args if kw_args else {}))