diff --git a/feature_engine/imputation/base_imputer.py b/feature_engine/imputation/base_imputer.py index 60dee5df9..0ab657a1d 100644 --- a/feature_engine/imputation/base_imputer.py +++ b/feature_engine/imputation/base_imputer.py @@ -60,9 +60,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: X = self._transform(X) # Replace missing data with learned parameters - X.fillna(value=self.imputer_dict_, inplace=True) - - return X + return X.fillna(value=self.imputer_dict_) def _get_feature_names_in(self, X): """Get the names and number of features in the train set (the dataframe diff --git a/feature_engine/imputation/categorical.py b/feature_engine/imputation/categorical.py index 16942f598..096dfa22e 100644 --- a/feature_engine/imputation/categorical.py +++ b/feature_engine/imputation/categorical.py @@ -134,7 +134,6 @@ def __init__( return_object: bool = False, ignore_format: bool = False, ) -> None: - if imputation_method not in ["missing", "frequent"]: raise ValueError( "imputation_method takes only values 'missing' or 'frequent'" @@ -180,7 +179,6 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): self.imputer_dict_ = {var: self.fill_value for var in self.variables_} elif self.imputation_method == "frequent": - # if imputing only 1 variable: if len(self.variables_) == 1: var = self.variables_[0] @@ -219,7 +217,6 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): return self def transform(self, X: pd.DataFrame) -> pd.DataFrame: - # Frequent category imputation if self.imputation_method == "frequent": X = super().transform(X) @@ -230,13 +227,18 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # if variable is of type category, we need to add the new # category, before filling in the nan + add_cats = {} for variable in self.variables_: if pd.api.types.is_categorical_dtype(X[variable]): - X[variable].cat.add_categories( - self.imputer_dict_[variable], inplace=True + add_cats.update( + { + variable: X[variable].cat.add_categories( + self.imputer_dict_[variable] + ) + } ) - X.fillna(self.imputer_dict_, inplace=True) + X = X.assign(**add_cats).fillna(self.imputer_dict_) # add additional step to return variables cast as object if self.return_object: diff --git a/tests/test_encoding/test_encoders/test_similarity_encoder.py b/tests/test_encoding/test_encoders/test_similarity_encoder.py index 8e2336131..3e74b3717 100644 --- a/tests/test_encoding/test_encoders/test_similarity_encoder.py +++ b/tests/test_encoding/test_encoders/test_similarity_encoder.py @@ -142,7 +142,7 @@ def test_nan_behaviour_impute(df_enc_big_na): def test_nan_behaviour_ignore(df_enc_big_na): encoder = StringSimilarityEncoder(missing_values="ignore") X = encoder.fit_transform(df_enc_big_na) - assert (X.isna().any(1) == df_enc_big_na.isna().any(1)).all() + assert (X.isna().any(axis=1) == df_enc_big_na.isna().any(axis=1)).all() assert encoder.encoder_dict_ == { "var_A": ["B", "D", "G", "A", "C", "E", "F"], "var_B": ["A", "D", "B", "G", "C", "E", "F"], diff --git a/tests/test_imputation/test_categorical_imputer.py b/tests/test_imputation/test_categorical_imputer.py index d10640b2c..95de56a44 100644 --- a/tests/test_imputation/test_categorical_imputer.py +++ b/tests/test_imputation/test_categorical_imputer.py @@ -150,7 +150,6 @@ def test_error_when_imputation_method_not_frequent_or_missing(): def test_error_when_variable_contains_multiple_modes(df_na): - msg = "The variable Name contains multiple frequent categories." imputer = CategoricalImputer(imputation_method="frequent", variables="Name") with pytest.raises(ValueError) as record: @@ -245,8 +244,9 @@ def test_variables_cast_as_category_missing(df_na): X_reference["Name"] = X_reference["Name"].fillna("Missing") X_reference["Studies"] = X_reference["Studies"].fillna("Missing") - X_reference["City"].cat.add_categories("Missing", inplace=True) - X_reference["City"] = X_reference["City"].fillna("Missing") + X_reference["City"] = ( + X_reference["City"].cat.add_categories("Missing").fillna("Missing") + ) # test fit attributes assert imputer.variables_ == ["Name", "City", "Studies"] @@ -265,7 +265,6 @@ def test_variables_cast_as_category_missing(df_na): def test_variables_cast_as_category_frequent(df_na): - df_na = df_na.copy() df_na["City"] = df_na["City"].astype("category") diff --git a/tests/test_selection/test_drop_high_psi_features.py b/tests/test_selection/test_drop_high_psi_features.py index 2764ea716..004a93baa 100644 --- a/tests/test_selection/test_drop_high_psi_features.py +++ b/tests/test_selection/test_drop_high_psi_features.py @@ -266,7 +266,7 @@ def test_split_col_not_included_in_variables(df): def test_error_if_na_in_split_col(df): """Test an error is raised if the split column contains missing values.""" data = df.copy() - data["var_3"].iloc[15] = np.nan + data.iloc[15, data.columns.get_loc("var_3")] = np.nan transformer = DropHighPSIFeatures(split_col="var_3") @@ -277,7 +277,7 @@ def test_error_if_na_in_split_col(df): def test_raise_error_if_na_in_df(df): """Test an error is raised when missing values is set to raise.""" data = df.copy() - data["var_3"].iloc[15] = np.nan + data.iloc[15, data.columns.get_loc("var_3")] = np.nan transformer = DropHighPSIFeatures(missing_values="raise") @@ -288,7 +288,7 @@ def test_raise_error_if_na_in_df(df): def test_missing_value_ignored(df): """Test if PSI are computed when missing values are present in the dataframe.""" data = df.copy() - data["var_3"].iloc[15] = np.nan + data.iloc[15, data.columns.get_loc("var_3")] = np.nan var_col = [col for col in data if "var" in col] @@ -301,7 +301,7 @@ def test_missing_value_ignored(df): def test_raise_error_if_inf_in_df(df): """Test an error is raised for inf when missing values is set to raise.""" data = df.copy() - data["var_3"].iloc[15] = np.inf + data.iloc[15, data.columns.get_loc("var_3")] = np.nan transformer = DropHighPSIFeatures(missing_values="raise") diff --git a/tests/test_wrappers/test_sklearn_wrapper.py b/tests/test_wrappers/test_sklearn_wrapper.py index 81a425a98..d600e2ae7 100644 --- a/tests/test_wrappers/test_sklearn_wrapper.py +++ b/tests/test_wrappers/test_sklearn_wrapper.py @@ -1,6 +1,7 @@ import numpy as np import pandas as pd import pytest +from sklearn import __version__ as skl_version from sklearn.base import clone from sklearn.datasets import fetch_california_housing from sklearn.decomposition import PCA @@ -50,11 +51,21 @@ ] +def _OneHotEncoder(sparse, drop=None, dtype=np.float64) -> OneHotEncoder: + """OneHotEncoder sparse argument has been renamed as sparse_output + in scikitlearn >=1.2""" + + if skl_version.split(".")[0] == "1" and int(skl_version.split(".")[1]) >= 2: + return OneHotEncoder(sparse_output=sparse, drop=drop, dtype=dtype) + else: + return OneHotEncoder(sparse=sparse, drop=drop, dtype=dtype) + + @pytest.mark.parametrize( "transformer", [ SimpleImputer(), - OneHotEncoder(sparse=False), + _OneHotEncoder(sparse=False), StandardScaler(), SelectKBest(), ], @@ -319,7 +330,7 @@ def test_sklearn_ohe_object_one_feature(df_vartypes): variables_to_encode = ["Name"] transformer = SklearnTransformerWrapper( - transformer=OneHotEncoder(sparse=False, dtype=np.int64), + transformer=_OneHotEncoder(sparse=False, dtype=np.int64), variables=variables_to_encode, ) @@ -341,7 +352,7 @@ def test_sklearn_ohe_object_many_features(df_vartypes): variables_to_encode = ["Name", "City"] transformer = SklearnTransformerWrapper( - transformer=OneHotEncoder(sparse=False, dtype=np.int64), + transformer=_OneHotEncoder(sparse=False, dtype=np.int64), variables=variables_to_encode, ) @@ -367,7 +378,7 @@ def test_sklearn_ohe_numeric(df_vartypes): variables_to_encode = ["Age"] transformer = SklearnTransformerWrapper( - transformer=OneHotEncoder(sparse=False, dtype=np.int64), + transformer=_OneHotEncoder(sparse=False, dtype=np.int64), variables=variables_to_encode, ) @@ -387,7 +398,7 @@ def test_sklearn_ohe_numeric(df_vartypes): def test_sklearn_ohe_all_features(df_vartypes): transformer = SklearnTransformerWrapper( - transformer=OneHotEncoder(sparse=False, dtype=np.int64) + transformer=_OneHotEncoder(sparse=False, dtype=np.int64) ) ref = pd.DataFrame( @@ -443,7 +454,7 @@ def test_sklearn_ohe_with_crossvalidation(): ( "encode_cat", SklearnTransformerWrapper( - transformer=OneHotEncoder(drop="first", sparse=False), + transformer=_OneHotEncoder(drop="first", sparse=False), variables=["AveBedrms_cat"], ), ), @@ -459,7 +470,7 @@ def test_sklearn_ohe_with_crossvalidation(): def test_wrap_one_hot_encoder_get_features_name_out(df_vartypes): - ohe_wrap = SklearnTransformerWrapper(transformer=OneHotEncoder(sparse=False)) + ohe_wrap = SklearnTransformerWrapper(transformer=_OneHotEncoder(sparse=False)) ohe_wrap.fit(df_vartypes) expected_features_all = [ @@ -541,7 +552,6 @@ def test_error_when_inverse_transform_not_implemented(transformer): ) @pytest.mark.parametrize("transformer", _transformers) def test_get_feature_names_out_transformers(varlist, transformer): - X = fetch_california_housing(as_frame=True).frame tr_wrap = SklearnTransformerWrapper(transformer=transformer, variables=varlist) Xw = tr_wrap.fit_transform(X) @@ -600,9 +610,8 @@ def test_get_feature_names_out_polynomialfeatures(varlist): @pytest.mark.parametrize("varlist", [["Name", "City"], None]) def test_get_feature_names_out_ohe(varlist, df_vartypes): - transformer = SklearnTransformerWrapper( - transformer=OneHotEncoder(sparse=False, dtype=np.int64), + transformer=_OneHotEncoder(sparse=False, dtype=np.int64), variables=varlist, )