From 0f7ef8d04eaf54011eece8d0d1b96bb16f81c7c5 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Mon, 4 Sep 2023 19:38:57 +0200 Subject: [PATCH 01/26] fix readme badge --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 13130850d..4df8471fb 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,7 @@ [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![GitHub contributors](https://img.shields.io/github/contributors/feature-engine/feature_engine?logo=GitHub)](https://github.com/feature-engine/feature_engine/graphs/contributors) [![Gitter](https://img.shields.io/gitter/room/feature-engine/feaure_engine?logo=Gitter)](https://gitter.im/feature_engine/community) -[![Total Downloads](https://pepy.tech/badge/feature-engine)](https://pepy.tech/project/feature-engine) -[![Monthly Downloads](https://pepy.tech/badge/feature-engine/month)](https://pepy.tech/project/feature-engine) +[![Monthly Downloads](https://img.shields.io/pypi/dm/feature-engine)](https://img.shields.io/pypi/dm/feature-engine) [![DOI](https://zenodo.org/badge/163630824.svg)](https://zenodo.org/badge/latestdoi/163630824) [![JOSS](https://joss.theoj.org/papers/10.21105/joss.03642/status.svg)](https://doi.org/10.21105/joss.03642) [![first-timers-only](https://img.shields.io/badge/first--timers--only-friendly-blue.svg?style=flat)](https://www.firsttimersonly.com/) From 147976c2ca736b05094fcef4625041237ebc7bdc Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Mon, 4 Sep 2023 20:35:20 +0200 Subject: [PATCH 02/26] fix datetime test failing --- tests/test_datetime/test_datetime_features.py | 40 ++++++++++++------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/tests/test_datetime/test_datetime_features.py b/tests/test_datetime/test_datetime_features.py index 1727f27b6..5ba862c3d 100644 --- a/tests/test_datetime/test_datetime_features.py +++ b/tests/test_datetime/test_datetime_features.py @@ -305,33 +305,43 @@ def test_extract_features_from_categorical_variable( ) -def test_extract_features_from_different_timezones( - df_datetime, df_datetime_transformed -): - time_zones = [4, -1, 9, -7] - tz_df = pd.DataFrame( - {"time_obj": df_datetime["time_obj"].add(["+4", "-1", "+9", "-7"])} +def test_extract_features_from_different_timezones(): + df = pd.DataFrame() + df["time"] = pd.concat( + [ + pd.Series( + pd.date_range( + start="2014-08-01 09:00", freq="H", periods=3, tz="Europe/Berlin" + ) + ), + pd.Series( + pd.date_range( + start="2014-08-01 09:00", freq="H", periods=3, tz="US/Central" + ) + ), + ], + axis=0, ) + df.reset_index(inplace=True, drop=True) + transformer = DatetimeFeatures( - variables="time_obj", features_to_extract=["hour"], utc=True + variables="time", features_to_extract=["hour"], utc=True ) - X = transformer.fit_transform(tz_df) + X = transformer.fit_transform(df) pd.testing.assert_frame_equal( X, - df_datetime_transformed[["time_obj_hour"]].apply( - lambda x: x.subtract(time_zones) - ), + pd.DataFrame({"time_hour": [7, 8, 9, 14, 15, 16]}), check_dtype=False, ) exp_err_msg = ( - "ValueError: variable(s) time_obj " - "could not be converted to datetime. Try setting utc=True" + "Tz-aware datetime.datetime cannot be converted to datetime64 " + "unless utc=True, at position 3" ) with pytest.raises(ValueError) as errinfo: assert DatetimeFeatures( - variables="time_obj", features_to_extract=["hour"], utc=False - ).fit_transform(tz_df) + variables="time", features_to_extract=["hour"], utc=False + ).fit_transform(df) assert str(errinfo.value) == exp_err_msg From b895b017e6786c6bb77bb0c84a7cd2cc5519c627 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Mon, 4 Sep 2023 21:11:14 +0200 Subject: [PATCH 03/26] add format parameter, updated documentation on datetime! --- docs/user_guide/datetime/DatetimeFeatures.rst | 29 ++++++++++++------- feature_engine/datetime/datetime.py | 12 ++++++++ 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/docs/user_guide/datetime/DatetimeFeatures.rst b/docs/user_guide/datetime/DatetimeFeatures.rst index 77572fa79..b5157a16b 100644 --- a/docs/user_guide/datetime/DatetimeFeatures.rst +++ b/docs/user_guide/datetime/DatetimeFeatures.rst @@ -77,7 +77,7 @@ First, we will create a toy dataframe with 2 date variables: toy_df = pd.DataFrame({ "var_date1": ['May-1989', 'Dec-2020', 'Jan-1999', 'Feb-2002'], - "var_date2": ['06/21/12', '02/10/98', '08/03/10', '10/31/20'], + "var_date2": ['06/21/2012', '02/10/1998', '08/03/2010', '10/31/2020'], }) Now, we will extract the variables month, month-end and the day of the year from the @@ -148,9 +148,14 @@ First, let's create a toy dataset with 2 time variables and an object variable. datetime. So if we want to extract time features from all our datetime variables, we don't need to specify them. +Note that from version 2.0.0 pandas deprecated the parameter `infer_datetime_format`. +Hence, if you want pandas to infer the datetime format and you have different formats, +you need to explicitly say so by passing `"mixed"` to the `format` parameter as shown +below. + .. code:: python - dfts = DatetimeFeatures(features_to_extract=["minute"]) + dfts = DatetimeFeatures(features_to_extract=["minute"], format="mixed") df_transf = dfts.fit_transform(toy_df) @@ -227,10 +232,11 @@ the features. variables=["var_dt1", "var_dt3"], features_to_extract=["year", "hour"], drop_original=False, + format="mixed", ) df_transf = dfts.fit_transform(toy_df) - print(df_transf) + df_transf We can see the resulting dataframe in the following output: @@ -373,10 +379,11 @@ And now we mistakenly extract only date features: dfts = DatetimeFeatures( features_to_extract=["year", "month", "day_of_week"], + format="mixed", ) df_transf = dfts.fit_transform(toy_df) - print(df_transf) + df_transf .. code:: python @@ -413,6 +420,7 @@ And we mistakenly extract the hour and the minute: dfts = DatetimeFeatures( features_to_extract=["hour", "minute"], + format="mixed", ) df_transf = dfts.fit_transform(toy_df) @@ -466,7 +474,7 @@ To do this, we leave the parameter `features_to_extract` to `None`. df_transf = dfts.fit_transform(toy_df) - print(df_transf) + df_transf .. code:: python @@ -614,7 +622,7 @@ from the dataset. .. code:: python pipe = Pipeline([ - ('datetime', DatetimeFeatures()), + ('datetime', DatetimeFeatures(format="mixed")), ('drop_constant', DropConstantFeatures()), ]) @@ -683,12 +691,13 @@ converts all data to UTC timezone. dfts = DatetimeFeatures( features_to_extract=["hour", "minute"], drop_original=False, - utc=True + utc=True, + format="mixed", ) df_transf = dfts.fit_transform(toy_df) - print(df_transf) + df_transf .. code:: python @@ -709,7 +718,7 @@ the datetime information extracted as if it were in UTC timezone. from feature_engine.datetime import DatetimeFeatures var_tz = pd.Series(['08/31/00 12:34:45', '12/01/90 23:01:02', '04/25/01 11:59:21']) - var_tz = pd.to_datetime(var_tz) + var_tz = pd.to_datetime(var_tz, format="mixed") var_tz = var_tz.dt.tz_localize("US/eastern") var_tz @@ -735,7 +744,7 @@ timezone. df_transf = dfts.fit_transform(toy_df) - print(df_transf) + df_transf .. code:: python diff --git a/feature_engine/datetime/datetime.py b/feature_engine/datetime/datetime.py index 246d281a2..e78c09e13 100644 --- a/feature_engine/datetime/datetime.py +++ b/feature_engine/datetime/datetime.py @@ -125,6 +125,14 @@ class DatetimeFeatures(BaseEstimator, TransformerMixin, GetFeatureNamesOutMixin) Return UTC DatetimeIndex if True (converting any tz-aware datetime.datetime objects as well). Same as in `pandas.to_datetime`. + format: str, default None + The strftime to parse time, e.g. "%d/%m/%Y". Check pandas `to_datetime()` for + more information on choices. If you have variables with different formats pass + “mixed”, to infer the format for each element individually. This is risky, + and you should probably use it along with dayfirst, according to pandas' + documentation. + + Attributes ---------- variables_: @@ -177,6 +185,7 @@ def __init__( dayfirst: bool = False, yearfirst: bool = False, utc: Union[None, bool] = None, + format: Union[None, str] = None, ) -> None: if features_to_extract: @@ -217,6 +226,7 @@ def __init__( self.yearfirst = yearfirst self.utc = utc self.features_to_extract = features_to_extract + self.format = format def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ @@ -316,6 +326,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: dayfirst=self.dayfirst, yearfirst=self.yearfirst, utc=self.utc, + format=self.format, ), index=X.index, ) @@ -337,6 +348,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: dayfirst=self.dayfirst, yearfirst=self.yearfirst, utc=self.utc, + format=self.format, ) for variable in self.variables_ ], From 5c61f89ebd9bdd5677070513747d3243ae38103c Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Mon, 4 Sep 2023 21:41:30 +0200 Subject: [PATCH 04/26] add test for strings on different timezones, remove unused logic, add changes to whats new --- docs/whats_new/v_160.rst | 26 +++++++++++++++++++ feature_engine/datetime/datetime.py | 20 +++++++------- tests/test_datetime/test_datetime_features.py | 21 +++++++++++++++ 3 files changed, 58 insertions(+), 9 deletions(-) diff --git a/docs/whats_new/v_160.rst b/docs/whats_new/v_160.rst index 27f2fa92f..24f5ede9d 100644 --- a/docs/whats_new/v_160.rst +++ b/docs/whats_new/v_160.rst @@ -1,6 +1,32 @@ Version 1.6.X ============= +Version 1.6.2 +------------- + +Deployed: xx September 2023 + +Contributors +~~~~~~~~~~~~ + +- `Soledad Galli `_ + +New functionality +~~~~~~~~~~~~~~~~~ + +- `DatetimeFeatures()` can now specify the format of the datetime variable (`Soledad Galli `_) + +Bug fixes +~~~~~~~~~ + +- Fix failing test for `DatetimeFeatures()` (`Soledad Galli `_) + +Code improvements +~~~~~~~~~~~~~~~~~ + +- Routine in `DatetimeFeatures()` does not enter into our check for utc=True when working with different timezones any more, commented out, to be removed in version 1.7.0 (`Soledad Galli `_) + + Version 1.6.1 ------------- diff --git a/feature_engine/datetime/datetime.py b/feature_engine/datetime/datetime.py index e78c09e13..cdc3e0f54 100644 --- a/feature_engine/datetime/datetime.py +++ b/feature_engine/datetime/datetime.py @@ -355,15 +355,17 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: axis=1, ) - non_dt_columns = datetime_df.columns[ - ~datetime_df.apply(is_datetime) - ].tolist() - if non_dt_columns: - raise ValueError( - "ValueError: variable(s) " - + (len(non_dt_columns) * "{} ").format(*non_dt_columns) - + "could not be converted to datetime. Try setting utc=True" - ) + # With pandas update, it does not enter here any more + #TODO: delete in a few releases from now + # non_dt_columns = datetime_df.columns[ + # ~datetime_df.apply(is_datetime) + # ].tolist() + # if non_dt_columns: + # raise ValueError( + # "ValueError: variable(s) " + # + (len(non_dt_columns) * "{} ").format(*non_dt_columns) + # + "could not be converted to datetime. Try setting utc=True" + # ) # create new features for var in self.variables_: diff --git a/tests/test_datetime/test_datetime_features.py b/tests/test_datetime/test_datetime_features.py index 5ba862c3d..de8d99032 100644 --- a/tests/test_datetime/test_datetime_features.py +++ b/tests/test_datetime/test_datetime_features.py @@ -345,6 +345,27 @@ def test_extract_features_from_different_timezones(): assert str(errinfo.value) == exp_err_msg +def test_extract_features_from_different_timezones_when_string( + df_datetime, df_datetime_transformed +): + time_zones = [4, -1, 9, -7] + tz_df = pd.DataFrame( + {"time_obj": df_datetime["time_obj"].add(["+4", "-1", "+9", "-7"])} + ) + transformer = DatetimeFeatures( + variables="time_obj", features_to_extract=["hour"], utc=True, format="mixed", + ) + X = transformer.fit_transform(tz_df) + + pd.testing.assert_frame_equal( + X, + df_datetime_transformed[["time_obj_hour"]].apply( + lambda x: x.subtract(time_zones) + ), + check_dtype=False, + ) + + def test_extract_features_from_localized_tz_variables(): tz_df = pd.DataFrame( { From 8c4c6832ec4324afb059e157cfdc954dd1baa53f Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 5 Sep 2023 10:25:14 +0200 Subject: [PATCH 05/26] fix compatibiity count encoder, removes downcast --- feature_engine/encoding/base_encoder.py | 4 ++-- tests/test_encoding/test_count_frequency_encoder.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/feature_engine/encoding/base_encoder.py b/feature_engine/encoding/base_encoder.py index c642cc4c1..8bafd8e89 100644 --- a/feature_engine/encoding/base_encoder.py +++ b/feature_engine/encoding/base_encoder.py @@ -220,7 +220,7 @@ def _encode(self, X: pd.DataFrame) -> pd.DataFrame: # if original variables are cast as categorical, they will remain # categorical after the encoding, and this is probably not desired - if pd.api.types.is_categorical_dtype(X[feature]): + if X[feature].dtype.name =="category": if all(isinstance(x, int) for x in X[feature]): X[feature] = X[feature].astype("int") else: @@ -228,7 +228,7 @@ def _encode(self, X: pd.DataFrame) -> pd.DataFrame: if self.unseen == "encode": X[self.variables_] = X[self.variables_].fillna( - self._unseen, downcast="infer" + self._unseen ) else: # check if nan values were introduced by the transformation diff --git a/tests/test_encoding/test_count_frequency_encoder.py b/tests/test_encoding/test_count_frequency_encoder.py index f961a2960..f98f8ccf5 100644 --- a/tests/test_encoding/test_count_frequency_encoder.py +++ b/tests/test_encoding/test_count_frequency_encoder.py @@ -279,7 +279,7 @@ def test_zero_encoding_for_new_categories(): # check that the counts are correct for both new and old expected_result = pd.DataFrame({"col1": [3, 0, 1, 3, 1], "col2": [2, 2, 1, 2, 0]}) - pd.testing.assert_frame_equal(result, expected_result) + pd.testing.assert_frame_equal(result, expected_result, check_dtype=False) def test_zero_encoding_for_unseen_categories_if_unseen_is_encode(): @@ -299,7 +299,7 @@ def test_zero_encoding_for_unseen_categories_if_unseen_is_encode(): # check that the counts are correct expected_result = pd.DataFrame({"col1": [3, 0, 1, 3, 1], "col2": [2, 2, 1, 2, 0]}) - pd.testing.assert_frame_equal(result, expected_result) + pd.testing.assert_frame_equal(result, expected_result, check_dtype=False) # with frequency encoder = CountFrequencyEncoder(encoding_method="frequency", unseen="encode").fit( From 35f4c90c85cf95d8b2d4345185535755011cc948 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 5 Sep 2023 11:01:59 +0200 Subject: [PATCH 06/26] fix style, type and add changes to whats new --- docs/whats_new/v_160.rst | 7 ++++++- feature_engine/creation/math_features.py | 2 +- feature_engine/datetime/datetime.py | 12 ------------ feature_engine/encoding/base_encoder.py | 6 ++---- feature_engine/selection/shuffle_features.py | 4 ++-- .../test_check_estimator_prediction.py | 8 ++++---- tests/test_selection/test_drop_features.py | 6 +++--- 7 files changed, 18 insertions(+), 27 deletions(-) diff --git a/docs/whats_new/v_160.rst b/docs/whats_new/v_160.rst index 24f5ede9d..1be889379 100644 --- a/docs/whats_new/v_160.rst +++ b/docs/whats_new/v_160.rst @@ -19,12 +19,17 @@ New functionality Bug fixes ~~~~~~~~~ +This bugs were introduced by the latest releases of pandas and other dependencies. + - Fix failing test for `DatetimeFeatures()` (`Soledad Galli `_) +- Fix failing test for many encoders: removed `downcast=infer` as it will be deprecated (`Soledad Galli `_) +- Fix version related failing style checks (`Soledad Galli `_) +- Fix version related failing type checks (`Soledad Galli `_) Code improvements ~~~~~~~~~~~~~~~~~ -- Routine in `DatetimeFeatures()` does not enter into our check for utc=True when working with different timezones any more, commented out, to be removed in version 1.7.0 (`Soledad Galli `_) +- Routine in `DatetimeFeatures()` does not enter into our check for `utc=True` when working with different timezones any more (`Soledad Galli `_) Version 1.6.1 diff --git a/feature_engine/creation/math_features.py b/feature_engine/creation/math_features.py index 0b3acce8b..35cbe73aa 100644 --- a/feature_engine/creation/math_features.py +++ b/feature_engine/creation/math_features.py @@ -228,7 +228,7 @@ def _get_new_features_name(self) -> List: if isinstance(self.func, list): functions = [ - fun if type(fun) == str else fun.__name__ for fun in self.func + fun if type(fun) is str else fun.__name__ for fun in self.func ] feature_names = [ f"{function}_{'_'.join(varlist)}" for function in functions diff --git a/feature_engine/datetime/datetime.py b/feature_engine/datetime/datetime.py index cdc3e0f54..a9eb64499 100644 --- a/feature_engine/datetime/datetime.py +++ b/feature_engine/datetime/datetime.py @@ -355,18 +355,6 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: axis=1, ) - # With pandas update, it does not enter here any more - #TODO: delete in a few releases from now - # non_dt_columns = datetime_df.columns[ - # ~datetime_df.apply(is_datetime) - # ].tolist() - # if non_dt_columns: - # raise ValueError( - # "ValueError: variable(s) " - # + (len(non_dt_columns) * "{} ").format(*non_dt_columns) - # + "could not be converted to datetime. Try setting utc=True" - # ) - # create new features for var in self.variables_: for feat in self.features_to_extract_: diff --git a/feature_engine/encoding/base_encoder.py b/feature_engine/encoding/base_encoder.py index 8bafd8e89..3fb773b26 100644 --- a/feature_engine/encoding/base_encoder.py +++ b/feature_engine/encoding/base_encoder.py @@ -220,16 +220,14 @@ def _encode(self, X: pd.DataFrame) -> pd.DataFrame: # if original variables are cast as categorical, they will remain # categorical after the encoding, and this is probably not desired - if X[feature].dtype.name =="category": + if X[feature].dtype.name == "category": if all(isinstance(x, int) for x in X[feature]): X[feature] = X[feature].astype("int") else: X[feature] = X[feature].astype("float") if self.unseen == "encode": - X[self.variables_] = X[self.variables_].fillna( - self._unseen - ) + X[self.variables_] = X[self.variables_].fillna(self._unseen) else: # check if nan values were introduced by the transformation self._check_nan_values_after_transformation(X) diff --git a/feature_engine/selection/shuffle_features.py b/feature_engine/selection/shuffle_features.py index 62805bf3a..cb93917b9 100644 --- a/feature_engine/selection/shuffle_features.py +++ b/feature_engine/selection/shuffle_features.py @@ -1,4 +1,4 @@ -from typing import List, Union +from typing import List, Union, MutableSequence import numpy as np import pandas as pd @@ -189,7 +189,7 @@ def fit( self, X: pd.DataFrame, y: pd.Series, - sample_weight: Union[np.array, pd.Series, List] = None, + sample_weight: Union[MutableSequence, None] = None, ): """ Find the important features. diff --git a/tests/test_prediction/test_check_estimator_prediction.py b/tests/test_prediction/test_check_estimator_prediction.py index 25d825be5..62a8ae5f2 100644 --- a/tests/test_prediction/test_check_estimator_prediction.py +++ b/tests/test_prediction/test_check_estimator_prediction.py @@ -220,17 +220,17 @@ def test_attributes_upon_fitting(_strategy, _bins, estimator): if _strategy == "equal_width": assert ( type(transformer._pipeline.named_steps["discretiser"]) - == EqualWidthDiscretiser + is EqualWidthDiscretiser ) else: assert ( type(transformer._pipeline.named_steps["discretiser"]) - == EqualFrequencyDiscretiser + is EqualFrequencyDiscretiser ) - assert type(transformer._pipeline.named_steps["encoder_num"]) == MeanEncoder + assert type(transformer._pipeline.named_steps["encoder_num"]) is MeanEncoder - assert type(transformer._pipeline.named_steps["encoder_cat"]) == MeanEncoder + assert type(transformer._pipeline.named_steps["encoder_cat"]) is MeanEncoder @pytest.mark.parametrize("estimator", _estimators) diff --git a/tests/test_selection/test_drop_features.py b/tests/test_selection/test_drop_features.py index 7836ad735..fb77e8c8f 100644 --- a/tests/test_selection/test_drop_features.py +++ b/tests/test_selection/test_drop_features.py @@ -23,7 +23,7 @@ def test_drop_1_variable(df_vartypes): # transform params assert X.shape == (4, 4) - assert type(X) == pd.DataFrame + assert type(X) is pd.DataFrame pd.testing.assert_frame_equal(X, df) @@ -49,7 +49,7 @@ def test_drop_1_variables_str_input(df_vartypes): # transform params assert X.shape == (4, 4) - assert type(X) == pd.DataFrame + assert type(X) is pd.DataFrame pd.testing.assert_frame_equal(X, df) @@ -71,7 +71,7 @@ def test_drop_2_variables(df_vartypes): # transform params assert X.shape == (4, 3) - assert type(X) == pd.DataFrame + assert type(X) is pd.DataFrame pd.testing.assert_frame_equal(X, df) From cccbbda30d65862afd1dd98059bd7eec29a08e93 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 5 Sep 2023 11:25:37 +0200 Subject: [PATCH 07/26] take out metadata_routing from docs --- docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 1d85c16e3..19cc91c1c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -173,7 +173,7 @@ autodoc_default_options = { "members": True, "inherited-members": True, - "exclude-members": "set_output", + "exclude-members": "set_output,metadata_routing", } # generate autosummary even if no references @@ -195,7 +195,7 @@ "pandas": ("https://pandas.pydata.org/docs/", None), "scipy": ("https://docs.scipy.org/doc/scipy/reference", None), "matplotlib": ("https://matplotlib.org/", None), - "sklearn": ("http://scikit-learn.org/stable", None), + "sklearn": ("https://scikit-learn.org/stable/", None), } # -- Options for LaTeX output --------------------------------------------- From 9171a9f82af6b240d4c76db67764ce7b4b7fd084 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 5 Sep 2023 11:52:36 +0200 Subject: [PATCH 08/26] expose format in datetimesubstraction --- docs/user_guide/datetime/DatetimeSubtraction.rst | 1 + docs/whats_new/v_160.rst | 3 ++- feature_engine/datetime/datetime_subtraction.py | 10 ++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/user_guide/datetime/DatetimeSubtraction.rst b/docs/user_guide/datetime/DatetimeSubtraction.rst index 50f65322c..fa71449ee 100644 --- a/docs/user_guide/datetime/DatetimeSubtraction.rst +++ b/docs/user_guide/datetime/DatetimeSubtraction.rst @@ -235,6 +235,7 @@ parameter `missing_values` to `"ignore"`. Here is a code example: .. code:: python + import numpy as np import pandas as pd from feature_engine.datetime import DatetimeSubtraction diff --git a/docs/whats_new/v_160.rst b/docs/whats_new/v_160.rst index 1be889379..19d477b17 100644 --- a/docs/whats_new/v_160.rst +++ b/docs/whats_new/v_160.rst @@ -14,7 +14,7 @@ Contributors New functionality ~~~~~~~~~~~~~~~~~ -- `DatetimeFeatures()` can now specify the format of the datetime variable (`Soledad Galli `_) +- `DatetimeFeatures()` and `DatetimeSubtraction()` can now specify the format of the datetime variable (`Soledad Galli `_) Bug fixes ~~~~~~~~~ @@ -25,6 +25,7 @@ This bugs were introduced by the latest releases of pandas and other dependencie - Fix failing test for many encoders: removed `downcast=infer` as it will be deprecated (`Soledad Galli `_) - Fix version related failing style checks (`Soledad Galli `_) - Fix version related failing type checks (`Soledad Galli `_) +- Fix version related failing doc checks (`Soledad Galli `_) Code improvements ~~~~~~~~~~~~~~~~~ diff --git a/feature_engine/datetime/datetime_subtraction.py b/feature_engine/datetime/datetime_subtraction.py index 8fdb83b2e..90b9db7d8 100644 --- a/feature_engine/datetime/datetime_subtraction.py +++ b/feature_engine/datetime/datetime_subtraction.py @@ -112,6 +112,13 @@ class DatetimeSubtraction(BaseCreation): Return UTC DatetimeIndex if True (converting any tz-aware datetime.datetime objects as well). Same as in `pandas.to_datetime`. + format: str, default None + The strftime to parse time, e.g. "%d/%m/%Y". Check pandas `to_datetime()` for + more information on choices. If you have variables with different formats pass + “mixed”, to infer the format for each element individually. This is risky, + and you should probably use it along with dayfirst, according to pandas' + documentation. + Attributes ---------- variables_: @@ -153,6 +160,7 @@ def __init__( dayfirst: bool = False, yearfirst: bool = False, utc: Union[None, bool] = None, + format: Union[None, str] = None, ) -> None: valid_output_units = { @@ -197,6 +205,7 @@ def __init__( self.dayfirst = dayfirst self.yearfirst = yearfirst self.utc = utc + self.format = format def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ @@ -294,6 +303,7 @@ def _to_datetime(self, X: pd.DataFrame): dayfirst=self.dayfirst, yearfirst=self.yearfirst, utc=self.utc, + format=self.format, ) for variable in set(self.variables_ + self.reference_) ], From 07b633e747800faa1aa52f329172072a1ac1feec Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 5 Sep 2023 12:31:49 +0200 Subject: [PATCH 09/26] fix metadata_routing --- docs/conf.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 19cc91c1c..6ae9bc8c4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -173,7 +173,7 @@ autodoc_default_options = { "members": True, "inherited-members": True, - "exclude-members": "set_output,metadata_routing", + "exclude-members": "set_output", } # generate autosummary even if no references @@ -273,9 +273,6 @@ # A list of files that should not be packed into the epub file. epub_exclude_files = ["search.html"] -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {"https://docs.python.org/": None} - # The following is used by sphinx.ext.linkcode to provide links to github linkcode_resolve = make_linkcode_resolve( "feature_engine", From 4c398994be64c6b4a1d7baebd784810d7db326df Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 5 Sep 2023 12:57:42 +0200 Subject: [PATCH 10/26] fix error in rare categories list comparison --- tests/test_encoding/test_rare_label_encoder.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_encoding/test_rare_label_encoder.py b/tests/test_encoding/test_rare_label_encoder.py index ba8529971..9594e1cc3 100644 --- a/tests/test_encoding/test_rare_label_encoder.py +++ b/tests/test_encoding/test_rare_label_encoder.py @@ -1,3 +1,5 @@ +from collections import Counter + import numpy as np import pandas as pd import pytest @@ -133,12 +135,13 @@ def test_correctly_ignores_nan_in_fit(df_enc_big): encoder.fit(df) # expected: - frequenc_cat = { + frequent_cat = { "var_A": ["B", "D", "A", "G", "C"], "var_B": ["A", "D", "B", "G", "C"], "var_C": ["C", "D", "B", "A"], } - assert encoder.encoder_dict_ == frequenc_cat + for key in frequent_cat.keys(): + assert Counter(encoder.encoder_dict_[key]) == Counter(frequent_cat[key]) # input t = pd.DataFrame( @@ -217,12 +220,13 @@ def test_correctly_ignores_nan_in_fit_when_var_is_numerical(df_enc_big): encoder.fit(df) # expected: - frequenc_cat = { + frequent_cat = { "var_A": ["B", "D", "A", "G", "C"], "var_B": ["A", "D", "B", "G", "C"], "var_C": [3, 4, 2, 1], } - assert encoder.encoder_dict_ == frequenc_cat + for key in frequent_cat.keys(): + assert Counter(encoder.encoder_dict_[key]) == Counter(frequent_cat[key]) # input t = pd.DataFrame( From 4ab8fe9a0cac5240f607efa60d7416dcd8edc7f9 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Thu, 7 Sep 2023 11:13:24 +0200 Subject: [PATCH 11/26] fix test yeojohnson --- docs/whats_new/v_160.rst | 2 ++ feature_engine/transformation/yeojohnson.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/docs/whats_new/v_160.rst b/docs/whats_new/v_160.rst index 19d477b17..522c08ce1 100644 --- a/docs/whats_new/v_160.rst +++ b/docs/whats_new/v_160.rst @@ -21,6 +21,8 @@ Bug fixes This bugs were introduced by the latest releases of pandas and other dependencies. +- Fix failing test for `YeoJohnsonTransformer()` (`Soledad Galli `_) +- Fix failing test for `RareLabelEncoder()` (`Soledad Galli `_) - Fix failing test for `DatetimeFeatures()` (`Soledad Galli `_) - Fix failing test for many encoders: removed `downcast=infer` as it will be deprecated (`Soledad Galli `_) - Fix version related failing style checks (`Soledad Galli `_) diff --git a/feature_engine/transformation/yeojohnson.py b/feature_engine/transformation/yeojohnson.py index 7f03b468e..243840244 100644 --- a/feature_engine/transformation/yeojohnson.py +++ b/feature_engine/transformation/yeojohnson.py @@ -17,6 +17,7 @@ ) from feature_engine._docstrings.methods import _fit_transform_docstring from feature_engine._docstrings.substitute import Substitution +from feature_engine.tags import _return_tags from feature_engine.variable_handling._init_parameter_checks import ( _check_init_parameter_variables, ) @@ -152,3 +153,18 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: X[feature] = stats.yeojohnson(X[feature], lmbda=self.lambda_dict_[feature]) return X + + def _more_tags(self): + tags_dict = _return_tags() + tags_dict["variables"] = "numerical" + + # ======= this tests fail because the transformers throw an error + # when the values are 0. Nothing to do with the test itself but + # mostly with the data created and used in the test + msg = ( + "Transformer raises error when it can't find the optimal lambda for " + "the transformation, thus this check fails." + ) + tags_dict["_xfail_checks"]["check_fit2d_1sample"] = msg + + return tags_dict From 1d1bfa3144bfc7adc63181dfd44698db0c6b070f Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Thu, 7 Sep 2023 11:40:55 +0200 Subject: [PATCH 12/26] fix future warning categorical imputer --- docs/whats_new/v_160.rst | 6 ++++-- feature_engine/imputation/categorical.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/whats_new/v_160.rst b/docs/whats_new/v_160.rst index 522c08ce1..a90f8922d 100644 --- a/docs/whats_new/v_160.rst +++ b/docs/whats_new/v_160.rst @@ -14,12 +14,12 @@ Contributors New functionality ~~~~~~~~~~~~~~~~~ -- `DatetimeFeatures()` and `DatetimeSubtraction()` can now specify the format of the datetime variable (`Soledad Galli `_) +- `DatetimeFeatures()` and `DatetimeSubtraction()` can now specify the format of the datetime variables (`Soledad Galli `_) Bug fixes ~~~~~~~~~ -This bugs were introduced by the latest releases of pandas and other dependencies. +This bugs were introduced by the latest releases of pandas, Scikit-learn and Scipy. - Fix failing test for `YeoJohnsonTransformer()` (`Soledad Galli `_) - Fix failing test for `RareLabelEncoder()` (`Soledad Galli `_) @@ -28,6 +28,8 @@ This bugs were introduced by the latest releases of pandas and other dependencie - Fix version related failing style checks (`Soledad Galli `_) - Fix version related failing type checks (`Soledad Galli `_) - Fix version related failing doc checks (`Soledad Galli `_) +- Fix future warning categorical imputation (`Soledad Galli `_) + Code improvements ~~~~~~~~~~~~~~~~~ diff --git a/feature_engine/imputation/categorical.py b/feature_engine/imputation/categorical.py index 096dfa22e..45a723a86 100644 --- a/feature_engine/imputation/categorical.py +++ b/feature_engine/imputation/categorical.py @@ -229,7 +229,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # category, before filling in the nan add_cats = {} for variable in self.variables_: - if pd.api.types.is_categorical_dtype(X[variable]): + if X[variable].dtype.name == "category": add_cats.update( { variable: X[variable].cat.add_categories( From 03b3adac7668305539f186edd5509ef674fc3228 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Thu, 7 Sep 2023 11:49:20 +0200 Subject: [PATCH 13/26] fix performance issue one hot encoder --- feature_engine/encoding/one_hot.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/feature_engine/encoding/one_hot.py b/feature_engine/encoding/one_hot.py index 68a219790..6fd6f6bf4 100644 --- a/feature_engine/encoding/one_hot.py +++ b/feature_engine/encoding/one_hot.py @@ -274,7 +274,10 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: for feature in self.variables_: for category in self.encoder_dict_[feature]: - X[f"{feature}_{category}"] = np.where(X[feature] == category, 1, 0) + dummy_df = pd.DataFrame( + {f"{feature}_{category}": np.where(X[feature] == category, 1, 0)} + ) + X = pd.concat([X, dummy_df], axis=1) # drop the original non-encoded variables. X.drop(labels=self.variables_, axis=1, inplace=True) From 0a61975236eb528595155a396115bed767c186ce Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Thu, 7 Sep 2023 11:51:14 +0200 Subject: [PATCH 14/26] update change log --- docs/whats_new/v_160.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/whats_new/v_160.rst b/docs/whats_new/v_160.rst index a90f8922d..b7af7f9c9 100644 --- a/docs/whats_new/v_160.rst +++ b/docs/whats_new/v_160.rst @@ -35,6 +35,7 @@ Code improvements ~~~~~~~~~~~~~~~~~ - Routine in `DatetimeFeatures()` does not enter into our check for `utc=True` when working with different timezones any more (`Soledad Galli `_) +- Improve performance in `OneHotEncoder()` (`Soledad Galli `_) Version 1.6.1 From 4d32c323e369162fbfbab11a947c9dedf23864e3 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sat, 9 Sep 2023 08:30:18 +0200 Subject: [PATCH 15/26] add match on indexes one hot encoder --- feature_engine/encoding/one_hot.py | 3 ++- tests/test_encoding/test_onehot_encoder.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/feature_engine/encoding/one_hot.py b/feature_engine/encoding/one_hot.py index 6fd6f6bf4..de62e44c9 100644 --- a/feature_engine/encoding/one_hot.py +++ b/feature_engine/encoding/one_hot.py @@ -275,7 +275,8 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: for feature in self.variables_: for category in self.encoder_dict_[feature]: dummy_df = pd.DataFrame( - {f"{feature}_{category}": np.where(X[feature] == category, 1, 0)} + {f"{feature}_{category}": np.where(X[feature] == category, 1, 0)}, + index=X.index, ) X = pd.concat([X, dummy_df], axis=1) diff --git a/tests/test_encoding/test_onehot_encoder.py b/tests/test_encoding/test_onehot_encoder.py index 42448be12..aca3448be 100644 --- a/tests/test_encoding/test_onehot_encoder.py +++ b/tests/test_encoding/test_onehot_encoder.py @@ -5,6 +5,24 @@ from feature_engine.encoding import OneHotEncoder +@pytest.mark.parametrize("index_", [[1, 2, 3], [3, 2, 1], [4, 9, 2]]) +def test_concat_with_non_ordered_index(index_): + df = pd.DataFrame({"varA": ["a", "b", "c"], "varB": ["d", "d", "a"]}, index=index_) + encoder = OneHotEncoder() + dft = encoder.fit_transform(df) + df_expected = pd.DataFrame( + { + "varA_a": [1, 0, 0], + "varA_b": [0, 1, 0], + "varA_c": [0, 0, 1], + "varB_d": [1, 1, 0], + "varB_a": [0, 0, 1], + }, + index=index_, + ) + pd.testing.assert_frame_equal(dft, df_expected, check_dtype=False) + + def test_encode_categories_in_k_binary_plus_select_vars_automatically(df_enc_big): # test case 1: encode all categories into k binary variables, select variables # automatically From dc620615ed7d652ddde526301302fa4cfd574ff3 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sat, 9 Sep 2023 20:20:21 +0200 Subject: [PATCH 16/26] tidy logic rare label encoder --- feature_engine/encoding/rare_label.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/feature_engine/encoding/rare_label.py b/feature_engine/encoding/rare_label.py index c7afb18d7..72288db8d 100644 --- a/feature_engine/encoding/rare_label.py +++ b/feature_engine/encoding/rare_label.py @@ -165,7 +165,7 @@ def __init__( if not isinstance(replace_with, (str, int, float)): raise ValueError( - "replace_with can should be a string, ingteger or float. " + "replace_with can should be a string, integer or float. " f"Got {replace_with} instead." ) @@ -245,21 +245,16 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # check if dataset contains na if self.missing_values == "raise": _check_optional_contains_na(X, self.variables_) - - for feature in self.variables_: - if X[feature].dtype == "category": - X[feature] = X[feature].cat.add_categories(self.replace_with) - X.loc[ - ~X[feature].isin(self.encoder_dict_[feature]), feature - ] = self.replace_with - + with_nan = [] else: - for feature in self.variables_: - if X[feature].dtype == "category": - X[feature] = X[feature].cat.add_categories(self.replace_with) - X.loc[ - ~X[feature].isin(self.encoder_dict_[feature] + [np.nan]), feature - ] = self.replace_with + with_nan = [np.nan] + + for feature in self.variables_: + if X[feature].dtype == "category": + X[feature] = X[feature].cat.add_categories(self.replace_with) + X.loc[ + ~X[feature].isin(self.encoder_dict_[feature] + with_nan), feature + ] = self.replace_with return X From b608bc6eacdbb6060bf09d50af4bdb7d6a021d34 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sat, 9 Sep 2023 20:30:27 +0200 Subject: [PATCH 17/26] edit rare label encoder user guide --- docs/user_guide/encoding/RareLabelEncoder.rst | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/docs/user_guide/encoding/RareLabelEncoder.rst b/docs/user_guide/encoding/RareLabelEncoder.rst index ce26f3036..3f8df4395 100644 --- a/docs/user_guide/encoding/RareLabelEncoder.rst +++ b/docs/user_guide/encoding/RareLabelEncoder.rst @@ -55,6 +55,7 @@ First, let's load the data and separate it into train and test: predictors_only=True, cabin="letter_only", ) + X["pclass"] = X["pclass"].astype("O") X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.3, random_state=0, @@ -73,19 +74,29 @@ We see the resulting data below: 1193 3 male 29.881135 0 0 7.7250 M Q 686 3 female 22.000000 0 0 7.7250 M Q +Let's explore the number of uniue categories in the variable `"cabin"`. + +.. code:: python + + X_train["cabin"].unique() + +We see the number of unique categories in the output below: + +.. code:: python + + array(['M', 'E', 'C', 'D', 'B', 'A', 'F', 'T', 'G'], dtype=object) + Now, we set up the :class:`RareLabelEncoder()` to group categories shown by less than 3% of the observations into a new group or category called 'Rare'. We will group the categories in the indicated variables if they have more than 2 unique categories each. .. code:: python - # set up the encoder encoder = RareLabelEncoder( tol=0.03, n_categories=2, variables=['cabin', 'pclass', 'embarked'], replace_with='Rare', - ignore_format=True, ) # fit the encoder @@ -116,6 +127,20 @@ Now we can go ahead and transform the variables: train_t = encoder.transform(X_train) test_t = encoder.transform(X_test) +Let's now inspect the number of unique categories in the variable `"cabin"` after the +transformation: + +.. code:: python + + X_train["cabin"].unique() + +In the output below, we see that the infrequent categories have been replaced by +`"Rare"`. + +.. code:: python + + array(['M', 'E', 'C', 'D', 'B', 'Rare'], dtype=object) + We can also specify the maximum number of categories that can be considered frequent using the `max_n_categories` parameter. From 7acf3845441ccc1507cf8d64fdb63dc1a379a64a Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 12 Sep 2023 11:41:50 +0200 Subject: [PATCH 18/26] modify code in user guide --- docs/user_guide/datetime/DatetimeSubtraction.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/user_guide/datetime/DatetimeSubtraction.rst b/docs/user_guide/datetime/DatetimeSubtraction.rst index fa71449ee..dd4785a07 100644 --- a/docs/user_guide/datetime/DatetimeSubtraction.rst +++ b/docs/user_guide/datetime/DatetimeSubtraction.rst @@ -81,8 +81,8 @@ example shows how to use this syntax: .. code:: python - data["diff"] = data["date1"].sub(data["date2"], axis=0).apply( - lambda x: x / np.timedelta64(1, "Y")) + data["diff"] = data["date1"].sub(data["date2"], axis=0).div( + np.timedelta64(1, "Y").astype("timedelta64[ns]")) print(data) @@ -294,6 +294,7 @@ the time difference in microseconds: reference="date2", utc=True, output_unit="ms", + format="mixed" ) new = dfts.fit_transform(data) From 06d69f769536084552332ac008c8743d06da9697 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 12 Sep 2023 11:51:33 +0200 Subject: [PATCH 19/26] fix command in user guide rare label --- docs/user_guide/encoding/RareLabelEncoder.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user_guide/encoding/RareLabelEncoder.rst b/docs/user_guide/encoding/RareLabelEncoder.rst index 3f8df4395..fdc974255 100644 --- a/docs/user_guide/encoding/RareLabelEncoder.rst +++ b/docs/user_guide/encoding/RareLabelEncoder.rst @@ -132,7 +132,7 @@ transformation: .. code:: python - X_train["cabin"].unique() + train_t["cabin"].unique() In the output below, we see that the infrequent categories have been replaced by `"Rare"`. From 65f6376d128e6ec4f215b68ef8d319b5cb4281ee Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 12 Sep 2023 12:18:25 +0200 Subject: [PATCH 20/26] fix output RFE --- .../selection/RecursiveFeatureAddition.rst | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/docs/user_guide/selection/RecursiveFeatureAddition.rst b/docs/user_guide/selection/RecursiveFeatureAddition.rst index e4e5dcca6..700ba3809 100644 --- a/docs/user_guide/selection/RecursiveFeatureAddition.rst +++ b/docs/user_guide/selection/RecursiveFeatureAddition.rst @@ -54,7 +54,7 @@ First, we load the data: # load dataset diabetes_X, diabetes_y = load_diabetes(return_X_y=True) X = pd.DataFrame(diabetes_X) - y = pd.DataFrame(diabetes_y) + y = pd.Series(diabetes_y) Now, we set up :class:`RecursiveFeatureAddition` to select features based on the r2 returned by a Linear Regression model, using 3 fold cross-validation. In this case, @@ -99,16 +99,16 @@ adding each feature. .. code:: python - {4: 0, - 8: 0.2837159006046677, - 2: 0.1377700238871593, - 5: 0.0023329006089969906, - 3: 0.0187608758643259, - 1: 0.0027994385024313617, - 7: 0.0026951300105543807, - 6: 0.002683967832484757, - 9: 0.0003040126429713075, - 0: -0.007386876030245182} + {0: -0.0032800993162502845, + 9: -0.00028194870232089997, + 6: -0.0006751427734088544, + 7: 0.00013890056776355575, + 1: 0.01195652626644067, + 3: 0.02863360798239445, + 5: 0.012639242239088355, + 2: 0.06630359039334816, + 8: 0.10937354113435072, + 4: 0.024318355833473526} :class:`RecursiveFeatureAddition` also stores the features that will be dropped based n the given threshold. @@ -130,10 +130,11 @@ If we now print the transformed data, we see that the features above were remove .. code:: python - 4 8 2 3 - 0 -0.044223 0.019908 0.061696 0.021872 - 1 -0.008449 -0.068330 -0.051474 -0.026328 - 2 -0.045599 0.002864 0.044451 -0.005671 - 3 0.012191 0.022692 -0.011595 -0.036656 - 4 0.003935 -0.031991 -0.036385 0.021872 + 1 2 3 4 5 8 + 0 0.050680 0.061696 0.021872 -0.044223 -0.034821 0.019907 + 1 -0.044642 -0.051474 -0.026328 -0.008449 -0.019163 -0.068332 + 2 0.050680 0.044451 -0.005670 -0.045599 -0.034194 0.002861 + 3 -0.044642 -0.011595 -0.036656 0.012191 0.024991 0.022688 + 4 -0.044642 -0.036385 0.021872 0.003935 0.015596 -0.031988 + From 746026f7077b44218d3f214471a70c3aa79c2c0b Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 12 Sep 2023 12:23:20 +0200 Subject: [PATCH 21/26] fix typo RFA --- .../selection/RecursiveFeatureAddition.rst | 40 +++++++++---------- .../selection/RecursiveFeatureElimination.rst | 15 ++++--- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/docs/user_guide/selection/RecursiveFeatureAddition.rst b/docs/user_guide/selection/RecursiveFeatureAddition.rst index 700ba3809..6903e9e69 100644 --- a/docs/user_guide/selection/RecursiveFeatureAddition.rst +++ b/docs/user_guide/selection/RecursiveFeatureAddition.rst @@ -49,7 +49,7 @@ First, we load the data: import pandas as pd from sklearn.datasets import load_diabetes from sklearn.linear_model import LinearRegression - from feature_engine.selection import RecursiveFeatureElimination + from feature_engine.selection import RecursiveFeatureAddition # load dataset diabetes_X, diabetes_y = load_diabetes(return_X_y=True) @@ -66,7 +66,7 @@ we leave the parameter `threshold` to the default value which is 0.01. linear_model = LinearRegression() # initialize feature selector - tr = RecursiveFeatureElimination(estimator=linear_model, scoring="r2", cv=3) + tr = RecursiveFeatureAddition(estimator=linear_model, scoring="r2", cv=3) With `fit()` the model finds the most useful features, that is, features that when added cause an increase in model performance bigger than 0.01. With `transform()`, the transformer @@ -99,16 +99,17 @@ adding each feature. .. code:: python - {0: -0.0032800993162502845, - 9: -0.00028194870232089997, - 6: -0.0006751427734088544, - 7: 0.00013890056776355575, - 1: 0.01195652626644067, - 3: 0.02863360798239445, - 5: 0.012639242239088355, - 2: 0.06630359039334816, - 8: 0.10937354113435072, - 4: 0.024318355833473526} + {4: 0, + 8: 0.28371458794131676, + 2: 0.1377714799388745, + 5: 0.0023327265047610735, + 3: 0.018759914615172735, + 1: 0.0027996354657459643, + 7: 0.002695149440021638, + 6: 0.002683934134630306, + 9: 0.000304067408860742, + 0: -0.007387230783454768} + :class:`RecursiveFeatureAddition` also stores the features that will be dropped based n the given threshold. @@ -120,7 +121,7 @@ n the given threshold. .. code:: python - [0, 6, 7, 9] + [0, 1, 5, 6, 7, 9] If we now print the transformed data, we see that the features above were removed. @@ -130,11 +131,10 @@ If we now print the transformed data, we see that the features above were remove .. code:: python - 1 2 3 4 5 8 - 0 0.050680 0.061696 0.021872 -0.044223 -0.034821 0.019907 - 1 -0.044642 -0.051474 -0.026328 -0.008449 -0.019163 -0.068332 - 2 0.050680 0.044451 -0.005670 -0.045599 -0.034194 0.002861 - 3 -0.044642 -0.011595 -0.036656 0.012191 0.024991 0.022688 - 4 -0.044642 -0.036385 0.021872 0.003935 0.015596 -0.031988 - + 2 3 4 8 + 0 0.061696 0.021872 -0.044223 0.019907 + 1 -0.051474 -0.026328 -0.008449 -0.068332 + 2 0.044451 -0.005670 -0.045599 0.002861 + 3 -0.011595 -0.036656 0.012191 0.022688 + 4 -0.036385 0.021872 0.003935 -0.031988 diff --git a/docs/user_guide/selection/RecursiveFeatureElimination.rst b/docs/user_guide/selection/RecursiveFeatureElimination.rst index 66d8e00a9..2e2c57f61 100644 --- a/docs/user_guide/selection/RecursiveFeatureElimination.rst +++ b/docs/user_guide/selection/RecursiveFeatureElimination.rst @@ -62,7 +62,7 @@ First, we load the data: # load dataset diabetes_X, diabetes_y = load_diabetes(return_X_y=True) X = pd.DataFrame(diabetes_X) - y = pd.DataFrame(diabetes_y) + y = pd.Series(diabetes_y) Now, we set up :class:`RecursiveFeatureElimination` to select features based on the r2 returned by a Linear Regression model, using 3 fold cross-validation. In this case, @@ -139,13 +139,12 @@ If we now print the transformed data, we see that the features above were remove .. code:: python - 1 3 5 2 8 4 - 0 0.050680 0.021872 -0.034821 0.061696 0.019908 -0.044223 - 1 -0.044642 -0.026328 -0.019163 -0.051474 -0.068330 -0.008449 - 2 0.050680 -0.005671 -0.034194 0.044451 0.002864 -0.045599 - 3 -0.044642 -0.036656 0.024991 -0.011595 0.022692 0.012191 - 4 -0.044642 0.021872 0.015596 -0.036385 -0.031991 0.003935 - + 1 2 3 4 5 8 + 0 0.050680 0.061696 0.021872 -0.044223 -0.034821 0.019907 + 1 -0.044642 -0.051474 -0.026328 -0.008449 -0.019163 -0.068332 + 2 0.050680 0.044451 -0.005670 -0.045599 -0.034194 0.002861 + 3 -0.044642 -0.011595 -0.036656 0.012191 0.024991 0.022688 + 4 -0.044642 -0.036385 0.021872 0.003935 0.015596 -0.031988 More details From 56839a98de5c8b18e92acc5db94f4fd9a777ddd5 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 12 Sep 2023 12:29:12 +0200 Subject: [PATCH 22/26] complete shuffling user guide --- .../selection/SelectByShuffling.rst | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/docs/user_guide/selection/SelectByShuffling.rst b/docs/user_guide/selection/SelectByShuffling.rst index e04315c45..5eb494d2c 100644 --- a/docs/user_guide/selection/SelectByShuffling.rst +++ b/docs/user_guide/selection/SelectByShuffling.rst @@ -38,7 +38,7 @@ First, we load the data: # load dataset diabetes_X, diabetes_y = load_diabetes(return_X_y=True) X = pd.DataFrame(diabetes_X) - y = pd.DataFrame(diabetes_y) + y = pd.Series(diabetes_y) Now, we set up the model for which we want to have the performance drop evaluated: @@ -88,16 +88,16 @@ an idea of where the threshold could be by looking at these values: .. code:: python - {0: -0.02368121940502793, - 1: 0.017909161264480666, - 2: 0.18565460365508413, - 3: 0.07655405817715671, - 4: 0.4327180164470878, - 5: 0.16394693824418372, - 6: -0.012876023845921625, - 7: 0.01048781540981647, - 8: 0.3921465005640224, - 9: -0.01427065640301245} + {0: -0.0035681361984126747, + 1: 0.041170843574652394, + 2: 0.1920054944393057, + 3: 0.07007527443645178, + 4: 0.49871458125373913, + 5: 0.1802858704499694, + 6: 0.025536233845966705, + 7: 0.024058931694668884, + 8: 0.40901959802129045, + 9: 0.004487448637912506} :class:`SelectByShuffling()` also stores the features that will be dropped based on the threshold indicated. @@ -110,3 +110,17 @@ threshold indicated. [0, 1, 3, 6, 7, 9] +If we now print the transformed data, we see that the features above were removed. + +.. code:: python + + print(Xt.head()) + +.. code:: python + + 2 4 5 8 + 0 0.061696 -0.044223 -0.034821 0.019907 + 1 -0.051474 -0.008449 -0.019163 -0.068332 + 2 0.044451 -0.045599 -0.034194 0.002861 + 3 -0.011595 0.012191 0.024991 0.022688 + 4 -0.036385 0.003935 0.015596 -0.031988 From ddb11b2a6727348b27266c276d857d084c058c5d Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 12 Sep 2023 12:31:36 +0200 Subject: [PATCH 23/26] add final df single feature classifier --- .../SelectBySingleFeaturePerformance.rst | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/user_guide/selection/SelectBySingleFeaturePerformance.rst b/docs/user_guide/selection/SelectBySingleFeaturePerformance.rst index fed8c956a..47bfd2764 100644 --- a/docs/user_guide/selection/SelectBySingleFeaturePerformance.rst +++ b/docs/user_guide/selection/SelectBySingleFeaturePerformance.rst @@ -32,7 +32,7 @@ First, we load the data: # load dataset diabetes_X, diabetes_y = load_diabetes(return_X_y=True) X = pd.DataFrame(diabetes_X) - y = pd.DataFrame(diabetes_y) + y = pd.Series(diabetes_y) Now, we start :class:`SelectBySingleFeaturePerformance()` to select features based on the r2 returned by a Linear regression, using 3 fold cross-validation. We want to select features @@ -89,6 +89,28 @@ With `transform()` we go ahead and remove the features from the dataset: # drop variables Xt = sel.transform(X) +If we now print the transformed data, we see that the features above were removed. + +.. code:: python + + print(Xt.head()) + +.. code:: python + + 0 2 3 4 5 6 7 \ + 0 0.038076 0.061696 0.021872 -0.044223 -0.034821 -0.043401 -0.002592 + 1 -0.001882 -0.051474 -0.026328 -0.008449 -0.019163 0.074412 -0.039493 + 2 0.085299 0.044451 -0.005670 -0.045599 -0.034194 -0.032356 -0.002592 + 3 -0.089063 -0.011595 -0.036656 0.012191 0.024991 -0.036038 0.034309 + 4 0.005383 -0.036385 0.021872 0.003935 0.015596 0.008142 -0.002592 + + 8 9 + 0 0.019907 -0.017646 + 1 -0.068332 -0.092204 + 2 0.002861 -0.025930 + 3 0.022688 -0.009362 + 4 -0.031988 -0.046641 + More details ^^^^^^^^^^^^ From 06d3cc32858f95fcd7a6dfbd638b372856377abf Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 12 Sep 2023 12:43:48 +0200 Subject: [PATCH 24/26] remove is_categorical_dtype from variable handling --- .../selection/SmartCorrelatedSelection.rst | 28 +++++++++---------- .../variable_type_selection.py | 5 ++-- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/docs/user_guide/selection/SmartCorrelatedSelection.rst b/docs/user_guide/selection/SmartCorrelatedSelection.rst index 8ac449367..883ebd039 100644 --- a/docs/user_guide/selection/SmartCorrelatedSelection.rst +++ b/docs/user_guide/selection/SmartCorrelatedSelection.rst @@ -125,23 +125,23 @@ have been removed. .. code:: python - print(print(Xt.head())) + print(Xt.head()) .. code:: python - var_1 var_2 var_3 var_5 var_10 var_11 var_8 \ - 0 -2.376400 -0.247208 1.210290 0.091527 2.070526 -1.989335 2.070483 - 1 1.969326 -0.126894 0.034598 -0.186802 1.184820 -1.309524 2.421477 - 2 1.499174 0.334123 -2.233844 -0.313881 -0.066448 -0.852703 2.263546 - 3 0.075341 1.627132 0.943132 -0.468041 0.713558 0.484649 2.792500 - 4 0.372213 0.338141 0.951526 0.729005 0.398790 -0.186530 2.186741 - - var_7 - 0 -2.230170 - 1 -1.447490 - 2 -2.240741 - 3 -3.534861 - 4 -2.053965 + var_1 var_2 var_3 var_5 var_7 var_8 var_10 \ + 0 -2.376400 -0.247208 1.210290 0.091527 -2.230170 2.070483 2.070526 + 1 1.969326 -0.126894 0.034598 -0.186802 -1.447490 2.421477 1.184820 + 2 1.499174 0.334123 -2.233844 -0.313881 -2.240741 2.263546 -0.066448 + 3 0.075341 1.627132 0.943132 -0.468041 -3.534861 2.792500 0.713558 + 4 0.372213 0.338141 0.951526 0.729005 -2.053965 2.186741 0.398790 + + var_11 + 0 -1.989335 + 1 -1.309524 + 2 -0.852703 + 3 0.484649 + 4 -0.186530 More details diff --git a/feature_engine/variable_handling/variable_type_selection.py b/feature_engine/variable_handling/variable_type_selection.py index 8ffef38b3..596ea0344 100644 --- a/feature_engine/variable_handling/variable_type_selection.py +++ b/feature_engine/variable_handling/variable_type_selection.py @@ -3,7 +3,6 @@ from typing import List, Tuple, Union import pandas as pd -from pandas.api.types import is_categorical_dtype as is_categorical from pandas.api.types import is_datetime64_any_dtype as is_datetime from pandas.api.types import is_numeric_dtype as is_numeric from pandas.api.types import is_object_dtype as is_object @@ -142,7 +141,7 @@ def find_or_check_categorical_variables( ) elif isinstance(variables, (str, int)): - if is_categorical(X[variables]) or is_object(X[variables]): + if X[variables].dtype.name == "category" or is_object(X[variables]): variables = [variables] else: raise TypeError("The variable entered is not categorical.") @@ -397,7 +396,7 @@ def find_categorical_and_numerical_variables( # If the user passes just 1 variable outside a list. if isinstance(variables, (str, int)): - if is_categorical(X[variables]) or is_object(X[variables]): + if X[variables].dtype.name == "category" or is_object(X[variables]): variables_cat = [variables] variables_num = [] elif is_numeric(X[variables]): From fca407c8bef38ac10038e80ee4728c1aac6fd4ab Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 12 Sep 2023 12:46:39 +0200 Subject: [PATCH 25/26] fix typo expanding windows --- .../timeseries/forecasting/ExpandingWindowFeatures.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user_guide/timeseries/forecasting/ExpandingWindowFeatures.rst b/docs/user_guide/timeseries/forecasting/ExpandingWindowFeatures.rst index a88d8e6f6..33044f14e 100644 --- a/docs/user_guide/timeseries/forecasting/ExpandingWindowFeatures.rst +++ b/docs/user_guide/timeseries/forecasting/ExpandingWindowFeatures.rst @@ -272,7 +272,7 @@ just need to remember to drop the original series after the transformation: .. code:: python - win_f = WindowFeatures( + win_f = ExpandingWindowFeatures( functions=["mean", "max"], drop_original=True, ) From efb425ddbd84dc0a8aa8b0658def2553f634e793 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 12 Sep 2023 12:53:57 +0200 Subject: [PATCH 26/26] fix typo arcsin --- docs/user_guide/transformation/ArcsinTransformer.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user_guide/transformation/ArcsinTransformer.rst b/docs/user_guide/transformation/ArcsinTransformer.rst index c412eaaa7..7754b5673 100644 --- a/docs/user_guide/transformation/ArcsinTransformer.rst +++ b/docs/user_guide/transformation/ArcsinTransformer.rst @@ -33,7 +33,7 @@ test sets. from sklearn.model_selection import train_test_split from sklearn.datasets import load_breast_cancer - from feature_engine import ArcsinTransformer + from feature_engine.transformation import ArcsinTransformer #Load dataset breast_cancer = load_breast_cancer()