diff --git a/README.md b/README.md index 13130850d..4df8471fb 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,7 @@ [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![GitHub contributors](https://img.shields.io/github/contributors/feature-engine/feature_engine?logo=GitHub)](https://github.com/feature-engine/feature_engine/graphs/contributors) [![Gitter](https://img.shields.io/gitter/room/feature-engine/feaure_engine?logo=Gitter)](https://gitter.im/feature_engine/community) -[![Total Downloads](https://pepy.tech/badge/feature-engine)](https://pepy.tech/project/feature-engine) -[![Monthly Downloads](https://pepy.tech/badge/feature-engine/month)](https://pepy.tech/project/feature-engine) +[![Monthly Downloads](https://img.shields.io/pypi/dm/feature-engine)](https://img.shields.io/pypi/dm/feature-engine) [![DOI](https://zenodo.org/badge/163630824.svg)](https://zenodo.org/badge/latestdoi/163630824) [![JOSS](https://joss.theoj.org/papers/10.21105/joss.03642/status.svg)](https://doi.org/10.21105/joss.03642) [![first-timers-only](https://img.shields.io/badge/first--timers--only-friendly-blue.svg?style=flat)](https://www.firsttimersonly.com/) diff --git a/docs/conf.py b/docs/conf.py index 1d85c16e3..6ae9bc8c4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -195,7 +195,7 @@ "pandas": ("https://pandas.pydata.org/docs/", None), "scipy": ("https://docs.scipy.org/doc/scipy/reference", None), "matplotlib": ("https://matplotlib.org/", None), - "sklearn": ("http://scikit-learn.org/stable", None), + "sklearn": ("https://scikit-learn.org/stable/", None), } # -- Options for LaTeX output --------------------------------------------- @@ -273,9 +273,6 @@ # A list of files that should not be packed into the epub file. epub_exclude_files = ["search.html"] -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {"https://docs.python.org/": None} - # The following is used by sphinx.ext.linkcode to provide links to github linkcode_resolve = make_linkcode_resolve( "feature_engine", diff --git a/docs/user_guide/datetime/DatetimeFeatures.rst b/docs/user_guide/datetime/DatetimeFeatures.rst index 77572fa79..b5157a16b 100644 --- a/docs/user_guide/datetime/DatetimeFeatures.rst +++ b/docs/user_guide/datetime/DatetimeFeatures.rst @@ -77,7 +77,7 @@ First, we will create a toy dataframe with 2 date variables: toy_df = pd.DataFrame({ "var_date1": ['May-1989', 'Dec-2020', 'Jan-1999', 'Feb-2002'], - "var_date2": ['06/21/12', '02/10/98', '08/03/10', '10/31/20'], + "var_date2": ['06/21/2012', '02/10/1998', '08/03/2010', '10/31/2020'], }) Now, we will extract the variables month, month-end and the day of the year from the @@ -148,9 +148,14 @@ First, let's create a toy dataset with 2 time variables and an object variable. datetime. So if we want to extract time features from all our datetime variables, we don't need to specify them. +Note that from version 2.0.0 pandas deprecated the parameter `infer_datetime_format`. +Hence, if you want pandas to infer the datetime format and you have different formats, +you need to explicitly say so by passing `"mixed"` to the `format` parameter as shown +below. + .. code:: python - dfts = DatetimeFeatures(features_to_extract=["minute"]) + dfts = DatetimeFeatures(features_to_extract=["minute"], format="mixed") df_transf = dfts.fit_transform(toy_df) @@ -227,10 +232,11 @@ the features. variables=["var_dt1", "var_dt3"], features_to_extract=["year", "hour"], drop_original=False, + format="mixed", ) df_transf = dfts.fit_transform(toy_df) - print(df_transf) + df_transf We can see the resulting dataframe in the following output: @@ -373,10 +379,11 @@ And now we mistakenly extract only date features: dfts = DatetimeFeatures( features_to_extract=["year", "month", "day_of_week"], + format="mixed", ) df_transf = dfts.fit_transform(toy_df) - print(df_transf) + df_transf .. code:: python @@ -413,6 +420,7 @@ And we mistakenly extract the hour and the minute: dfts = DatetimeFeatures( features_to_extract=["hour", "minute"], + format="mixed", ) df_transf = dfts.fit_transform(toy_df) @@ -466,7 +474,7 @@ To do this, we leave the parameter `features_to_extract` to `None`. df_transf = dfts.fit_transform(toy_df) - print(df_transf) + df_transf .. code:: python @@ -614,7 +622,7 @@ from the dataset. .. code:: python pipe = Pipeline([ - ('datetime', DatetimeFeatures()), + ('datetime', DatetimeFeatures(format="mixed")), ('drop_constant', DropConstantFeatures()), ]) @@ -683,12 +691,13 @@ converts all data to UTC timezone. dfts = DatetimeFeatures( features_to_extract=["hour", "minute"], drop_original=False, - utc=True + utc=True, + format="mixed", ) df_transf = dfts.fit_transform(toy_df) - print(df_transf) + df_transf .. code:: python @@ -709,7 +718,7 @@ the datetime information extracted as if it were in UTC timezone. from feature_engine.datetime import DatetimeFeatures var_tz = pd.Series(['08/31/00 12:34:45', '12/01/90 23:01:02', '04/25/01 11:59:21']) - var_tz = pd.to_datetime(var_tz) + var_tz = pd.to_datetime(var_tz, format="mixed") var_tz = var_tz.dt.tz_localize("US/eastern") var_tz @@ -735,7 +744,7 @@ timezone. df_transf = dfts.fit_transform(toy_df) - print(df_transf) + df_transf .. code:: python diff --git a/docs/user_guide/datetime/DatetimeSubtraction.rst b/docs/user_guide/datetime/DatetimeSubtraction.rst index 50f65322c..dd4785a07 100644 --- a/docs/user_guide/datetime/DatetimeSubtraction.rst +++ b/docs/user_guide/datetime/DatetimeSubtraction.rst @@ -81,8 +81,8 @@ example shows how to use this syntax: .. code:: python - data["diff"] = data["date1"].sub(data["date2"], axis=0).apply( - lambda x: x / np.timedelta64(1, "Y")) + data["diff"] = data["date1"].sub(data["date2"], axis=0).div( + np.timedelta64(1, "Y").astype("timedelta64[ns]")) print(data) @@ -235,6 +235,7 @@ parameter `missing_values` to `"ignore"`. Here is a code example: .. code:: python + import numpy as np import pandas as pd from feature_engine.datetime import DatetimeSubtraction @@ -293,6 +294,7 @@ the time difference in microseconds: reference="date2", utc=True, output_unit="ms", + format="mixed" ) new = dfts.fit_transform(data) diff --git a/docs/user_guide/encoding/RareLabelEncoder.rst b/docs/user_guide/encoding/RareLabelEncoder.rst index ce26f3036..fdc974255 100644 --- a/docs/user_guide/encoding/RareLabelEncoder.rst +++ b/docs/user_guide/encoding/RareLabelEncoder.rst @@ -55,6 +55,7 @@ First, let's load the data and separate it into train and test: predictors_only=True, cabin="letter_only", ) + X["pclass"] = X["pclass"].astype("O") X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.3, random_state=0, @@ -73,19 +74,29 @@ We see the resulting data below: 1193 3 male 29.881135 0 0 7.7250 M Q 686 3 female 22.000000 0 0 7.7250 M Q +Let's explore the number of uniue categories in the variable `"cabin"`. + +.. code:: python + + X_train["cabin"].unique() + +We see the number of unique categories in the output below: + +.. code:: python + + array(['M', 'E', 'C', 'D', 'B', 'A', 'F', 'T', 'G'], dtype=object) + Now, we set up the :class:`RareLabelEncoder()` to group categories shown by less than 3% of the observations into a new group or category called 'Rare'. We will group the categories in the indicated variables if they have more than 2 unique categories each. .. code:: python - # set up the encoder encoder = RareLabelEncoder( tol=0.03, n_categories=2, variables=['cabin', 'pclass', 'embarked'], replace_with='Rare', - ignore_format=True, ) # fit the encoder @@ -116,6 +127,20 @@ Now we can go ahead and transform the variables: train_t = encoder.transform(X_train) test_t = encoder.transform(X_test) +Let's now inspect the number of unique categories in the variable `"cabin"` after the +transformation: + +.. code:: python + + train_t["cabin"].unique() + +In the output below, we see that the infrequent categories have been replaced by +`"Rare"`. + +.. code:: python + + array(['M', 'E', 'C', 'D', 'B', 'Rare'], dtype=object) + We can also specify the maximum number of categories that can be considered frequent using the `max_n_categories` parameter. diff --git a/docs/user_guide/selection/RecursiveFeatureAddition.rst b/docs/user_guide/selection/RecursiveFeatureAddition.rst index e4e5dcca6..6903e9e69 100644 --- a/docs/user_guide/selection/RecursiveFeatureAddition.rst +++ b/docs/user_guide/selection/RecursiveFeatureAddition.rst @@ -49,12 +49,12 @@ First, we load the data: import pandas as pd from sklearn.datasets import load_diabetes from sklearn.linear_model import LinearRegression - from feature_engine.selection import RecursiveFeatureElimination + from feature_engine.selection import RecursiveFeatureAddition # load dataset diabetes_X, diabetes_y = load_diabetes(return_X_y=True) X = pd.DataFrame(diabetes_X) - y = pd.DataFrame(diabetes_y) + y = pd.Series(diabetes_y) Now, we set up :class:`RecursiveFeatureAddition` to select features based on the r2 returned by a Linear Regression model, using 3 fold cross-validation. In this case, @@ -66,7 +66,7 @@ we leave the parameter `threshold` to the default value which is 0.01. linear_model = LinearRegression() # initialize feature selector - tr = RecursiveFeatureElimination(estimator=linear_model, scoring="r2", cv=3) + tr = RecursiveFeatureAddition(estimator=linear_model, scoring="r2", cv=3) With `fit()` the model finds the most useful features, that is, features that when added cause an increase in model performance bigger than 0.01. With `transform()`, the transformer @@ -100,15 +100,16 @@ adding each feature. .. code:: python {4: 0, - 8: 0.2837159006046677, - 2: 0.1377700238871593, - 5: 0.0023329006089969906, - 3: 0.0187608758643259, - 1: 0.0027994385024313617, - 7: 0.0026951300105543807, - 6: 0.002683967832484757, - 9: 0.0003040126429713075, - 0: -0.007386876030245182} + 8: 0.28371458794131676, + 2: 0.1377714799388745, + 5: 0.0023327265047610735, + 3: 0.018759914615172735, + 1: 0.0027996354657459643, + 7: 0.002695149440021638, + 6: 0.002683934134630306, + 9: 0.000304067408860742, + 0: -0.007387230783454768} + :class:`RecursiveFeatureAddition` also stores the features that will be dropped based n the given threshold. @@ -120,7 +121,7 @@ n the given threshold. .. code:: python - [0, 6, 7, 9] + [0, 1, 5, 6, 7, 9] If we now print the transformed data, we see that the features above were removed. @@ -130,10 +131,10 @@ If we now print the transformed data, we see that the features above were remove .. code:: python - 4 8 2 3 - 0 -0.044223 0.019908 0.061696 0.021872 - 1 -0.008449 -0.068330 -0.051474 -0.026328 - 2 -0.045599 0.002864 0.044451 -0.005671 - 3 0.012191 0.022692 -0.011595 -0.036656 - 4 0.003935 -0.031991 -0.036385 0.021872 + 2 3 4 8 + 0 0.061696 0.021872 -0.044223 0.019907 + 1 -0.051474 -0.026328 -0.008449 -0.068332 + 2 0.044451 -0.005670 -0.045599 0.002861 + 3 -0.011595 -0.036656 0.012191 0.022688 + 4 -0.036385 0.021872 0.003935 -0.031988 diff --git a/docs/user_guide/selection/RecursiveFeatureElimination.rst b/docs/user_guide/selection/RecursiveFeatureElimination.rst index 66d8e00a9..2e2c57f61 100644 --- a/docs/user_guide/selection/RecursiveFeatureElimination.rst +++ b/docs/user_guide/selection/RecursiveFeatureElimination.rst @@ -62,7 +62,7 @@ First, we load the data: # load dataset diabetes_X, diabetes_y = load_diabetes(return_X_y=True) X = pd.DataFrame(diabetes_X) - y = pd.DataFrame(diabetes_y) + y = pd.Series(diabetes_y) Now, we set up :class:`RecursiveFeatureElimination` to select features based on the r2 returned by a Linear Regression model, using 3 fold cross-validation. In this case, @@ -139,13 +139,12 @@ If we now print the transformed data, we see that the features above were remove .. code:: python - 1 3 5 2 8 4 - 0 0.050680 0.021872 -0.034821 0.061696 0.019908 -0.044223 - 1 -0.044642 -0.026328 -0.019163 -0.051474 -0.068330 -0.008449 - 2 0.050680 -0.005671 -0.034194 0.044451 0.002864 -0.045599 - 3 -0.044642 -0.036656 0.024991 -0.011595 0.022692 0.012191 - 4 -0.044642 0.021872 0.015596 -0.036385 -0.031991 0.003935 - + 1 2 3 4 5 8 + 0 0.050680 0.061696 0.021872 -0.044223 -0.034821 0.019907 + 1 -0.044642 -0.051474 -0.026328 -0.008449 -0.019163 -0.068332 + 2 0.050680 0.044451 -0.005670 -0.045599 -0.034194 0.002861 + 3 -0.044642 -0.011595 -0.036656 0.012191 0.024991 0.022688 + 4 -0.044642 -0.036385 0.021872 0.003935 0.015596 -0.031988 More details diff --git a/docs/user_guide/selection/SelectByShuffling.rst b/docs/user_guide/selection/SelectByShuffling.rst index e04315c45..5eb494d2c 100644 --- a/docs/user_guide/selection/SelectByShuffling.rst +++ b/docs/user_guide/selection/SelectByShuffling.rst @@ -38,7 +38,7 @@ First, we load the data: # load dataset diabetes_X, diabetes_y = load_diabetes(return_X_y=True) X = pd.DataFrame(diabetes_X) - y = pd.DataFrame(diabetes_y) + y = pd.Series(diabetes_y) Now, we set up the model for which we want to have the performance drop evaluated: @@ -88,16 +88,16 @@ an idea of where the threshold could be by looking at these values: .. code:: python - {0: -0.02368121940502793, - 1: 0.017909161264480666, - 2: 0.18565460365508413, - 3: 0.07655405817715671, - 4: 0.4327180164470878, - 5: 0.16394693824418372, - 6: -0.012876023845921625, - 7: 0.01048781540981647, - 8: 0.3921465005640224, - 9: -0.01427065640301245} + {0: -0.0035681361984126747, + 1: 0.041170843574652394, + 2: 0.1920054944393057, + 3: 0.07007527443645178, + 4: 0.49871458125373913, + 5: 0.1802858704499694, + 6: 0.025536233845966705, + 7: 0.024058931694668884, + 8: 0.40901959802129045, + 9: 0.004487448637912506} :class:`SelectByShuffling()` also stores the features that will be dropped based on the threshold indicated. @@ -110,3 +110,17 @@ threshold indicated. [0, 1, 3, 6, 7, 9] +If we now print the transformed data, we see that the features above were removed. + +.. code:: python + + print(Xt.head()) + +.. code:: python + + 2 4 5 8 + 0 0.061696 -0.044223 -0.034821 0.019907 + 1 -0.051474 -0.008449 -0.019163 -0.068332 + 2 0.044451 -0.045599 -0.034194 0.002861 + 3 -0.011595 0.012191 0.024991 0.022688 + 4 -0.036385 0.003935 0.015596 -0.031988 diff --git a/docs/user_guide/selection/SelectBySingleFeaturePerformance.rst b/docs/user_guide/selection/SelectBySingleFeaturePerformance.rst index fed8c956a..47bfd2764 100644 --- a/docs/user_guide/selection/SelectBySingleFeaturePerformance.rst +++ b/docs/user_guide/selection/SelectBySingleFeaturePerformance.rst @@ -32,7 +32,7 @@ First, we load the data: # load dataset diabetes_X, diabetes_y = load_diabetes(return_X_y=True) X = pd.DataFrame(diabetes_X) - y = pd.DataFrame(diabetes_y) + y = pd.Series(diabetes_y) Now, we start :class:`SelectBySingleFeaturePerformance()` to select features based on the r2 returned by a Linear regression, using 3 fold cross-validation. We want to select features @@ -89,6 +89,28 @@ With `transform()` we go ahead and remove the features from the dataset: # drop variables Xt = sel.transform(X) +If we now print the transformed data, we see that the features above were removed. + +.. code:: python + + print(Xt.head()) + +.. code:: python + + 0 2 3 4 5 6 7 \ + 0 0.038076 0.061696 0.021872 -0.044223 -0.034821 -0.043401 -0.002592 + 1 -0.001882 -0.051474 -0.026328 -0.008449 -0.019163 0.074412 -0.039493 + 2 0.085299 0.044451 -0.005670 -0.045599 -0.034194 -0.032356 -0.002592 + 3 -0.089063 -0.011595 -0.036656 0.012191 0.024991 -0.036038 0.034309 + 4 0.005383 -0.036385 0.021872 0.003935 0.015596 0.008142 -0.002592 + + 8 9 + 0 0.019907 -0.017646 + 1 -0.068332 -0.092204 + 2 0.002861 -0.025930 + 3 0.022688 -0.009362 + 4 -0.031988 -0.046641 + More details ^^^^^^^^^^^^ diff --git a/docs/user_guide/selection/SmartCorrelatedSelection.rst b/docs/user_guide/selection/SmartCorrelatedSelection.rst index 8ac449367..883ebd039 100644 --- a/docs/user_guide/selection/SmartCorrelatedSelection.rst +++ b/docs/user_guide/selection/SmartCorrelatedSelection.rst @@ -125,23 +125,23 @@ have been removed. .. code:: python - print(print(Xt.head())) + print(Xt.head()) .. code:: python - var_1 var_2 var_3 var_5 var_10 var_11 var_8 \ - 0 -2.376400 -0.247208 1.210290 0.091527 2.070526 -1.989335 2.070483 - 1 1.969326 -0.126894 0.034598 -0.186802 1.184820 -1.309524 2.421477 - 2 1.499174 0.334123 -2.233844 -0.313881 -0.066448 -0.852703 2.263546 - 3 0.075341 1.627132 0.943132 -0.468041 0.713558 0.484649 2.792500 - 4 0.372213 0.338141 0.951526 0.729005 0.398790 -0.186530 2.186741 - - var_7 - 0 -2.230170 - 1 -1.447490 - 2 -2.240741 - 3 -3.534861 - 4 -2.053965 + var_1 var_2 var_3 var_5 var_7 var_8 var_10 \ + 0 -2.376400 -0.247208 1.210290 0.091527 -2.230170 2.070483 2.070526 + 1 1.969326 -0.126894 0.034598 -0.186802 -1.447490 2.421477 1.184820 + 2 1.499174 0.334123 -2.233844 -0.313881 -2.240741 2.263546 -0.066448 + 3 0.075341 1.627132 0.943132 -0.468041 -3.534861 2.792500 0.713558 + 4 0.372213 0.338141 0.951526 0.729005 -2.053965 2.186741 0.398790 + + var_11 + 0 -1.989335 + 1 -1.309524 + 2 -0.852703 + 3 0.484649 + 4 -0.186530 More details diff --git a/docs/user_guide/timeseries/forecasting/ExpandingWindowFeatures.rst b/docs/user_guide/timeseries/forecasting/ExpandingWindowFeatures.rst index a88d8e6f6..33044f14e 100644 --- a/docs/user_guide/timeseries/forecasting/ExpandingWindowFeatures.rst +++ b/docs/user_guide/timeseries/forecasting/ExpandingWindowFeatures.rst @@ -272,7 +272,7 @@ just need to remember to drop the original series after the transformation: .. code:: python - win_f = WindowFeatures( + win_f = ExpandingWindowFeatures( functions=["mean", "max"], drop_original=True, ) diff --git a/docs/user_guide/transformation/ArcsinTransformer.rst b/docs/user_guide/transformation/ArcsinTransformer.rst index c412eaaa7..7754b5673 100644 --- a/docs/user_guide/transformation/ArcsinTransformer.rst +++ b/docs/user_guide/transformation/ArcsinTransformer.rst @@ -33,7 +33,7 @@ test sets. from sklearn.model_selection import train_test_split from sklearn.datasets import load_breast_cancer - from feature_engine import ArcsinTransformer + from feature_engine.transformation import ArcsinTransformer #Load dataset breast_cancer = load_breast_cancer() diff --git a/docs/whats_new/v_160.rst b/docs/whats_new/v_160.rst index 27f2fa92f..b7af7f9c9 100644 --- a/docs/whats_new/v_160.rst +++ b/docs/whats_new/v_160.rst @@ -1,6 +1,43 @@ Version 1.6.X ============= +Version 1.6.2 +------------- + +Deployed: xx September 2023 + +Contributors +~~~~~~~~~~~~ + +- `Soledad Galli `_ + +New functionality +~~~~~~~~~~~~~~~~~ + +- `DatetimeFeatures()` and `DatetimeSubtraction()` can now specify the format of the datetime variables (`Soledad Galli `_) + +Bug fixes +~~~~~~~~~ + +This bugs were introduced by the latest releases of pandas, Scikit-learn and Scipy. + +- Fix failing test for `YeoJohnsonTransformer()` (`Soledad Galli `_) +- Fix failing test for `RareLabelEncoder()` (`Soledad Galli `_) +- Fix failing test for `DatetimeFeatures()` (`Soledad Galli `_) +- Fix failing test for many encoders: removed `downcast=infer` as it will be deprecated (`Soledad Galli `_) +- Fix version related failing style checks (`Soledad Galli `_) +- Fix version related failing type checks (`Soledad Galli `_) +- Fix version related failing doc checks (`Soledad Galli `_) +- Fix future warning categorical imputation (`Soledad Galli `_) + + +Code improvements +~~~~~~~~~~~~~~~~~ + +- Routine in `DatetimeFeatures()` does not enter into our check for `utc=True` when working with different timezones any more (`Soledad Galli `_) +- Improve performance in `OneHotEncoder()` (`Soledad Galli `_) + + Version 1.6.1 ------------- diff --git a/feature_engine/creation/math_features.py b/feature_engine/creation/math_features.py index 0b3acce8b..35cbe73aa 100644 --- a/feature_engine/creation/math_features.py +++ b/feature_engine/creation/math_features.py @@ -228,7 +228,7 @@ def _get_new_features_name(self) -> List: if isinstance(self.func, list): functions = [ - fun if type(fun) == str else fun.__name__ for fun in self.func + fun if type(fun) is str else fun.__name__ for fun in self.func ] feature_names = [ f"{function}_{'_'.join(varlist)}" for function in functions diff --git a/feature_engine/datetime/datetime.py b/feature_engine/datetime/datetime.py index 246d281a2..a9eb64499 100644 --- a/feature_engine/datetime/datetime.py +++ b/feature_engine/datetime/datetime.py @@ -125,6 +125,14 @@ class DatetimeFeatures(BaseEstimator, TransformerMixin, GetFeatureNamesOutMixin) Return UTC DatetimeIndex if True (converting any tz-aware datetime.datetime objects as well). Same as in `pandas.to_datetime`. + format: str, default None + The strftime to parse time, e.g. "%d/%m/%Y". Check pandas `to_datetime()` for + more information on choices. If you have variables with different formats pass + “mixed”, to infer the format for each element individually. This is risky, + and you should probably use it along with dayfirst, according to pandas' + documentation. + + Attributes ---------- variables_: @@ -177,6 +185,7 @@ def __init__( dayfirst: bool = False, yearfirst: bool = False, utc: Union[None, bool] = None, + format: Union[None, str] = None, ) -> None: if features_to_extract: @@ -217,6 +226,7 @@ def __init__( self.yearfirst = yearfirst self.utc = utc self.features_to_extract = features_to_extract + self.format = format def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ @@ -316,6 +326,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: dayfirst=self.dayfirst, yearfirst=self.yearfirst, utc=self.utc, + format=self.format, ), index=X.index, ) @@ -337,22 +348,13 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: dayfirst=self.dayfirst, yearfirst=self.yearfirst, utc=self.utc, + format=self.format, ) for variable in self.variables_ ], axis=1, ) - non_dt_columns = datetime_df.columns[ - ~datetime_df.apply(is_datetime) - ].tolist() - if non_dt_columns: - raise ValueError( - "ValueError: variable(s) " - + (len(non_dt_columns) * "{} ").format(*non_dt_columns) - + "could not be converted to datetime. Try setting utc=True" - ) - # create new features for var in self.variables_: for feat in self.features_to_extract_: diff --git a/feature_engine/datetime/datetime_subtraction.py b/feature_engine/datetime/datetime_subtraction.py index 8fdb83b2e..90b9db7d8 100644 --- a/feature_engine/datetime/datetime_subtraction.py +++ b/feature_engine/datetime/datetime_subtraction.py @@ -112,6 +112,13 @@ class DatetimeSubtraction(BaseCreation): Return UTC DatetimeIndex if True (converting any tz-aware datetime.datetime objects as well). Same as in `pandas.to_datetime`. + format: str, default None + The strftime to parse time, e.g. "%d/%m/%Y". Check pandas `to_datetime()` for + more information on choices. If you have variables with different formats pass + “mixed”, to infer the format for each element individually. This is risky, + and you should probably use it along with dayfirst, according to pandas' + documentation. + Attributes ---------- variables_: @@ -153,6 +160,7 @@ def __init__( dayfirst: bool = False, yearfirst: bool = False, utc: Union[None, bool] = None, + format: Union[None, str] = None, ) -> None: valid_output_units = { @@ -197,6 +205,7 @@ def __init__( self.dayfirst = dayfirst self.yearfirst = yearfirst self.utc = utc + self.format = format def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ @@ -294,6 +303,7 @@ def _to_datetime(self, X: pd.DataFrame): dayfirst=self.dayfirst, yearfirst=self.yearfirst, utc=self.utc, + format=self.format, ) for variable in set(self.variables_ + self.reference_) ], diff --git a/feature_engine/encoding/base_encoder.py b/feature_engine/encoding/base_encoder.py index c642cc4c1..3fb773b26 100644 --- a/feature_engine/encoding/base_encoder.py +++ b/feature_engine/encoding/base_encoder.py @@ -220,16 +220,14 @@ def _encode(self, X: pd.DataFrame) -> pd.DataFrame: # if original variables are cast as categorical, they will remain # categorical after the encoding, and this is probably not desired - if pd.api.types.is_categorical_dtype(X[feature]): + if X[feature].dtype.name == "category": if all(isinstance(x, int) for x in X[feature]): X[feature] = X[feature].astype("int") else: X[feature] = X[feature].astype("float") if self.unseen == "encode": - X[self.variables_] = X[self.variables_].fillna( - self._unseen, downcast="infer" - ) + X[self.variables_] = X[self.variables_].fillna(self._unseen) else: # check if nan values were introduced by the transformation self._check_nan_values_after_transformation(X) diff --git a/feature_engine/encoding/one_hot.py b/feature_engine/encoding/one_hot.py index 68a219790..de62e44c9 100644 --- a/feature_engine/encoding/one_hot.py +++ b/feature_engine/encoding/one_hot.py @@ -274,7 +274,11 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: for feature in self.variables_: for category in self.encoder_dict_[feature]: - X[f"{feature}_{category}"] = np.where(X[feature] == category, 1, 0) + dummy_df = pd.DataFrame( + {f"{feature}_{category}": np.where(X[feature] == category, 1, 0)}, + index=X.index, + ) + X = pd.concat([X, dummy_df], axis=1) # drop the original non-encoded variables. X.drop(labels=self.variables_, axis=1, inplace=True) diff --git a/feature_engine/encoding/rare_label.py b/feature_engine/encoding/rare_label.py index c7afb18d7..72288db8d 100644 --- a/feature_engine/encoding/rare_label.py +++ b/feature_engine/encoding/rare_label.py @@ -165,7 +165,7 @@ def __init__( if not isinstance(replace_with, (str, int, float)): raise ValueError( - "replace_with can should be a string, ingteger or float. " + "replace_with can should be a string, integer or float. " f"Got {replace_with} instead." ) @@ -245,21 +245,16 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # check if dataset contains na if self.missing_values == "raise": _check_optional_contains_na(X, self.variables_) - - for feature in self.variables_: - if X[feature].dtype == "category": - X[feature] = X[feature].cat.add_categories(self.replace_with) - X.loc[ - ~X[feature].isin(self.encoder_dict_[feature]), feature - ] = self.replace_with - + with_nan = [] else: - for feature in self.variables_: - if X[feature].dtype == "category": - X[feature] = X[feature].cat.add_categories(self.replace_with) - X.loc[ - ~X[feature].isin(self.encoder_dict_[feature] + [np.nan]), feature - ] = self.replace_with + with_nan = [np.nan] + + for feature in self.variables_: + if X[feature].dtype == "category": + X[feature] = X[feature].cat.add_categories(self.replace_with) + X.loc[ + ~X[feature].isin(self.encoder_dict_[feature] + with_nan), feature + ] = self.replace_with return X diff --git a/feature_engine/imputation/categorical.py b/feature_engine/imputation/categorical.py index 096dfa22e..45a723a86 100644 --- a/feature_engine/imputation/categorical.py +++ b/feature_engine/imputation/categorical.py @@ -229,7 +229,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # category, before filling in the nan add_cats = {} for variable in self.variables_: - if pd.api.types.is_categorical_dtype(X[variable]): + if X[variable].dtype.name == "category": add_cats.update( { variable: X[variable].cat.add_categories( diff --git a/feature_engine/selection/shuffle_features.py b/feature_engine/selection/shuffle_features.py index 62805bf3a..cb93917b9 100644 --- a/feature_engine/selection/shuffle_features.py +++ b/feature_engine/selection/shuffle_features.py @@ -1,4 +1,4 @@ -from typing import List, Union +from typing import List, Union, MutableSequence import numpy as np import pandas as pd @@ -189,7 +189,7 @@ def fit( self, X: pd.DataFrame, y: pd.Series, - sample_weight: Union[np.array, pd.Series, List] = None, + sample_weight: Union[MutableSequence, None] = None, ): """ Find the important features. diff --git a/feature_engine/transformation/yeojohnson.py b/feature_engine/transformation/yeojohnson.py index 7f03b468e..243840244 100644 --- a/feature_engine/transformation/yeojohnson.py +++ b/feature_engine/transformation/yeojohnson.py @@ -17,6 +17,7 @@ ) from feature_engine._docstrings.methods import _fit_transform_docstring from feature_engine._docstrings.substitute import Substitution +from feature_engine.tags import _return_tags from feature_engine.variable_handling._init_parameter_checks import ( _check_init_parameter_variables, ) @@ -152,3 +153,18 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: X[feature] = stats.yeojohnson(X[feature], lmbda=self.lambda_dict_[feature]) return X + + def _more_tags(self): + tags_dict = _return_tags() + tags_dict["variables"] = "numerical" + + # ======= this tests fail because the transformers throw an error + # when the values are 0. Nothing to do with the test itself but + # mostly with the data created and used in the test + msg = ( + "Transformer raises error when it can't find the optimal lambda for " + "the transformation, thus this check fails." + ) + tags_dict["_xfail_checks"]["check_fit2d_1sample"] = msg + + return tags_dict diff --git a/feature_engine/variable_handling/variable_type_selection.py b/feature_engine/variable_handling/variable_type_selection.py index 8ffef38b3..596ea0344 100644 --- a/feature_engine/variable_handling/variable_type_selection.py +++ b/feature_engine/variable_handling/variable_type_selection.py @@ -3,7 +3,6 @@ from typing import List, Tuple, Union import pandas as pd -from pandas.api.types import is_categorical_dtype as is_categorical from pandas.api.types import is_datetime64_any_dtype as is_datetime from pandas.api.types import is_numeric_dtype as is_numeric from pandas.api.types import is_object_dtype as is_object @@ -142,7 +141,7 @@ def find_or_check_categorical_variables( ) elif isinstance(variables, (str, int)): - if is_categorical(X[variables]) or is_object(X[variables]): + if X[variables].dtype.name == "category" or is_object(X[variables]): variables = [variables] else: raise TypeError("The variable entered is not categorical.") @@ -397,7 +396,7 @@ def find_categorical_and_numerical_variables( # If the user passes just 1 variable outside a list. if isinstance(variables, (str, int)): - if is_categorical(X[variables]) or is_object(X[variables]): + if X[variables].dtype.name == "category" or is_object(X[variables]): variables_cat = [variables] variables_num = [] elif is_numeric(X[variables]): diff --git a/tests/test_datetime/test_datetime_features.py b/tests/test_datetime/test_datetime_features.py index 1727f27b6..de8d99032 100644 --- a/tests/test_datetime/test_datetime_features.py +++ b/tests/test_datetime/test_datetime_features.py @@ -305,7 +305,47 @@ def test_extract_features_from_categorical_variable( ) -def test_extract_features_from_different_timezones( +def test_extract_features_from_different_timezones(): + df = pd.DataFrame() + df["time"] = pd.concat( + [ + pd.Series( + pd.date_range( + start="2014-08-01 09:00", freq="H", periods=3, tz="Europe/Berlin" + ) + ), + pd.Series( + pd.date_range( + start="2014-08-01 09:00", freq="H", periods=3, tz="US/Central" + ) + ), + ], + axis=0, + ) + df.reset_index(inplace=True, drop=True) + + transformer = DatetimeFeatures( + variables="time", features_to_extract=["hour"], utc=True + ) + X = transformer.fit_transform(df) + + pd.testing.assert_frame_equal( + X, + pd.DataFrame({"time_hour": [7, 8, 9, 14, 15, 16]}), + check_dtype=False, + ) + exp_err_msg = ( + "Tz-aware datetime.datetime cannot be converted to datetime64 " + "unless utc=True, at position 3" + ) + with pytest.raises(ValueError) as errinfo: + assert DatetimeFeatures( + variables="time", features_to_extract=["hour"], utc=False + ).fit_transform(df) + assert str(errinfo.value) == exp_err_msg + + +def test_extract_features_from_different_timezones_when_string( df_datetime, df_datetime_transformed ): time_zones = [4, -1, 9, -7] @@ -313,7 +353,7 @@ def test_extract_features_from_different_timezones( {"time_obj": df_datetime["time_obj"].add(["+4", "-1", "+9", "-7"])} ) transformer = DatetimeFeatures( - variables="time_obj", features_to_extract=["hour"], utc=True + variables="time_obj", features_to_extract=["hour"], utc=True, format="mixed", ) X = transformer.fit_transform(tz_df) @@ -324,15 +364,6 @@ def test_extract_features_from_different_timezones( ), check_dtype=False, ) - exp_err_msg = ( - "ValueError: variable(s) time_obj " - "could not be converted to datetime. Try setting utc=True" - ) - with pytest.raises(ValueError) as errinfo: - assert DatetimeFeatures( - variables="time_obj", features_to_extract=["hour"], utc=False - ).fit_transform(tz_df) - assert str(errinfo.value) == exp_err_msg def test_extract_features_from_localized_tz_variables(): diff --git a/tests/test_encoding/test_count_frequency_encoder.py b/tests/test_encoding/test_count_frequency_encoder.py index f961a2960..f98f8ccf5 100644 --- a/tests/test_encoding/test_count_frequency_encoder.py +++ b/tests/test_encoding/test_count_frequency_encoder.py @@ -279,7 +279,7 @@ def test_zero_encoding_for_new_categories(): # check that the counts are correct for both new and old expected_result = pd.DataFrame({"col1": [3, 0, 1, 3, 1], "col2": [2, 2, 1, 2, 0]}) - pd.testing.assert_frame_equal(result, expected_result) + pd.testing.assert_frame_equal(result, expected_result, check_dtype=False) def test_zero_encoding_for_unseen_categories_if_unseen_is_encode(): @@ -299,7 +299,7 @@ def test_zero_encoding_for_unseen_categories_if_unseen_is_encode(): # check that the counts are correct expected_result = pd.DataFrame({"col1": [3, 0, 1, 3, 1], "col2": [2, 2, 1, 2, 0]}) - pd.testing.assert_frame_equal(result, expected_result) + pd.testing.assert_frame_equal(result, expected_result, check_dtype=False) # with frequency encoder = CountFrequencyEncoder(encoding_method="frequency", unseen="encode").fit( diff --git a/tests/test_encoding/test_onehot_encoder.py b/tests/test_encoding/test_onehot_encoder.py index 42448be12..aca3448be 100644 --- a/tests/test_encoding/test_onehot_encoder.py +++ b/tests/test_encoding/test_onehot_encoder.py @@ -5,6 +5,24 @@ from feature_engine.encoding import OneHotEncoder +@pytest.mark.parametrize("index_", [[1, 2, 3], [3, 2, 1], [4, 9, 2]]) +def test_concat_with_non_ordered_index(index_): + df = pd.DataFrame({"varA": ["a", "b", "c"], "varB": ["d", "d", "a"]}, index=index_) + encoder = OneHotEncoder() + dft = encoder.fit_transform(df) + df_expected = pd.DataFrame( + { + "varA_a": [1, 0, 0], + "varA_b": [0, 1, 0], + "varA_c": [0, 0, 1], + "varB_d": [1, 1, 0], + "varB_a": [0, 0, 1], + }, + index=index_, + ) + pd.testing.assert_frame_equal(dft, df_expected, check_dtype=False) + + def test_encode_categories_in_k_binary_plus_select_vars_automatically(df_enc_big): # test case 1: encode all categories into k binary variables, select variables # automatically diff --git a/tests/test_encoding/test_rare_label_encoder.py b/tests/test_encoding/test_rare_label_encoder.py index ba8529971..9594e1cc3 100644 --- a/tests/test_encoding/test_rare_label_encoder.py +++ b/tests/test_encoding/test_rare_label_encoder.py @@ -1,3 +1,5 @@ +from collections import Counter + import numpy as np import pandas as pd import pytest @@ -133,12 +135,13 @@ def test_correctly_ignores_nan_in_fit(df_enc_big): encoder.fit(df) # expected: - frequenc_cat = { + frequent_cat = { "var_A": ["B", "D", "A", "G", "C"], "var_B": ["A", "D", "B", "G", "C"], "var_C": ["C", "D", "B", "A"], } - assert encoder.encoder_dict_ == frequenc_cat + for key in frequent_cat.keys(): + assert Counter(encoder.encoder_dict_[key]) == Counter(frequent_cat[key]) # input t = pd.DataFrame( @@ -217,12 +220,13 @@ def test_correctly_ignores_nan_in_fit_when_var_is_numerical(df_enc_big): encoder.fit(df) # expected: - frequenc_cat = { + frequent_cat = { "var_A": ["B", "D", "A", "G", "C"], "var_B": ["A", "D", "B", "G", "C"], "var_C": [3, 4, 2, 1], } - assert encoder.encoder_dict_ == frequenc_cat + for key in frequent_cat.keys(): + assert Counter(encoder.encoder_dict_[key]) == Counter(frequent_cat[key]) # input t = pd.DataFrame( diff --git a/tests/test_prediction/test_check_estimator_prediction.py b/tests/test_prediction/test_check_estimator_prediction.py index 25d825be5..62a8ae5f2 100644 --- a/tests/test_prediction/test_check_estimator_prediction.py +++ b/tests/test_prediction/test_check_estimator_prediction.py @@ -220,17 +220,17 @@ def test_attributes_upon_fitting(_strategy, _bins, estimator): if _strategy == "equal_width": assert ( type(transformer._pipeline.named_steps["discretiser"]) - == EqualWidthDiscretiser + is EqualWidthDiscretiser ) else: assert ( type(transformer._pipeline.named_steps["discretiser"]) - == EqualFrequencyDiscretiser + is EqualFrequencyDiscretiser ) - assert type(transformer._pipeline.named_steps["encoder_num"]) == MeanEncoder + assert type(transformer._pipeline.named_steps["encoder_num"]) is MeanEncoder - assert type(transformer._pipeline.named_steps["encoder_cat"]) == MeanEncoder + assert type(transformer._pipeline.named_steps["encoder_cat"]) is MeanEncoder @pytest.mark.parametrize("estimator", _estimators) diff --git a/tests/test_selection/test_drop_features.py b/tests/test_selection/test_drop_features.py index 7836ad735..fb77e8c8f 100644 --- a/tests/test_selection/test_drop_features.py +++ b/tests/test_selection/test_drop_features.py @@ -23,7 +23,7 @@ def test_drop_1_variable(df_vartypes): # transform params assert X.shape == (4, 4) - assert type(X) == pd.DataFrame + assert type(X) is pd.DataFrame pd.testing.assert_frame_equal(X, df) @@ -49,7 +49,7 @@ def test_drop_1_variables_str_input(df_vartypes): # transform params assert X.shape == (4, 4) - assert type(X) == pd.DataFrame + assert type(X) is pd.DataFrame pd.testing.assert_frame_equal(X, df) @@ -71,7 +71,7 @@ def test_drop_2_variables(df_vartypes): # transform params assert X.shape == (4, 3) - assert type(X) == pd.DataFrame + assert type(X) is pd.DataFrame pd.testing.assert_frame_equal(X, df)