From c82b4d2ddaebbd535312ef7166601cad28e3172b Mon Sep 17 00:00:00 2001 From: "kyle.gilde" Date: Mon, 27 Jun 2022 22:54:30 -0500 Subject: [PATCH 01/19] created class --- .../datetime/datetime_subtraction.py | 231 ++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 feature_engine/datetime/datetime_subtraction.py diff --git a/feature_engine/datetime/datetime_subtraction.py b/feature_engine/datetime/datetime_subtraction.py new file mode 100644 index 000000000..0549a2822 --- /dev/null +++ b/feature_engine/datetime/datetime_subtraction.py @@ -0,0 +1,231 @@ +# Authors: Kyle Gilde + +from typing import List, Optional, Union + +import numpy as np +import pandas as pd +from sklearn.utils.validation import check_is_fitted + +from feature_engine.creation.base_creation import BaseCreation +from feature_engine._docstrings.methods import ( + _fit_not_learn_docstring, + _fit_transform_docstring, +) +from feature_engine._docstrings.fit_attributes import ( + _feature_names_in_docstring, + _n_features_in_docstring, +) +from feature_engine._docstrings.class_inputs import ( + _drop_original_docstring, + _missing_values_docstring, +) + +from feature_engine._docstrings.substitute import Substitution +from feature_engine.variable_manipulation import _find_or_check_datetime_variables + + +@Substitution( + missing_values=_missing_values_docstring, + drop_original=_drop_original_docstring, + feature_names_in_=_feature_names_in_docstring, + n_features_in_=_n_features_in_docstring, + fit=_fit_not_learn_docstring, + transform=BaseCreation._transform_docstring, + fit_transform=_fit_transform_docstring, +) +class RelativeFeatures(BaseCreation): + """ + DatetimeSubtraction() applies datetime subtraction between a group + of variables and one or more reference features. It adds one or more additional + features to the dataframe with the result of the operations. + + In other words, DatetimeSubtraction() subtracts a group of features from a group of + reference variables, and returns the result as new variables in the dataframe. + + The transformed dataframe will contain the additional features indicated in the + new_variables_name list plus the original set of variables. + + More details in the :ref:`User Guide `. + + Parameters + ---------- + variables: list + The list of datetime variables that the reference variables will be subtracted + from. + + reference: list + The list of datetime reference variables that will be subtracted from the + `variables`. + + output_unit: string, default='D' + The string representation of the output unit of the datetime differences. + The default is `D` for day. This parameter is passed to numpy.timedelta64. + Other possible values are `Y` for year, `M` for month, `W` for week, + `h` for hour, `m` for minute, `s` for second, `ms` for millisecond, + `us` or `μs` for microsecond, `ns` for nanosecond, `ps` for picosecond, + `fs` for femtosecond and `as` for attosecond. + + {missing_values} + + {drop_original} + + Attributes + ---------- + {feature_names_in_} + + {n_features_in_} + + Methods + ------- + {fit} + + {fit_transform} + + {transform} + + """ + + def __init__( + self, + variables: List[Union[str, int]], + reference: List[Union[str, int]], + output_unit: str = 'D', + missing_values: str = "ignore", + drop_original: bool = False, + ) -> None: + + if ( + not isinstance(variables, list) + or not all(isinstance(var, (int, str)) for var in variables) + or len(set(variables)) != len(variables) + ): + raise ValueError( + "variables must be a list of strings or integers. " + f"Got {variables} instead." + ) + + if ( + not isinstance(reference, list) + or not all(isinstance(var, (int, str)) for var in reference) + or len(set(reference)) != len(reference) + ): + raise ValueError( + "reference must be a list of strings or integers. " + f"Got {reference} instead." + ) + + valid_output_units = {'D', 'Y', 'M', 'W', 'h', 'm', 's', 'ms', 'us', 'μs', 'ns', + 'ps', 'fs', 'as'} + + if output_unit not in valid_output_units: + raise ValueError(f"output_unit accepts the following values: " + f"{valid_output_units}") + + super().__init__(missing_values, drop_original) + self.variables = variables + self.reference = reference + self.output_unit = output_unit + + def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): + """ + This transformer does not learn any parameter. + + Parameters + ---------- + X: pandas dataframe of shape = [n_samples, n_features] + The training input samples. Can be the entire dataframe, not just the + variables to transform. + + y: pandas Series, or np.array. Default=None. + It is not needed in this transformer. You can pass y or None. + """ + # Common checks and attributes + X = super().fit(X, y) + + # check variables are datetime + self.reference = _find_or_check_datetime_variables(X, self.reference) + self.variables = _find_or_check_datetime_variables(X, self.variables) + + return self + + def transform(self, X: pd.DataFrame) -> pd.DataFrame: + """ + Add new features. + + Parameters + ---------- + X: pandas dataframe of shape = [n_samples, n_features] + The data to transform. + + Returns + ------- + X_new: Pandas dataframe + The input dataframe plus the new variables. + """ + + X = super().transform(X) + + self._sub(X) + + if self.drop_original: + X.drop( + columns=set(self.variables + self.reference), + inplace=True, + ) + + return X + + def _sub(self, X): + + for reference in self.reference: + varname = [f"{var}_sub_{reference}" for var in self.variables] + X[varname] = ( + X[self.variables].sub(X[reference], axis=0) + .apply(lambda s: s / np.timedelta64(1, self.output_unit)) + ) + + return X + + def get_feature_names_out(self, input_features: Optional[bool] = None) -> List: + """Get output feature names for transformation. + + Parameters + ---------- + input_features: bool, default=None + If `input_features` is `None`, then the names of all the variables in the + transformed dataset (original + new variables) is returned. Alternatively, + if `input_features` is True, only the names for the new features will be + returned. + + Returns + ------- + feature_names_out: list + The feature names. + """ + check_is_fitted(self) + + if input_features is not None and not isinstance(input_features, bool): + raise ValueError( + "input_features takes None or a boolean, True or False. " + f"Got {input_features} instead." + ) + + # Names of new features + feature_names = [] + for reference in self.reference: + varname = [f"{var}_sub_{reference}" for var in self.variables] + feature_names.extend(varname) + + if input_features is None or input_features is False: + if self.drop_original is True: + # Remove names of variables to drop. + original = [ + f + for f in self.feature_names_in_ + if f not in self.variables + self.reference + ] + feature_names = original + feature_names + else: + feature_names = self.feature_names_in_ + feature_names + + return feature_names \ No newline at end of file From 1b316718c2d80cc1b526cc64c15b5c596d1ea852 Mon Sep 17 00:00:00 2001 From: "kyle.gilde" Date: Tue, 28 Jun 2022 10:01:47 -0500 Subject: [PATCH 02/19] fixed bugs --- .../datetime/datetime_subtraction.py | 51 ++- .../test_datetime_subtraction.py | 402 ++++++++++++++++++ 2 files changed, 448 insertions(+), 5 deletions(-) create mode 100644 tests/test_datetime/test_datetime_subtraction.py diff --git a/feature_engine/datetime/datetime_subtraction.py b/feature_engine/datetime/datetime_subtraction.py index 0549a2822..1af162117 100644 --- a/feature_engine/datetime/datetime_subtraction.py +++ b/feature_engine/datetime/datetime_subtraction.py @@ -5,8 +5,15 @@ import numpy as np import pandas as pd from sklearn.utils.validation import check_is_fitted +from sklearn.base import BaseEstimator, TransformerMixin from feature_engine.creation.base_creation import BaseCreation +from feature_engine.dataframe_checks import ( + _check_contains_na, + _check_contains_inf, + _check_X_matches_training_df, + check_X, +) from feature_engine._docstrings.methods import ( _fit_not_learn_docstring, _fit_transform_docstring, @@ -33,7 +40,7 @@ transform=BaseCreation._transform_docstring, fit_transform=_fit_transform_docstring, ) -class RelativeFeatures(BaseCreation): +class DatetimeSubtraction(BaseEstimator, TransformerMixin): """ DatetimeSubtraction() applies datetime subtraction between a group of variables and one or more reference features. It adds one or more additional @@ -89,9 +96,9 @@ def __init__( self, variables: List[Union[str, int]], reference: List[Union[str, int]], - output_unit: str = 'D', missing_values: str = "ignore", drop_original: bool = False, + output_unit: str = 'D', ) -> None: if ( @@ -114,6 +121,18 @@ def __init__( f"Got {reference} instead." ) + if not isinstance(drop_original, bool): + raise ValueError( + "drop_original takes only booleans True or False. " + f"Got {drop_original} instead." + ) + + if missing_values not in ["raise", "ignore"]: + raise ValueError( + "missing_values takes only values 'raise' or 'ignore'. " + f"Got {missing_values} instead." + ) + valid_output_units = {'D', 'Y', 'M', 'W', 'h', 'm', 's', 'ms', 'us', 'μs', 'ns', 'ps', 'fs', 'as'} @@ -121,9 +140,10 @@ def __init__( raise ValueError(f"output_unit accepts the following values: " f"{valid_output_units}") - super().__init__(missing_values, drop_original) self.variables = variables self.reference = reference + self.drop_original = drop_original + self.missing_values = missing_values self.output_unit = output_unit def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): @@ -140,12 +160,23 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): It is not needed in this transformer. You can pass y or None. """ # Common checks and attributes - X = super().fit(X, y) + X = check_X(X) # check variables are datetime self.reference = _find_or_check_datetime_variables(X, self.reference) self.variables = _find_or_check_datetime_variables(X, self.variables) + # check if dataset contains na + if self.missing_values == "raise": + _check_contains_na(X, self.variables) + _check_contains_inf(X, self.variables) + + # save input features + self.feature_names_in_ = X.columns.tolist() + + # save train set shape + self.n_features_in_ = X.shape[1] + return self def transform(self, X: pd.DataFrame) -> pd.DataFrame: @@ -163,7 +194,17 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: The input dataframe plus the new variables. """ - X = super().transform(X) + # Check method fit has been called + check_is_fitted(self) + + # check that input is a dataframe + X = check_X(X) + + # Check if input data contains same number of columns as dataframe used to fit. + _check_X_matches_training_df(X, self.n_features_in_) + + # reorder variables to match train set + X = X[self.feature_names_in_] self._sub(X) diff --git a/tests/test_datetime/test_datetime_subtraction.py b/tests/test_datetime/test_datetime_subtraction.py new file mode 100644 index 000000000..ff4378d83 --- /dev/null +++ b/tests/test_datetime/test_datetime_subtraction.py @@ -0,0 +1,402 @@ +import numpy as np +import pandas as pd +import pytest + +from sklearn.pipeline import Pipeline + +from feature_engine.creation import RelativeFeatures + + +def test_mandatory_init_parameters(): + with pytest.raises(TypeError): + RelativeFeatures(reference=["var1"], func=["add"]) + with pytest.raises(TypeError): + RelativeFeatures(variables=["var1"], func=["add"]) + with pytest.raises(TypeError): + RelativeFeatures(variables=["var1"], reference=["var2"]) + + +_variables = ["var1", ["var1", "var1", "var2"], ["var1", 0.5], ("Age", "Name")] + + +@pytest.mark.parametrize("_variables", _variables) +def test_error_when_param_variables_not_permitted(_variables): + with pytest.raises(ValueError): + RelativeFeatures( + variables=_variables, reference=["Age", "Name"], func=["add", "mul"] + ) + + +@pytest.mark.parametrize("_variables", _variables) +def test_error_when_param_reference_not_permitted(_variables): + with pytest.raises(ValueError): + RelativeFeatures( + reference=_variables, variables=["Age", "Name"], func=["add", "mul"] + ) + + +_operations = [ + "add", + ["add", "add", "mul"], + ["add", "multiply"], + ("add", "mul"), + [np.mean, "add"], +] + + +@pytest.mark.parametrize("_func", _operations) +def test_error_if_func_not_supported(_func): + with pytest.raises(ValueError): + RelativeFeatures( + variables=["Age", "Name"], + reference=["Age", "Name"], + func=_func, + ) + + +def test_error_when_drop_original_not_bool(): + for drop_original in ["True", [True]]: + with pytest.raises(TypeError): + RelativeFeatures( + variables=["Age"], + reference=["Marks"], + func=["add", "mul"], + drop_original=drop_original, + ) + + +def test_error_when_variables_not_numeric(df_vartypes): + transformer = RelativeFeatures( + variables=["Name", "Age", "Marks"], + reference=["Age", "Name"], + func=["sub"], + ) + with pytest.raises(TypeError): + transformer.fit_transform(df_vartypes) + + transformer = RelativeFeatures( + reference=["Name", "Age", "Marks"], + variables=["Age", "Name"], + func=["sub"], + ) + with pytest.raises(TypeError): + transformer.fit_transform(df_vartypes) + + +def test_error_when_entered_variables_not_in_df(df_vartypes): + transformer = RelativeFeatures( + variables=["FeatOutsideDataset", "Age"], + reference=["Age", "Name"], + func=["sub"], + ) + with pytest.raises(KeyError): + transformer.fit_transform(df_vartypes) + + transformer = RelativeFeatures( + reference=["FeatOutsideDataset", "Age"], + variables=["Age", "Name"], + func=["sub"], + ) + with pytest.raises(TypeError): + transformer.fit_transform(df_vartypes) + + +def test_classic_binary_operation(df_vartypes): + + transformer = RelativeFeatures( + variables=["Age"], + reference=["Marks"], + func=["sub", "div", "add", "mul"], + ) + + X = transformer.fit_transform(df_vartypes) + + ref = pd.DataFrame.from_dict( + { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], + "dob": pd.date_range("2020-02-24", periods=4, freq="T"), + "Age_sub_Marks": [19.1, 20.2, 18.3, 17.4], + "Age_div_Marks": [22.22222222222222, 26.25, 27.142857142857146, 30.0], + "Age_add_Marks": [20.9, 21.8, 19.7, 18.6], + "Age_mul_Marks": [18.0, 16.8, 13.299999999999999, 10.799999999999999], + } + ) + + pd.testing.assert_frame_equal(X, ref) + + +def test_alternative_operation(df_vartypes): + + # input df + df = df_vartypes.copy() + + # Expected result + dft = df.copy() + dft["Age_truediv_Marks"] = dft["Age"].truediv(dft["Marks"]) + dft["Age_floordiv_Marks"] = dft["Age"].floordiv(dft["Marks"]) + dft["Age_mod_Marks"] = dft["Age"].mod(dft["Marks"]) + dft["Age_pow_Marks"] = dft["Age"].pow(dft["Marks"]) + + transformer = RelativeFeatures( + variables=["Age"], + reference=["Marks"], + func=["truediv", "floordiv", "mod", "pow"], + ) + X = transformer.fit_transform(df) + + pd.testing.assert_frame_equal(X, dft) + + +def test_operations_with_multiple_variables(df_vartypes): + transformer = RelativeFeatures( + variables=["Age", "Marks"], + reference=["Age", "Marks"], + func=["sub"], + ) + + X = transformer.fit_transform(df_vartypes) + + ref = pd.DataFrame.from_dict( + { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], + "dob": pd.date_range("2020-02-24", periods=4, freq="T"), + "Age_sub_Age": [0, 0, 0, 0], + "Marks_sub_Age": [-19.1, -20.2, -18.3, -17.4], + "Age_sub_Marks": [19.1, 20.2, 18.3, 17.4], + "Marks_sub_Marks": [0.0, 0.0, 0.0, 0.0], + } + ) + + pd.testing.assert_frame_equal(X, ref) + + +def test_multiple_operations_with_multiple_variables(df_vartypes): + transformer = RelativeFeatures( + variables=["Age", "Marks"], + reference=["Age", "Marks"], + func=["sub", "add"], + ) + + X = transformer.fit_transform(df_vartypes) + + ref = pd.DataFrame.from_dict( + { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], + "dob": pd.date_range("2020-02-24", periods=4, freq="T"), + "Age_sub_Age": [0, 0, 0, 0], + "Marks_sub_Age": [-19.1, -20.2, -18.3, -17.4], + "Age_sub_Marks": [19.1, 20.2, 18.3, 17.4], + "Marks_sub_Marks": [0.0, 0.0, 0.0, 0.0], + "Age_add_Age": [40, 42, 38, 36], + "Marks_add_Age": [20.9, 21.8, 19.7, 18.6], + "Age_add_Marks": [20.9, 21.8, 19.7, 18.6], + "Marks_add_Marks": [1.8, 1.6, 1.4, 1.2], + } + ) + + pd.testing.assert_frame_equal(X, ref) + + transformer = RelativeFeatures( + variables=["Age", "Marks"], + reference=["Age", "Marks"], + func=["add", "sub"], + ) + + X = transformer.fit_transform(df_vartypes) + + ref = pd.DataFrame.from_dict( + { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], + "dob": pd.date_range("2020-02-24", periods=4, freq="T"), + "Age_add_Age": [40, 42, 38, 36], + "Marks_add_Age": [20.9, 21.8, 19.7, 18.6], + "Age_add_Marks": [20.9, 21.8, 19.7, 18.6], + "Marks_add_Marks": [1.8, 1.6, 1.4, 1.2], + "Age_sub_Age": [0, 0, 0, 0], + "Marks_sub_Age": [-19.1, -20.2, -18.3, -17.4], + "Age_sub_Marks": [19.1, 20.2, 18.3, 17.4], + "Marks_sub_Marks": [0.0, 0.0, 0.0, 0.0], + } + ) + + pd.testing.assert_frame_equal(X, ref) + + +def test_when_missing_values_is_ignore(df_vartypes): + + df_na = df_vartypes.copy() + df_na.loc[1, "Age"] = np.nan + + transformer = RelativeFeatures( + variables=["Age", "Marks"], + reference=["Age", "Marks"], + func=["sub"], + missing_values="ignore", + ) + + X = transformer.fit_transform(df_na) + + ref = pd.DataFrame.from_dict( + { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, np.nan, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], + "dob": pd.date_range("2020-02-24", periods=4, freq="T"), + "Age_sub_Age": [0, np.nan, 0, 0], + "Marks_sub_Age": [-19.1, np.nan, -18.3, -17.4], + "Age_sub_Marks": [19.1, np.nan, 18.3, 17.4], + "Marks_sub_Marks": [0.0, 0.0, 0.0, 0.0], + } + ) + + pd.testing.assert_frame_equal(X, ref) + + +def test_error_when_null_values_in_variable(df_vartypes): + + df_na = df_vartypes.copy() + df_na.loc[1, "Age"] = np.nan + + transformer = RelativeFeatures( + variables=["Age", "Marks"], + reference=["Age", "Marks"], + func=["add", "mul"], + missing_values="raise", + ) + + with pytest.raises(ValueError): + transformer.fit(df_na) + + transformer.fit(df_vartypes) + with pytest.raises(ValueError): + transformer.transform(df_na) + + +def test_when_df_cols_are_integers(df_vartypes): + df = df_vartypes.copy() + df.columns = [0, 1, 2, 3, 4] + + transformer = RelativeFeatures( + variables=[2, 3], + reference=[2, 3], + func=["sub", "add"], + ) + + X = transformer.fit_transform(df) + + ref = pd.DataFrame.from_dict( + { + 0: ["tom", "nick", "krish", "jack"], + 1: ["London", "Manchester", "Liverpool", "Bristol"], + 2: [20, 21, 19, 18], + 3: [0.9, 0.8, 0.7, 0.6], + 4: pd.date_range("2020-02-24", periods=4, freq="T"), + "2_sub_2": [0, 0, 0, 0], + "3_sub_2": [-19.1, -20.2, -18.3, -17.4], + "2_sub_3": [19.1, 20.2, 18.3, 17.4], + "3_sub_3": [0.0, 0.0, 0.0, 0.0], + "2_add_2": [40, 42, 38, 36], + "3_add_2": [20.9, 21.8, 19.7, 18.6], + "2_add_3": [20.9, 21.8, 19.7, 18.6], + "3_add_3": [1.8, 1.6, 1.4, 1.2], + } + ) + + pd.testing.assert_frame_equal(X, ref) + + +@pytest.mark.parametrize("_func", [["div"], ["truediv"], ["floordiv"], ["mod"]]) +def test_error_when_division_by_zero(_func, df_vartypes): + + df_zero = df_vartypes.copy() + df_zero.loc[1, "Marks"] = 0 + + transformer = RelativeFeatures( + variables=["Age"], + reference=["Marks"], + func=_func, + ) + transformer.fit(df_vartypes) + with pytest.raises(ValueError): + transformer.transform(df_zero) + + +@pytest.mark.parametrize("_drop", [True, False]) +def test_get_feature_names_out(_drop, df_vartypes): + transformer = RelativeFeatures( + variables=["Age", "Marks"], + reference=["Age", "Marks"], + func=["add", "sub"], + drop_original=_drop, + ) + varnames = [ + "Age_add_Age", + "Marks_add_Age", + "Age_add_Marks", + "Marks_add_Marks", + "Age_sub_Age", + "Marks_sub_Age", + "Age_sub_Marks", + "Marks_sub_Marks", + ] + + X = transformer.fit_transform(df_vartypes) + assert list(X.columns) == transformer.get_feature_names_out(input_features=None) + assert list(X.columns) == transformer.get_feature_names_out(input_features=False) + assert varnames == transformer.get_feature_names_out(input_features=True) + + +@pytest.mark.parametrize("_drop", [True, False]) +def test_get_feature_names_out_from_pipeline(_drop, df_vartypes): + transformer = RelativeFeatures( + variables=["Age", "Marks"], + reference=["Age", "Marks"], + func=["add", "sub"], + drop_original=_drop, + ) + + pipe = Pipeline([("transformer", transformer)]) + + varnames = [ + "Age_add_Age", + "Marks_add_Age", + "Age_add_Marks", + "Marks_add_Marks", + "Age_sub_Age", + "Marks_sub_Age", + "Age_sub_Marks", + "Marks_sub_Marks", + ] + + X = pipe.fit_transform(df_vartypes) + assert list(X.columns) == pipe.get_feature_names_out(input_features=None) + assert list(X.columns) == pipe.get_feature_names_out(input_features=False) + assert varnames == pipe.get_feature_names_out(input_features=True) + + +@pytest.mark.parametrize("_input_features", ["hola", ["Age", "Marks"]]) +def test_get_feature_names_out_raises_error_when_wrong_param( + _input_features, df_vartypes +): + transformer = RelativeFeatures( + variables=["Age", "Marks"], + reference=["Age", "Marks"], + func=["add", "sub"], + ) + transformer.fit(df_vartypes) + + with pytest.raises(ValueError): + transformer.get_feature_names_out(input_features=_input_features) From 44cd7476c54fa7b9cca6ffecb40a7dd708f564bc Mon Sep 17 00:00:00 2001 From: "kyle.gilde" Date: Tue, 12 Jul 2022 09:39:56 -0500 Subject: [PATCH 03/19] added last newline --- feature_engine/datetime/datetime_subtraction.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/feature_engine/datetime/datetime_subtraction.py b/feature_engine/datetime/datetime_subtraction.py index 1af162117..5b261afc4 100644 --- a/feature_engine/datetime/datetime_subtraction.py +++ b/feature_engine/datetime/datetime_subtraction.py @@ -269,4 +269,5 @@ def get_feature_names_out(self, input_features: Optional[bool] = None) -> List: else: feature_names = self.feature_names_in_ + feature_names - return feature_names \ No newline at end of file + return feature_names + \ No newline at end of file From c4cc75f7ad2f181e0bd802ed93933bc7f590a77b Mon Sep 17 00:00:00 2001 From: "kyle.gilde" Date: Tue, 12 Jul 2022 09:42:46 -0500 Subject: [PATCH 04/19] added last newline --- feature_engine/datetime/datetime_subtraction.py | 1 - 1 file changed, 1 deletion(-) diff --git a/feature_engine/datetime/datetime_subtraction.py b/feature_engine/datetime/datetime_subtraction.py index 5b261afc4..8f63605df 100644 --- a/feature_engine/datetime/datetime_subtraction.py +++ b/feature_engine/datetime/datetime_subtraction.py @@ -270,4 +270,3 @@ def get_feature_names_out(self, input_features: Optional[bool] = None) -> List: feature_names = self.feature_names_in_ + feature_names return feature_names - \ No newline at end of file From 9b4e82b3f2f3fbba61a7d1d7a0eb886a26b5e0b5 Mon Sep 17 00:00:00 2001 From: "kyle.gilde" Date: Tue, 12 Jul 2022 09:45:12 -0500 Subject: [PATCH 05/19] updated init --- feature_engine/datetime/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/feature_engine/datetime/__init__.py b/feature_engine/datetime/__init__.py index ac06ff156..aea69f9fe 100644 --- a/feature_engine/datetime/__init__.py +++ b/feature_engine/datetime/__init__.py @@ -1,5 +1,6 @@ "The module datetime computes features from dates and times." from .datetime import DatetimeFeatures +from .datetime_subtraction import DatetimeSubtraction -__all__ = ["DatetimeFeatures"] +__all__ = ["DatetimeFeatures", "DatetimeSubtraction"] From d006e35f1fc6bb923c54d2861b51eaff92eba22a Mon Sep 17 00:00:00 2001 From: "kyle.gilde" Date: Thu, 4 Aug 2022 21:53:17 -0500 Subject: [PATCH 06/19] implemented feedback --- .../datetime/datetime_subtraction.py | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/feature_engine/datetime/datetime_subtraction.py b/feature_engine/datetime/datetime_subtraction.py index 8f63605df..6ab8e3306 100644 --- a/feature_engine/datetime/datetime_subtraction.py +++ b/feature_engine/datetime/datetime_subtraction.py @@ -7,10 +7,10 @@ from sklearn.utils.validation import check_is_fitted from sklearn.base import BaseEstimator, TransformerMixin +from feature_engine.tags import _return_tags from feature_engine.creation.base_creation import BaseCreation from feature_engine.dataframe_checks import ( _check_contains_na, - _check_contains_inf, _check_X_matches_training_df, check_X, ) @@ -138,7 +138,7 @@ def __init__( if output_unit not in valid_output_units: raise ValueError(f"output_unit accepts the following values: " - f"{valid_output_units}") + f"{valid_output_units}. Got {output_unit} instead.") self.variables = variables self.reference = reference @@ -168,8 +168,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # check if dataset contains na if self.missing_values == "raise": - _check_contains_na(X, self.variables) - _check_contains_inf(X, self.variables) + _check_contains_na(X, self.variables + self.reference) # save input features self.feature_names_in_ = X.columns.tolist() @@ -203,6 +202,9 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # Check if input data contains same number of columns as dataframe used to fit. _check_X_matches_training_df(X, self.n_features_in_) + if self.missing_values == 'raise': + _check_contains_na(X, self.variables + self.reference) + # reorder variables to match train set X = X[self.feature_names_in_] @@ -219,8 +221,8 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: def _sub(self, X): for reference in self.reference: - varname = [f"{var}_sub_{reference}" for var in self.variables] - X[varname] = ( + new_varnames = [f"{var}_sub_{reference}" for var in self.variables] + X[new_varnames] = ( X[self.variables].sub(X[reference], axis=0) .apply(lambda s: s / np.timedelta64(1, self.output_unit)) ) @@ -270,3 +272,17 @@ def get_feature_names_out(self, input_features: Optional[bool] = None) -> List: feature_names = self.feature_names_in_ + feature_names return feature_names + + def _more_tags(self): + tags_dict = _return_tags() + tags_dict["allow_nan"] = True + tags_dict["variables"] = "skip" + # Tests that are OK to fail: + tags_dict["_xfail_checks"][ + "check_parameters_default_constructible" + ] = "transformer has 1 mandatory parameter" + tags_dict["_xfail_checks"][ + "check_fit2d_1feature" + ] = "this transformer works with datasets that contain at least 2 variables. " \ + "Otherwise, there is nothing to combine" + return tags_dict \ No newline at end of file From 2ed4616b79fb3996f85efdc5d45096482b7287e8 Mon Sep 17 00:00:00 2001 From: "kyle.gilde" Date: Thu, 4 Aug 2022 23:47:50 -0500 Subject: [PATCH 07/19] fixed indent --- .../datetime/datetime_subtraction.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/feature_engine/datetime/datetime_subtraction.py b/feature_engine/datetime/datetime_subtraction.py index 6ab8e3306..53a336dc5 100644 --- a/feature_engine/datetime/datetime_subtraction.py +++ b/feature_engine/datetime/datetime_subtraction.py @@ -273,16 +273,16 @@ def get_feature_names_out(self, input_features: Optional[bool] = None) -> List: return feature_names - def _more_tags(self): - tags_dict = _return_tags() - tags_dict["allow_nan"] = True - tags_dict["variables"] = "skip" - # Tests that are OK to fail: - tags_dict["_xfail_checks"][ - "check_parameters_default_constructible" - ] = "transformer has 1 mandatory parameter" - tags_dict["_xfail_checks"][ - "check_fit2d_1feature" - ] = "this transformer works with datasets that contain at least 2 variables. " \ + def _more_tags(self): + tags_dict = _return_tags() + tags_dict["allow_nan"] = True + tags_dict["variables"] = "skip" + # Tests that are OK to fail: + tags_dict["_xfail_checks"][ + "check_parameters_default_constructible" + ] = "transformer has 1 mandatory parameter" + tags_dict["_xfail_checks"][ + "check_fit2d_1feature" + ] = "this transformer works with datasets that contain at least 2 variables. " \ "Otherwise, there is nothing to combine" - return tags_dict \ No newline at end of file + return tags_dict \ No newline at end of file From d89073fd309e973ba5c606ebc34eb20eaafa1237 Mon Sep 17 00:00:00 2001 From: "kyle.gilde" Date: Fri, 5 Aug 2022 16:02:34 -0500 Subject: [PATCH 08/19] used black --- .../datetime/datetime_subtraction.py | 41 +++++++++++++------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/feature_engine/datetime/datetime_subtraction.py b/feature_engine/datetime/datetime_subtraction.py index 53a336dc5..d39e2473c 100644 --- a/feature_engine/datetime/datetime_subtraction.py +++ b/feature_engine/datetime/datetime_subtraction.py @@ -98,7 +98,7 @@ def __init__( reference: List[Union[str, int]], missing_values: str = "ignore", drop_original: bool = False, - output_unit: str = 'D', + output_unit: str = "D", ) -> None: if ( @@ -133,12 +133,28 @@ def __init__( f"Got {missing_values} instead." ) - valid_output_units = {'D', 'Y', 'M', 'W', 'h', 'm', 's', 'ms', 'us', 'μs', 'ns', - 'ps', 'fs', 'as'} + valid_output_units = { + "D", + "Y", + "M", + "W", + "h", + "m", + "s", + "ms", + "us", + "μs", + "ns", + "ps", + "fs", + "as", + } if output_unit not in valid_output_units: - raise ValueError(f"output_unit accepts the following values: " - f"{valid_output_units}. Got {output_unit} instead.") + raise ValueError( + f"output_unit accepts the following values: " + f"{valid_output_units}. Got {output_unit} instead." + ) self.variables = variables self.reference = reference @@ -202,7 +218,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # Check if input data contains same number of columns as dataframe used to fit. _check_X_matches_training_df(X, self.n_features_in_) - if self.missing_values == 'raise': + if self.missing_values == "raise": _check_contains_na(X, self.variables + self.reference) # reorder variables to match train set @@ -223,7 +239,8 @@ def _sub(self, X): for reference in self.reference: new_varnames = [f"{var}_sub_{reference}" for var in self.variables] X[new_varnames] = ( - X[self.variables].sub(X[reference], axis=0) + X[self.variables] + .sub(X[reference], axis=0) .apply(lambda s: s / np.timedelta64(1, self.output_unit)) ) @@ -281,8 +298,8 @@ def _more_tags(self): tags_dict["_xfail_checks"][ "check_parameters_default_constructible" ] = "transformer has 1 mandatory parameter" - tags_dict["_xfail_checks"][ - "check_fit2d_1feature" - ] = "this transformer works with datasets that contain at least 2 variables. " \ - "Otherwise, there is nothing to combine" - return tags_dict \ No newline at end of file + tags_dict["_xfail_checks"]["check_fit2d_1feature"] = ( + "this transformer works with datasets that contain at least 2 variables. " + "Otherwise, there is nothing to combine" + ) + return tags_dict From 5527c57014ea9de8662e403c47c3a0b73cbeb588 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Wed, 24 Aug 2022 14:35:35 +0200 Subject: [PATCH 09/19] fix imports --- feature_engine/datetime/datetime_subtraction.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/feature_engine/datetime/datetime_subtraction.py b/feature_engine/datetime/datetime_subtraction.py index d39e2473c..7ce1b7110 100644 --- a/feature_engine/datetime/datetime_subtraction.py +++ b/feature_engine/datetime/datetime_subtraction.py @@ -22,13 +22,15 @@ _feature_names_in_docstring, _n_features_in_docstring, ) -from feature_engine._docstrings.class_inputs import ( +from feature_engine._docstrings.init_parameters import ( _drop_original_docstring, _missing_values_docstring, ) from feature_engine._docstrings.substitute import Substitution -from feature_engine.variable_manipulation import _find_or_check_datetime_variables +from feature_engine._variable_handling.variable_type_selection import ( + _find_or_check_datetime_variables, +) @Substitution( From ac6337beea460442f75b5b66c768147cfab727bc Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Thu, 9 Mar 2023 22:34:17 +0100 Subject: [PATCH 10/19] refactor subtraction class --- .../datetime/datetime_subtraction.py | 261 ++++++++---------- 1 file changed, 115 insertions(+), 146 deletions(-) diff --git a/feature_engine/datetime/datetime_subtraction.py b/feature_engine/datetime/datetime_subtraction.py index 7ce1b7110..f56d72da5 100644 --- a/feature_engine/datetime/datetime_subtraction.py +++ b/feature_engine/datetime/datetime_subtraction.py @@ -1,14 +1,16 @@ -# Authors: Kyle Gilde - from typing import List, Optional, Union import numpy as np import pandas as pd +from pandas.api.types import is_datetime64_any_dtype as is_datetime + from sklearn.utils.validation import check_is_fitted -from sklearn.base import BaseEstimator, TransformerMixin -from feature_engine.tags import _return_tags +from feature_engine.variable_handling._init_parameter_checks import ( + _check_init_parameter_variables, +) from feature_engine.creation.base_creation import BaseCreation +from feature_engine.tags import _return_tags from feature_engine.dataframe_checks import ( _check_contains_na, _check_X_matches_training_df, @@ -17,20 +19,19 @@ from feature_engine._docstrings.methods import ( _fit_not_learn_docstring, _fit_transform_docstring, + _transform_creation_docstring, ) from feature_engine._docstrings.fit_attributes import ( _feature_names_in_docstring, _n_features_in_docstring, ) -from feature_engine._docstrings.init_parameters import ( +from feature_engine._docstrings.init_parameters.all_trasnformers import ( _drop_original_docstring, _missing_values_docstring, ) from feature_engine._docstrings.substitute import Substitution -from feature_engine._variable_handling.variable_type_selection import ( - _find_or_check_datetime_variables, -) +from feature_engine.variable_handling import find_or_check_datetime_variables @Substitution( @@ -39,20 +40,14 @@ feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, fit=_fit_not_learn_docstring, - transform=BaseCreation._transform_docstring, + transform=_transform_creation_docstring, fit_transform=_fit_transform_docstring, ) -class DatetimeSubtraction(BaseEstimator, TransformerMixin): +class DatetimeSubtraction(BaseCreation): """ - DatetimeSubtraction() applies datetime subtraction between a group - of variables and one or more reference features. It adds one or more additional - features to the dataframe with the result of the operations. - - In other words, DatetimeSubtraction() subtracts a group of features from a group of - reference variables, and returns the result as new variables in the dataframe. - - The transformed dataframe will contain the additional features indicated in the - new_variables_name list plus the original set of variables. + DatetimeSubtraction() applies datetime subtraction between a group of datetime + variables and one or more datetime features, adding the resulting variables to the + dataframe. More details in the :ref:`User Guide `. @@ -60,15 +55,15 @@ class DatetimeSubtraction(BaseEstimator, TransformerMixin): ---------- variables: list The list of datetime variables that the reference variables will be subtracted - from. + from (left side of the subtraction operation). reference: list - The list of datetime reference variables that will be subtracted from the - `variables`. + The list of datetime reference variables that will be subtracted from + `variables` (right side of the subtraction operation). output_unit: string, default='D' The string representation of the output unit of the datetime differences. - The default is `D` for day. This parameter is passed to numpy.timedelta64. + The default is `D` for day. This parameter is passed to `numpy.timedelta64`. Other possible values are `Y` for year, `M` for month, `W` for week, `h` for hour, `m` for minute, `s` for second, `ms` for millisecond, `us` or `μs` for microsecond, `ns` for nanosecond, `ps` for picosecond, @@ -78,6 +73,23 @@ class DatetimeSubtraction(BaseEstimator, TransformerMixin): {drop_original} + dayfirst: bool, default="False" + Specify a date parse order if arg is str or is list-like. If True, parses + dates with the day first, e.g. 10/11/12 is parsed as 2012-11-10. Same as in + `pandas.to_datetime`. + + yearfirst: bool, default="False" + Specify a date parse order if arg is str or is list-like. + Same as in `pandas.to_datetime`. + + - If True parses dates with the year first, e.g. 10/11/12 is parsed as + 2010-11-12. + - If both dayfirst and yearfirst are True, yearfirst is preceded. + + utc: bool, default=None + Return UTC DatetimeIndex if True (converting any tz-aware datetime.datetime + objects as well). Same as in `pandas.to_datetime`. + Attributes ---------- {feature_names_in_} @@ -92,64 +104,36 @@ class DatetimeSubtraction(BaseEstimator, TransformerMixin): {transform} + Examples + -------- + + >>> import pandas as pd + >>> from feature_engine.datetime import DatetimeSubtraction + >>> X = pd.DataFrame({ + >>> "date1" : ["2022-09-18", "2022-10-27", "2022-12-24"], + >>> "date2" : ["2022-08-18", "2022-08-27", "2022-06-24"]}) + >>> dtf = DatetimeSubtraction(variables=["date1"], reference=["date2"]) + >>> dtf.fit(X) + >>> dtf.transform(X) + date1 date2 date1_sub_date2 + 0 2022-09-18 2022-08-18 31.0 + 1 2022-10-27 2022-08-27 61.0 + 2 2022-12-24 2022-06-24 183.0 """ - def __init__( self, - variables: List[Union[str, int]], - reference: List[Union[str, int]], + variables: Union[None, int, str, List[Union[str, int]]], + reference: Union[None, int, str, List[Union[str, int]]], + output_unit: str = "D", missing_values: str = "ignore", drop_original: bool = False, - output_unit: str = "D", + dayfirst: bool = False, + yearfirst: bool = False, + utc: Union[None, bool] = None, ) -> None: - if ( - not isinstance(variables, list) - or not all(isinstance(var, (int, str)) for var in variables) - or len(set(variables)) != len(variables) - ): - raise ValueError( - "variables must be a list of strings or integers. " - f"Got {variables} instead." - ) - - if ( - not isinstance(reference, list) - or not all(isinstance(var, (int, str)) for var in reference) - or len(set(reference)) != len(reference) - ): - raise ValueError( - "reference must be a list of strings or integers. " - f"Got {reference} instead." - ) - - if not isinstance(drop_original, bool): - raise ValueError( - "drop_original takes only booleans True or False. " - f"Got {drop_original} instead." - ) - - if missing_values not in ["raise", "ignore"]: - raise ValueError( - "missing_values takes only values 'raise' or 'ignore'. " - f"Got {missing_values} instead." - ) - valid_output_units = { - "D", - "Y", - "M", - "W", - "h", - "m", - "s", - "ms", - "us", - "μs", - "ns", - "ps", - "fs", - "as", + "D", "Y", "M", "W", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as", } if output_unit not in valid_output_units: @@ -158,11 +142,13 @@ def __init__( f"{valid_output_units}. Got {output_unit} instead." ) - self.variables = variables - self.reference = reference - self.drop_original = drop_original - self.missing_values = missing_values + super().__init__(missing_values, drop_original) + self.variables = _check_init_parameter_variables(variables) + self.reference = _check_init_parameter_variables(reference) self.output_unit = output_unit + self.dayfirst = dayfirst + self.yearfirst = yearfirst + self.utc = utc def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ @@ -181,12 +167,12 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): X = check_X(X) # check variables are datetime - self.reference = _find_or_check_datetime_variables(X, self.reference) - self.variables = _find_or_check_datetime_variables(X, self.variables) + self.reference_ = find_or_check_datetime_variables(X, self.reference) + self.variables_ = find_or_check_datetime_variables(X, self.variables) # check if dataset contains na if self.missing_values == "raise": - _check_contains_na(X, self.variables + self.reference) + _check_contains_na(X, self.variables_ + self.reference_) # save input features self.feature_names_in_ = X.columns.tolist() @@ -221,87 +207,70 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: _check_X_matches_training_df(X, self.n_features_in_) if self.missing_values == "raise": - _check_contains_na(X, self.variables + self.reference) + _check_contains_na(X, self.variables_ + self.reference_) # reorder variables to match train set X = X[self.feature_names_in_] - self._sub(X) + X_dt = self._to_datetime(X) - if self.drop_original: - X.drop( - columns=set(self.variables + self.reference), - inplace=True, - ) + new_features = self._sub(X_dt) - return X - - def _sub(self, X): + X = pd.concat([X, new_features], axis=1) - for reference in self.reference: - new_varnames = [f"{var}_sub_{reference}" for var in self.variables] - X[new_varnames] = ( - X[self.variables] - .sub(X[reference], axis=0) - .apply(lambda s: s / np.timedelta64(1, self.output_unit)) + if self.drop_original: + X = X.drop( + columns=set(self.variables_ + self.reference_), ) return X - def get_feature_names_out(self, input_features: Optional[bool] = None) -> List: - """Get output feature names for transformation. - - Parameters - ---------- - input_features: bool, default=None - If `input_features` is `None`, then the names of all the variables in the - transformed dataset (original + new variables) is returned. Alternatively, - if `input_features` is True, only the names for the new features will be - returned. + def _to_datetime(self, X: pd.DataFrame): + """covert variables to datetime.""" + # convert datetime variables + datetime_df = pd.concat( + [ + pd.to_datetime( + X[variable], + dayfirst=self.dayfirst, + yearfirst=self.yearfirst, + utc=self.utc, + ) + for variable in set(self.variables_+self.reference_) + ], + axis=1, + ) - Returns - ------- - feature_names_out: list - The feature names. - """ - check_is_fitted(self) + non_dt_columns = datetime_df.columns[ + ~datetime_df.apply(is_datetime) + ].tolist() - if input_features is not None and not isinstance(input_features, bool): + if non_dt_columns: raise ValueError( - "input_features takes None or a boolean, True or False. " - f"Got {input_features} instead." + "ValueError: variable(s) " + + (len(non_dt_columns) * "{} ").format(*non_dt_columns) + + "could not be converted to datetime. Try setting utc=True" + ) + return datetime_df + + def _sub(self, dt_df: pd.DataFrame): + """make datetime subtraction""" + new_df = pd.DataFrame() + for reference in self.reference_: + new_varnames = [f"{var}_sub_{reference}" for var in self.variables_] + new_df[new_varnames] = ( + dt_df[self.variables_] + .sub(dt_df[reference], axis=0) + .apply(lambda s: s / np.timedelta64(1, self.output_unit)) ) - # Names of new features - feature_names = [] - for reference in self.reference: - varname = [f"{var}_sub_{reference}" for var in self.variables] - feature_names.extend(varname) - - if input_features is None or input_features is False: - if self.drop_original is True: - # Remove names of variables to drop. - original = [ - f - for f in self.feature_names_in_ - if f not in self.variables + self.reference - ] - feature_names = original + feature_names - else: - feature_names = self.feature_names_in_ + feature_names + return new_df + def _get_new_features_name(self) -> List: + """Return names of the created features.""" + feature_names = [ + f"{var}_sub_{reference}" + for reference in self.reference_ + for var in self.variables_ + ] return feature_names - - def _more_tags(self): - tags_dict = _return_tags() - tags_dict["allow_nan"] = True - tags_dict["variables"] = "skip" - # Tests that are OK to fail: - tags_dict["_xfail_checks"][ - "check_parameters_default_constructible" - ] = "transformer has 1 mandatory parameter" - tags_dict["_xfail_checks"]["check_fit2d_1feature"] = ( - "this transformer works with datasets that contain at least 2 variables. " - "Otherwise, there is nothing to combine" - ) - return tags_dict From 2ad385ad23ad508f6a97882cba6c1c24903c07b5 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Thu, 9 Mar 2023 22:37:41 +0100 Subject: [PATCH 11/19] add new class to general tests --- tests/test_datetime/test_check_estimator_datetime.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_datetime/test_check_estimator_datetime.py b/tests/test_datetime/test_check_estimator_datetime.py index 6fe6d1986..4b081e946 100644 --- a/tests/test_datetime/test_check_estimator_datetime.py +++ b/tests/test_datetime/test_check_estimator_datetime.py @@ -1,9 +1,9 @@ import pytest -from feature_engine.datetime import DatetimeFeatures +from feature_engine.datetime import DatetimeFeatures, DatetimeSubtraction from tests.estimator_checks.estimator_checks import check_feature_engine_estimator -_estimators = [DatetimeFeatures()] +_estimators = [DatetimeFeatures(), DatetimeSubtraction()] @pytest.mark.parametrize("estimator", _estimators) From 5166ac3827d9f4201426939aefbf1a3f5d7183c2 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sat, 11 Mar 2023 22:25:17 +0100 Subject: [PATCH 12/19] initial docs setting --- README.md | 1 + docs/api_doc/datetime/DatetimeSubtraction.rst | 6 + docs/api_doc/datetime/index.rst | 1 + docs/index.rst | 1 + .../datetime/DatetimeSubtraction.rst | 791 ++++++++++++++++++ docs/user_guide/datetime/index.rst | 3 +- 6 files changed, 802 insertions(+), 1 deletion(-) create mode 100644 docs/api_doc/datetime/DatetimeSubtraction.rst create mode 100644 docs/user_guide/datetime/DatetimeSubtraction.rst diff --git a/README.md b/README.md index 48f3bee7f..6a26eb4fc 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,7 @@ transforming parameters from the data and then transform it. ### Datetime * DatetimeFeatures + * DatetimeSubtraction ### Time Series * LagFeatures diff --git a/docs/api_doc/datetime/DatetimeSubtraction.rst b/docs/api_doc/datetime/DatetimeSubtraction.rst new file mode 100644 index 000000000..da1e28343 --- /dev/null +++ b/docs/api_doc/datetime/DatetimeSubtraction.rst @@ -0,0 +1,6 @@ +DatetimeSubtraction +=================== + +.. autoclass:: feature_engine.datetime.DatetimeSubtraction + :members: + diff --git a/docs/api_doc/datetime/index.rst b/docs/api_doc/datetime/index.rst index c30432a08..c81b6bef8 100644 --- a/docs/api_doc/datetime/index.rst +++ b/docs/api_doc/datetime/index.rst @@ -10,4 +10,5 @@ features from existing datetime or object-like data. :maxdepth: 1 DatetimeFeatures + DatetimeSubtraction diff --git a/docs/index.rst b/docs/index.rst index 1faf9cc4d..5f7685a90 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -184,6 +184,7 @@ Datetime: --------- - :doc:`api_doc/datetime/DatetimeFeatures`: extract features from datetime variables +- :doc:`api_doc/datetime/DatetimeSubtraction`: computes subtractions between datetime variables Forecasting: ------------ diff --git a/docs/user_guide/datetime/DatetimeSubtraction.rst b/docs/user_guide/datetime/DatetimeSubtraction.rst new file mode 100644 index 000000000..5efc9a191 --- /dev/null +++ b/docs/user_guide/datetime/DatetimeSubtraction.rst @@ -0,0 +1,791 @@ +.. _datetime_subtraction: + +.. currentmodule:: feature_engine.datetime + +DatetimeSubtraction +=================== + +In datasets commonly used in data science and machine learning projects, the variables very +often contain information about date and time. **Date of birth** and **time of purchase** are two +examples of these variables. They are commonly referred to as “datetime features”, that is, +data whose data type is date and time. + +We don’t normally use datetime variables in their raw format to train machine learning models, +like those for regression, classification, or clustering. Instead, we can extract a lot of information +from these variables by extracting the different date and time components of the datetime +variable. + +Examples of date and time components are the year, the month, the week_of_year, the day +of the week, the hour, the minutes, and the seconds. + +Datetime features with pandas +----------------------------- + +In Python, we can extract date and time components through the `dt` module of the open-source +library pandas. For example, by executing the following: + +.. code:: python + + data = pd.DataFrame({"date": pd.date_range("2019-03-05", periods=20, freq="D")}) + + data["year"] = data["date"].dt.year + data["quarter"] = data["date"].dt.quarter + data["month"] = data["date"].dt.month + +In the former code block we created 3 features from the timestamp variable: the *year*, the +*quarter* and the *month*. + + +Datetime features with Feature-engine +------------------------------------- + +:class:`DatetimeFeatures()` automatically extracts several date and time features from +datetime variables. It works with variables whose dtype is datetime, as well as with +object-like and categorical variables, provided that they can be parsed into datetime +format. It *cannot* extract features from numerical variables. + +:class:`DatetimeFeatures()` uses the pandas `dt` module under the hood, therefore automating +datetime feature engineering. In two lines of code and by specifying which features we +want to create with :class:`DatetimeFeatures()`, we can create multiple date and time variables +from various variables simultaneously. + +:class:`DatetimeFeatures()` can automatically create all features supported by pandas `dt` +and a few more, like, for example, a binary feature indicating if the event occurred on +a weekend and also the semester. + +With :class:`DatetimeFeatures()` we can choose which date and time features to extract +from the datetime variables. We can also extract date and time features from one or more +datetime variables at the same time. + +Through the following examples we highlight the functionality and versatility of :class:`DatetimeFeatures()` +for tabular data. + +Extract date features +~~~~~~~~~~~~~~~~~~~~~ + +In this example, we are going to extract three **date features** from a +specific variable in the dataframe. In particular, we are interested +in the *month*, the *day of the year*, and whether that day was the *last +day the month*. + +First, we will create a toy dataframe with 2 date variables: + +.. code:: python + + import pandas as pd + from feature_engine.datetime import DatetimeFeatures + + toy_df = pd.DataFrame({ + "var_date1": ['May-1989', 'Dec-2020', 'Jan-1999', 'Feb-2002'], + "var_date2": ['06/21/12', '02/10/98', '08/03/10', '10/31/20'], + }) + +Now, we will extract the variables month, month-end and the day of the year from the +second datetime variable in our dataset. + +.. code:: python + + dtfs = DatetimeFeatures( + variables="var_date2", + features_to_extract=["month", "month_end", "day_of_year"] + ) + + df_transf = dtfs.fit_transform(toy_df) + + df_transf + +With `transform()`, the features extracted from the datetime variable are added to the +dataframe. + +We see the new features in the following output: + +.. code:: python + + var_date1 var_date2_month var_date2_month_end var_date2_day_of_year + 0 May-1989 6 0 173 + 1 Dec-2020 2 0 41 + 2 Jan-1999 8 0 215 + 3 Feb-2002 10 1 305 + +By default, :class:`DatetimeFeatures()` drops the variable from which the date and time +features were extracted, in this case, *var_date2*. To keep the variable, we just need +to indicate `drop_original=False` when initializing the transformer. + +Finally, we can obtain the name of the variables in the returned data as follows: + +.. code:: python + + dtfs.get_feature_names_out() + +.. code:: python + + ['var_date1', + 'var_date2_month', + 'var_date2_month_end', + 'var_date2_day_of_year'] + + +Extract time features +~~~~~~~~~~~~~~~~~~~~~ + +In this example, we are going to extract the feature *minute* from the two time +variables in our dataset. + +First, let's create a toy dataset with 2 time variables and an object variable. + +.. code:: python + + import pandas as pd + from feature_engine.datetime import DatetimeFeatures + + toy_df = pd.DataFrame({ + "not_a_dt": ['not', 'a', 'date', 'time'], + "var_time1": ['12:34:45', '23:01:02', '11:59:21', '08:44:23'], + "var_time2": ['02:27:26', '10:10:55', '17:30:00', '18:11:18'], + }) + +:class:`DatetimeFeatures()` automatically finds all variables that can be parsed to +datetime. So if we want to extract time features from all our datetime variables, we +don't need to specify them. + +.. code:: python + + dfts = DatetimeFeatures(features_to_extract=["minute"]) + + df_transf = dfts.fit_transform(toy_df) + + df_transf + +We see the new features in the following output: + +.. code:: python + + not_a_dt var_time1_minute var_time2_minute + 0 not 34 27 + 1 a 1 10 + 2 date 59 30 + 3 time 44 11 + + +The transformer found two variables in the dataframe that can be cast to datetime and +proceeded to extract the requested feature from them. + +The variables detected as datetime are stored in the transformer's `variables_` attribute: + +.. code:: python + + dfts.variables_ + +.. code:: python + + ['var_time1', 'var_time2'] + +The original datetime variables are dropped from the data by default. This leaves the +dataset ready to train machine learning algorithms like linear regression or random forests. + +If we want to keep the datetime variables, we just need to indicate `drop_original=False` +when initializing the transformer. + +Finally, if we want to obtain the names of the variables in the output data, we can use: + +.. code:: python + + dfts.get_feature_names_out() + +.. code:: python + + ['not_a_dt', 'var_time1_minute', 'var_time2_minute'] + + +Extract date and time features +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In this example, we will combine what we have seen in the previous two examples +and extract a date feature - *year* - and time feature - *hour* - +from two variables that contain both date and time information. + +Let's go ahead and create a toy dataset with 3 datetime variables. + +.. code:: python + + import pandas as pd + from feature_engine.datetime import DatetimeFeatures + + toy_df = pd.DataFrame({ + "var_dt1": pd.date_range("2018-01-01", periods=3, freq="H"), + "var_dt2": ['08/31/00 12:34:45', '12/01/90 23:01:02', '04/25/01 11:59:21'], + "var_dt3": ['03/02/15 02:27:26', '02/28/97 10:10:55', '11/11/03 17:30:00'], + }) + +Now, we set up the :class:`DatetimeFeatures()` to extract features from 2 of the datetime +variables. In this case, we do not want to drop the datetime variable after extracting +the features. + +.. code:: python + + dfts = DatetimeFeatures( + variables=["var_dt1", "var_dt3"], + features_to_extract=["year", "hour"], + drop_original=False, + ) + df_transf = dfts.fit_transform(toy_df) + + print(df_transf) + +We can see the resulting dataframe in the following output: + +.. code:: python + + var_dt1 var_dt2 var_dt3 var_dt1_year \ + 0 2018-01-01 00:00:00 08/31/00 12:34:45 03/02/15 02:27:26 2018 + 1 2018-01-01 01:00:00 12/01/90 23:01:02 02/28/97 10:10:55 2018 + 2 2018-01-01 02:00:00 04/25/01 11:59:21 11/11/03 17:30:00 2018 + + var_dt1_hour var_dt3_year var_dt3_hour + 0 0 2015 2 + 1 1 1997 10 + 2 2 2003 17 + +And that is it. The new features are now added to the dataframe. + +Time series +~~~~~~~~~~~ + +Time series data consists of datapoints indexed in time order. The time is usually in +the index of the dataframe. We can extract features from the timestamp index and use them +for time series regression or classification, as well as for time series forecasting. + +With :class:`DatetimeFeatures()` we can also create date and time features from the +dataframe index. + +Let's create a toy dataframe with datetime in the index. + +.. code:: python + + import pandas as pd + + X = {"ambient_temp": [31.31, 31.51, 32.15, 32.39, 32.62, 32.5, 32.52, 32.68], + "module_temp": [49.18, 49.84, 52.35, 50.63, 49.61, 47.01, 46.67, 47.52], + "irradiation": [0.51, 0.79, 0.65, 0.76, 0.42, 0.49, 0.57, 0.56], + "color": ["green"] * 4 + ["blue"] * 4, + } + + X = pd.DataFrame(X) + X.index = pd.date_range("2020-05-15 12:00:00", periods=8, freq="15min") + + X.head() + +Below we see the output of our toy dataframe: + +.. code:: python + + ambient_temp module_temp irradiation color + 2020-05-15 12:00:00 31.31 49.18 0.51 green + 2020-05-15 12:15:00 31.51 49.84 0.79 green + 2020-05-15 12:30:00 32.15 52.35 0.65 green + 2020-05-15 12:45:00 32.39 50.63 0.76 green + 2020-05-15 13:00:00 32.62 49.61 0.42 blue + +We can extract features from the index as follows: + +.. code:: python + + from feature_engine.datetime import DatetimeFeatures + + dtf = DatetimeFeatures(variables="index") + + Xtr = dtf.fit_transform(X) + + Xtr + +We can see that the transformer created the default time features and added them at +the end of the dataframe. + +.. code:: python + + ambient_temp module_temp irradiation color month \ + 2020-05-15 12:00:00 31.31 49.18 0.51 green 5 + 2020-05-15 12:15:00 31.51 49.84 0.79 green 5 + 2020-05-15 12:30:00 32.15 52.35 0.65 green 5 + 2020-05-15 12:45:00 32.39 50.63 0.76 green 5 + 2020-05-15 13:00:00 32.62 49.61 0.42 blue 5 + 2020-05-15 13:15:00 32.50 47.01 0.49 blue 5 + 2020-05-15 13:30:00 32.52 46.67 0.57 blue 5 + 2020-05-15 13:45:00 32.68 47.52 0.56 blue 5 + + year day_of_week day_of_month hour minute second + 2020-05-15 12:00:00 2020 4 15 12 0 0 + 2020-05-15 12:15:00 2020 4 15 12 15 0 + 2020-05-15 12:30:00 2020 4 15 12 30 0 + 2020-05-15 12:45:00 2020 4 15 12 45 0 + 2020-05-15 13:00:00 2020 4 15 13 0 0 + 2020-05-15 13:15:00 2020 4 15 13 15 0 + 2020-05-15 13:30:00 2020 4 15 13 30 0 + 2020-05-15 13:45:00 2020 4 15 13 45 0 + +We can obtain the name of all the variables in the output dataframe as follows: + +.. code:: python + + dtf.get_feature_names_out() + +.. code:: python + + ['ambient_temp', + 'module_temp', + 'irradiation', + 'color', + 'month', + 'year', + 'day_of_week', + 'day_of_month', + 'hour', + 'minute', + 'second'] + + +Important +--------- + +We highly recommend specifying the date and time features that you would like to extract +from your datetime variables. + +If you have too many time variables, this might not be possible. In this case, keep in +mind that if you extract date features from variables that have only time, or time features +from variables that have only dates, your features will be meaningless. + +Let's explore the outcome with an example. We create a dataset with only time variables. + +.. code:: python + + import pandas as pd + from feature_engine.datetime import DatetimeFeatures + + toy_df = pd.DataFrame({ + "not_a_dt": ['not', 'a', 'date', 'time'], + "var_time1": ['12:34:45', '23:01:02', '11:59:21', '08:44:23'], + "var_time2": ['02:27:26', '10:10:55', '17:30:00', '18:11:18'], + }) + +And now we mistakenly extract only date features: + +.. code:: python + + dfts = DatetimeFeatures( + features_to_extract=["year", "month", "day_of_week"], + ) + df_transf = dfts.fit_transform(toy_df) + + print(df_transf) + +.. code:: python + + not_a_dt var_time1_year var_time1_month var_time1_day_of_week var_time2_year \ + 0 not 2021 12 2 2021 + 1 a 2021 12 2 2021 + 2 date 2021 12 2 2021 + 3 time 2021 12 2 2021 + + var_time2_month var_time2_day_of_week + 0 12 2 + 1 12 2 + 2 12 2 + 3 12 2 + +The transformer will still create features derived from today's date (the date of +creating the docs). + +If instead we have a dataframe with only date variables: + +.. code:: python + + import pandas as pd + from feature_engine.datetime import DatetimeFeatures + + toy_df = pd.DataFrame({ + "var_date1": ['May-1989', 'Dec-2020', 'Jan-1999', 'Feb-2002'], + "var_date2": ['06/21/12', '02/10/98', '08/03/10', '10/31/20'], + }) + +And we mistakenly extract the hour and the minute: + +.. code:: python + + dfts = DatetimeFeatures( + features_to_extract=["hour", "minute"], + ) + df_transf = dfts.fit_transform(toy_df) + + print(df_transf) + +.. code:: python + + var_date1_hour var_date1_minute var_date2_hour var_date2_minute + 0 0 0 0 0 + 1 0 0 0 0 + 2 0 0 0 0 + 3 0 0 0 0 + +The new features will contain the value 0. + +Automating feature extraction +----------------------------- + +We can indicate which features we want to extract from the datetime variables as we did +in the previous examples, by passing the feature names in lists. + +Alternatively, :class:`DatetimeFeatures()` has default options to extract a group of +commonly used features, or all supported features. + +Let's first create a toy dataframe: + +.. code:: python + + import pandas as pd + from feature_engine.datetime import DatetimeFeatures + + toy_df = pd.DataFrame({ + "var_dt1": pd.date_range("2018-01-01", periods=3, freq="H"), + "var_dt2": ['08/31/00 12:34:45', '12/01/90 23:01:02', '04/25/01 11:59:21'], + "var_dt3": ['03/02/15 02:27:26', '02/28/97 10:10:55', '11/11/03 17:30:00'], + }) + +Most common features +~~~~~~~~~~~~~~~~~~~~ + +Now, we will extract the **most common** date and time features from one of the variables. +To do this, we leave the parameter `features_to_extract` to `None`. + +.. code:: python + + dfts = DatetimeFeatures( + variables=["var_dt1"], + features_to_extract=None, + drop_original=False, + ) + + df_transf = dfts.fit_transform(toy_df) + + print(df_transf) + +.. code:: python + + var_dt1 var_dt2 var_dt3 var_dt1_month \ + 0 2018-01-01 00:00:00 08/31/00 12:34:45 03/02/15 02:27:26 1 + 1 2018-01-01 01:00:00 12/01/90 23:01:02 02/28/97 10:10:55 1 + 2 2018-01-01 02:00:00 04/25/01 11:59:21 11/11/03 17:30:00 1 + + var_dt1_year var_dt1_day_of_week var_dt1_day_of_month var_dt1_hour \ + 0 2018 0 1 0 + 1 2018 0 1 + 2 2018 0 1 2 + + var_dt1_minute var_dt1_second + 0 0 0 + 1 0 0 + 2 0 0 + +Our new dataset contains the original features plus the new variables extracted +from them. + +We can find the group of features extracted by the transformer in its attribute: + +.. code:: python + + dfts.features_to_extract_ + +.. code:: python + + ['month', + 'year', + 'day_of_week', + 'day_of_month', + 'hour', + 'minute', + 'second'] + +All supported features +~~~~~~~~~~~~~~~~~~~~~~ + +We can also extract all supported features automatically, by setting the parameter +`features_to_extract` to `"all"`: + +.. code:: python + + dfts = DatetimeFeatures( + variables=["var_dt1"], + features_to_extract='all', + drop_original=False, + ) + + df_transf = dfts.fit_transform(toy_df) + + print(df_transf) + +.. code:: python + + var_dt1 var_dt2 var_dt3 var_dt1_month \ + 0 2018-01-01 00:00:00 08/31/00 12:34:45 03/02/15 02:27:26 1 + 1 2018-01-01 01:00:00 12/01/90 23:01:02 02/28/97 10:10:55 1 + 2 2018-01-01 02:00:00 04/25/01 11:59:21 11/11/03 17:30:00 1 + + var_dt1_quarter var_dt1_semester var_dt1_year \ + 0 1 1 2018 + 1 1 1 2018 + 2 1 1 2018 + + var_dt1_week var_dt1_day_of_week ... var_dt1_month_end var_dt1_quarter_start \ + 0 1 0 ... 0 1 + 1 1 0 ... 0 1 + 2 1 0 ... 0 1 + + var_dt1_quarter_end var_dt1_year_start var_dt1_year_end \ + 0 0 1 0 + 1 0 1 0 + 2 0 1 0 + + var_dt1_leap_year var_dt1_days_in_month var_dt1_hour var_dt1_minute \ + 0 0 31 0 0 + 1 0 31 1 0 + 2 0 31 2 0 + + var_dt1_second + 0 0 + 1 0 + 2 0 + +We can find the group of features extracted by the transformer in its attribute: + +.. code:: python + + dfts.features_to_extract_ + +.. code:: python + + ['month', + 'quarter', + 'semester', + 'year', + 'week', + 'day_of_week', + 'day_of_month', + 'day_of_year', + 'weekend', + 'month_start', + 'month_end', + 'quarter_start', + 'quarter_end', + 'year_start', + 'year_end', + 'leap_year', + 'days_in_month', + 'hour', + 'minute', + 'second'] + +Extract and select features automatically +----------------------------------------- + +If we have a dataframe with date variables, time variables and date and time variables, +we can extract all features, or the most common features from all the variables, and then +go ahead and remove the irrelevant features with the `DropConstantFeatures()` class. + +Let's create a dataframe with a mix of datetime variables. + +.. code:: python + + import pandas as pd + from sklearn.pipeline import Pipeline + from feature_engine.datetime import DatetimeFeatures + from feature_engine.selection import DropConstantFeatures + + toy_df = pd.DataFrame({ + "var_date": ['06/21/12', '02/10/98', '08/03/10', '10/31/20'], + "var_time1": ['12:34:45', '23:01:02', '11:59:21', '08:44:23'], + "var_dt": ['08/31/00 12:34:45', '12/01/90 23:01:02', '04/25/01 11:59:21', '04/25/01 11:59:21'], + }) + +Now, we line up in a Scikit-learn pipeline the :class:`DatetimeFeatures` and the +`DropConstantFeatures()`. The :class:`DatetimeFeatures` will create date features +derived from today for the time variable, and time features with the value 0 for the +date only variable. `DropConstantFeatures()` will identify and remove these features +from the dataset. + +.. code:: python + + pipe = Pipeline([ + ('datetime', DatetimeFeatures()), + ('drop_constant', DropConstantFeatures()), + ]) + + pipe.fit(toy_df) + +.. code:: python + + Pipeline(steps=[('datetime', DatetimeFeatures()), + ('drop_constant', DropConstantFeatures())]) + +.. code:: python + + df_transf = pipe.transform(toy_df) + + print(df_transf) + +.. code:: python + + var_date_month var_date_year var_date_day_of_week var_date_day_of_month \ + 0 6 2012 3 21 + 1 2 1998 1 10 + 2 8 2010 1 3 + 3 10 2020 5 31 + + var_time1_hour var_time1_minute var_time1_second var_dt_month \ + 0 12 34 45 8 + 1 23 1 2 12 + 2 11 59 21 4 + 3 8 44 23 4 + + var_dt_year var_dt_day_of_week var_dt_day_of_month var_dt_hour \ + 0 2000 3 31 12 + 1 1990 5 1 23 + 2 2001 2 25 11 + 3 2001 2 25 11 + + var_dt_minute var_dt_second + 0 34 45 + 1 1 2 + 2 59 21 + 3 59 21 + +As you can see, we do not have the constant features in the transformed dataset. + +Working with different timezones +-------------------------------- + +Time-aware datetime variables can be particularly cumbersome to work with as far +as the format goes. We will briefly show how :class:`DatetimeFeatures()` deals +with such variables in three different scenarios. + +**Case 1**: our dataset contains a time-aware variable in object format, +with potentially different timezones across different observations. +We pass `utc=True` when initializing the transformer to make sure it +converts all data to UTC timezone. + +.. code:: python + + import pandas as pd + from feature_engine.datetime import DatetimeFeatures + + toy_df = pd.DataFrame({ + "var_tz": ['12:34:45+3', '23:01:02-6', '11:59:21-8', '08:44:23Z'] + }) + + dfts = DatetimeFeatures( + features_to_extract=["hour", "minute"], + drop_original=False, + utc=True + ) + + df_transf = dfts.fit_transform(toy_df) + + print(df_transf) + +.. code:: python + + var_tz var_tz_hour var_tz_minute + 0 12:34:45+3 9 34 + 1 23:01:02-6 5 1 + 2 11:59:21-8 19 59 + 3 08:44:23Z 8 44 + + +**Case 2**: our dataset contains a variable that is cast as a localized +datetime in a particular timezone. However, we decide that we want to get all +the datetime information extracted as if it were in UTC timezone. + +.. code:: python + + import pandas as pd + from feature_engine.datetime import DatetimeFeatures + + var_tz = pd.Series(['08/31/00 12:34:45', '12/01/90 23:01:02', '04/25/01 11:59:21']) + var_tz = pd.to_datetime(var_tz) + var_tz = var_tz.dt.tz_localize("US/eastern") + var_tz + +.. code:: python + + 0 2000-08-31 12:34:45-04:00 + 1 1990-12-01 23:01:02-05:00 + 2 2001-04-25 11:59:21-04:00 + dtype: datetime64[ns, US/Eastern] + +We need to pass `utc=True` when initializing the transformer to revert back to the UTC +timezone. + +.. code:: python + + toy_df = pd.DataFrame({"var_tz": var_tz}) + + dfts = DatetimeFeatures( + features_to_extract=["day_of_month", "hour"], + drop_original=False, + utc=True, + ) + + df_transf = dfts.fit_transform(toy_df) + + print(df_transf) + +.. code:: python + + var_tz var_tz_day_of_month var_tz_hour + 0 2000-08-31 12:34:45-04:00 31 16 + 1 1990-12-01 23:01:02-05:00 2 4 + 2 2001-04-25 11:59:21-04:00 25 15 + + +**Case 3**: given a variable like *var_tz* in the example above, we now want +to extract the features keeping the original timezone localization, +therefore we pass `utc=False` or `None`. In this case, we leave it to `None` which +is the default option. + +.. code:: python + + dfts = DatetimeFeatures( + features_to_extract=["day_of_month", "hour"], + drop_original=False, + utc=None, + ) + + df_transf = dfts.fit_transform(toy_df) + + print(df_transf) + +.. code:: python + + var_tz var_tz_day_of_month var_tz_hour + 0 2000-08-31 12:34:45-04:00 31 12 + 1 1990-12-01 23:01:02-05:00 1 23 + 2 2001-04-25 11:59:21-04:00 25 11 + +Note that the hour extracted from the variable differ in this dataframe respect to the +one obtained in **Case 2**. + +Missing timestamps +------------------ + +:class:`DatetimeFeatures` has the option to ignore missing timestamps, or raise an error +when a missing value is encountered in a datetime variable. + + +Additional resources +-------------------- + +You can find an example of how to use :class:`DatetimeFeatures()` with a real dataset in +the following `Jupyter notebook `_ + +For tutorials on how to create and use features from datetime columns, check the following courses: + +- `Feature Engineering for Machine Learning `_. +- `Feature Engineering for Time Series Forecasting `_. \ No newline at end of file diff --git a/docs/user_guide/datetime/index.rst b/docs/user_guide/datetime/index.rst index f4976c68d..c68608fa2 100644 --- a/docs/user_guide/datetime/index.rst +++ b/docs/user_guide/datetime/index.rst @@ -9,4 +9,5 @@ features from existing datetime or object-like data. .. toctree:: :maxdepth: 1 - DatetimeFeatures \ No newline at end of file + DatetimeFeatures + DatetimeSubtraction \ No newline at end of file From 70523b87ffc9d4542fe7d6e7f6cbac5e23ef6a75 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sat, 11 Mar 2023 22:25:34 +0100 Subject: [PATCH 13/19] initial tests --- .../datetime/datetime_subtraction.py | 70 +-- .../test_datetime_subtraction.py | 434 +++--------------- 2 files changed, 114 insertions(+), 390 deletions(-) diff --git a/feature_engine/datetime/datetime_subtraction.py b/feature_engine/datetime/datetime_subtraction.py index f56d72da5..6d5954362 100644 --- a/feature_engine/datetime/datetime_subtraction.py +++ b/feature_engine/datetime/datetime_subtraction.py @@ -3,24 +3,8 @@ import numpy as np import pandas as pd from pandas.api.types import is_datetime64_any_dtype as is_datetime - from sklearn.utils.validation import check_is_fitted -from feature_engine.variable_handling._init_parameter_checks import ( - _check_init_parameter_variables, -) -from feature_engine.creation.base_creation import BaseCreation -from feature_engine.tags import _return_tags -from feature_engine.dataframe_checks import ( - _check_contains_na, - _check_X_matches_training_df, - check_X, -) -from feature_engine._docstrings.methods import ( - _fit_not_learn_docstring, - _fit_transform_docstring, - _transform_creation_docstring, -) from feature_engine._docstrings.fit_attributes import ( _feature_names_in_docstring, _n_features_in_docstring, @@ -29,9 +13,28 @@ _drop_original_docstring, _missing_values_docstring, ) - +from feature_engine._docstrings.methods import ( + _fit_not_learn_docstring, + _fit_transform_docstring, + _transform_creation_docstring, +) from feature_engine._docstrings.substitute import Substitution +from feature_engine.creation.base_creation import BaseCreation +from feature_engine.dataframe_checks import ( + _check_contains_na, + _check_X_matches_training_df, + check_X, +) from feature_engine.variable_handling import find_or_check_datetime_variables +from feature_engine.variable_handling._init_parameter_checks import ( + _check_init_parameter_variables, +) + +_demo_df = """ + >>> X = pd.DataFrame({ + >>> "date1": ["2022-09-18", "2022-10-27", "2022-12-24"], + >>> "date2": ["2022-08-18", "2022-08-27", "2022-06-24"]}) + """.rstrip() @Substitution( @@ -42,6 +45,7 @@ fit=_fit_not_learn_docstring, transform=_transform_creation_docstring, fit_transform=_fit_transform_docstring, + demo_df=_demo_df, ) class DatetimeSubtraction(BaseCreation): """ @@ -49,6 +53,10 @@ class DatetimeSubtraction(BaseCreation): variables and one or more datetime features, adding the resulting variables to the dataframe. + DatetimeSubtraction() works with variables cast as datetime or object. It subtracts + the variables listed in the parameter `reference` from those listed in the + parameter `variables`. + More details in the :ref:`User Guide `. Parameters @@ -109,9 +117,7 @@ class DatetimeSubtraction(BaseCreation): >>> import pandas as pd >>> from feature_engine.datetime import DatetimeSubtraction - >>> X = pd.DataFrame({ - >>> "date1" : ["2022-09-18", "2022-10-27", "2022-12-24"], - >>> "date2" : ["2022-08-18", "2022-08-27", "2022-06-24"]}) + {demo_df} >>> dtf = DatetimeSubtraction(variables=["date1"], reference=["date2"]) >>> dtf.fit(X) >>> dtf.transform(X) @@ -120,6 +126,7 @@ class DatetimeSubtraction(BaseCreation): 1 2022-10-27 2022-08-27 61.0 2 2022-12-24 2022-06-24 183.0 """ + def __init__( self, variables: Union[None, int, str, List[Union[str, int]]], @@ -133,10 +140,23 @@ def __init__( ) -> None: valid_output_units = { - "D", "Y", "M", "W", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as", + "D", + "Y", + "M", + "W", + "h", + "m", + "s", + "ms", + "us", + "μs", + "ns", + "ps", + "fs", + "as", } - if output_unit not in valid_output_units: + if not isinstance(output_unit, str) or output_unit not in valid_output_units: raise ValueError( f"output_unit accepts the following values: " f"{valid_output_units}. Got {output_unit} instead." @@ -236,14 +256,12 @@ def _to_datetime(self, X: pd.DataFrame): yearfirst=self.yearfirst, utc=self.utc, ) - for variable in set(self.variables_+self.reference_) + for variable in set(self.variables_ + self.reference_) ], axis=1, ) - non_dt_columns = datetime_df.columns[ - ~datetime_df.apply(is_datetime) - ].tolist() + non_dt_columns = datetime_df.columns[~datetime_df.apply(is_datetime)].tolist() if non_dt_columns: raise ValueError( diff --git a/tests/test_datetime/test_datetime_subtraction.py b/tests/test_datetime/test_datetime_subtraction.py index ff4378d83..db7f6f9d3 100644 --- a/tests/test_datetime/test_datetime_subtraction.py +++ b/tests/test_datetime/test_datetime_subtraction.py @@ -4,399 +4,105 @@ from sklearn.pipeline import Pipeline -from feature_engine.creation import RelativeFeatures - - -def test_mandatory_init_parameters(): - with pytest.raises(TypeError): - RelativeFeatures(reference=["var1"], func=["add"]) - with pytest.raises(TypeError): - RelativeFeatures(variables=["var1"], func=["add"]) - with pytest.raises(TypeError): - RelativeFeatures(variables=["var1"], reference=["var2"]) - - -_variables = ["var1", ["var1", "var1", "var2"], ["var1", 0.5], ("Age", "Name")] - - -@pytest.mark.parametrize("_variables", _variables) -def test_error_when_param_variables_not_permitted(_variables): +from feature_engine.datetime import DatetimeSubtraction + + +@pytest.mark.parametrize( + "_input_vars", + [ + ("var1", "var2"), + {"var1": 1, "var2": 2}, + ["var1", "var2", "var2", "var3"], + [0, 1, 1, 2], + ], +) +def test_init_parameters_variables_and_reference_raises_errors(_input_vars): with pytest.raises(ValueError): - RelativeFeatures( - variables=_variables, reference=["Age", "Name"], func=["add", "mul"] - ) - - -@pytest.mark.parametrize("_variables", _variables) -def test_error_when_param_reference_not_permitted(_variables): - with pytest.raises(ValueError): - RelativeFeatures( - reference=_variables, variables=["Age", "Name"], func=["add", "mul"] - ) - - -_operations = [ - "add", - ["add", "add", "mul"], - ["add", "multiply"], - ("add", "mul"), - [np.mean, "add"], -] - - -@pytest.mark.parametrize("_func", _operations) -def test_error_if_func_not_supported(_func): + assert DatetimeSubtraction(variables=_input_vars, reference=["var1"]) with pytest.raises(ValueError): - RelativeFeatures( - variables=["Age", "Name"], - reference=["Age", "Name"], - func=_func, - ) + assert DatetimeSubtraction(reference=_input_vars, variables=["var1"]) -def test_error_when_drop_original_not_bool(): - for drop_original in ["True", [True]]: - with pytest.raises(TypeError): - RelativeFeatures( - variables=["Age"], - reference=["Marks"], - func=["add", "mul"], - drop_original=drop_original, - ) +@pytest.mark.parametrize("_input_vars", ["var1", ["var1"], ["var1", "var2"]]) +def test_init_parameters_variables_and_reference(_input_vars): + transformer = DatetimeSubtraction(variables=_input_vars, reference=_input_vars) + assert transformer.variables == _input_vars + assert transformer.reference == _input_vars -def test_error_when_variables_not_numeric(df_vartypes): - transformer = RelativeFeatures( - variables=["Name", "Age", "Marks"], - reference=["Age", "Name"], - func=["sub"], - ) +@pytest.mark.parametrize("_input_vars", ["var1", ["var1"], ["var1", "var2"]]) +def test_mandatory_init_parameters(_input_vars): with pytest.raises(TypeError): - transformer.fit_transform(df_vartypes) - - transformer = RelativeFeatures( - reference=["Name", "Age", "Marks"], - variables=["Age", "Name"], - func=["sub"], - ) + DatetimeSubtraction(reference=["var1"]) with pytest.raises(TypeError): - transformer.fit_transform(df_vartypes) - - -def test_error_when_entered_variables_not_in_df(df_vartypes): - transformer = RelativeFeatures( - variables=["FeatOutsideDataset", "Age"], - reference=["Age", "Name"], - func=["sub"], - ) - with pytest.raises(KeyError): - transformer.fit_transform(df_vartypes) - - transformer = RelativeFeatures( - reference=["FeatOutsideDataset", "Age"], - variables=["Age", "Name"], - func=["sub"], - ) - with pytest.raises(TypeError): - transformer.fit_transform(df_vartypes) - - -def test_classic_binary_operation(df_vartypes): - - transformer = RelativeFeatures( - variables=["Age"], - reference=["Marks"], - func=["sub", "div", "add", "mul"], - ) - - X = transformer.fit_transform(df_vartypes) - - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, 21, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": pd.date_range("2020-02-24", periods=4, freq="T"), - "Age_sub_Marks": [19.1, 20.2, 18.3, 17.4], - "Age_div_Marks": [22.22222222222222, 26.25, 27.142857142857146, 30.0], - "Age_add_Marks": [20.9, 21.8, 19.7, 18.6], - "Age_mul_Marks": [18.0, 16.8, 13.299999999999999, 10.799999999999999], - } - ) - - pd.testing.assert_frame_equal(X, ref) - - -def test_alternative_operation(df_vartypes): - - # input df - df = df_vartypes.copy() - - # Expected result - dft = df.copy() - dft["Age_truediv_Marks"] = dft["Age"].truediv(dft["Marks"]) - dft["Age_floordiv_Marks"] = dft["Age"].floordiv(dft["Marks"]) - dft["Age_mod_Marks"] = dft["Age"].mod(dft["Marks"]) - dft["Age_pow_Marks"] = dft["Age"].pow(dft["Marks"]) - - transformer = RelativeFeatures( - variables=["Age"], - reference=["Marks"], - func=["truediv", "floordiv", "mod", "pow"], - ) - X = transformer.fit_transform(df) - - pd.testing.assert_frame_equal(X, dft) + DatetimeSubtraction(variables=["var1"]) -def test_operations_with_multiple_variables(df_vartypes): - transformer = RelativeFeatures( - variables=["Age", "Marks"], - reference=["Age", "Marks"], - func=["sub"], - ) +@pytest.mark.parametrize("output", [ "D", "Y", "M", "W", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as",]) +def test_valid_output_unit_param(output): + transformer = DatetimeSubtraction(variables=["var1"], reference=["var1"], output_unit=output) + assert transformer.output_unit == output - X = transformer.fit_transform(df_vartypes) - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, 21, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": pd.date_range("2020-02-24", periods=4, freq="T"), - "Age_sub_Age": [0, 0, 0, 0], - "Marks_sub_Age": [-19.1, -20.2, -18.3, -17.4], - "Age_sub_Marks": [19.1, 20.2, 18.3, 17.4], - "Marks_sub_Marks": [0.0, 0.0, 0.0, 0.0], - } - ) - - pd.testing.assert_frame_equal(X, ref) - - -def test_multiple_operations_with_multiple_variables(df_vartypes): - transformer = RelativeFeatures( - variables=["Age", "Marks"], - reference=["Age", "Marks"], - func=["sub", "add"], - ) - - X = transformer.fit_transform(df_vartypes) - - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, 21, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": pd.date_range("2020-02-24", periods=4, freq="T"), - "Age_sub_Age": [0, 0, 0, 0], - "Marks_sub_Age": [-19.1, -20.2, -18.3, -17.4], - "Age_sub_Marks": [19.1, 20.2, 18.3, 17.4], - "Marks_sub_Marks": [0.0, 0.0, 0.0, 0.0], - "Age_add_Age": [40, 42, 38, 36], - "Marks_add_Age": [20.9, 21.8, 19.7, 18.6], - "Age_add_Marks": [20.9, 21.8, 19.7, 18.6], - "Marks_add_Marks": [1.8, 1.6, 1.4, 1.2], - } - ) - - pd.testing.assert_frame_equal(X, ref) - - transformer = RelativeFeatures( - variables=["Age", "Marks"], - reference=["Age", "Marks"], - func=["add", "sub"], - ) - - X = transformer.fit_transform(df_vartypes) - - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, 21, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": pd.date_range("2020-02-24", periods=4, freq="T"), - "Age_add_Age": [40, 42, 38, 36], - "Marks_add_Age": [20.9, 21.8, 19.7, 18.6], - "Age_add_Marks": [20.9, 21.8, 19.7, 18.6], - "Marks_add_Marks": [1.8, 1.6, 1.4, 1.2], - "Age_sub_Age": [0, 0, 0, 0], - "Marks_sub_Age": [-19.1, -20.2, -18.3, -17.4], - "Age_sub_Marks": [19.1, 20.2, 18.3, 17.4], - "Marks_sub_Marks": [0.0, 0.0, 0.0, 0.0], - } - ) - - pd.testing.assert_frame_equal(X, ref) - - -def test_when_missing_values_is_ignore(df_vartypes): - - df_na = df_vartypes.copy() - df_na.loc[1, "Age"] = np.nan +@pytest.mark.parametrize("output", [ ["D"], "J", True, 1, 1.5]) +def test_output_unit_raises_error_when_not_valid(output): + with pytest.raises(ValueError): + DatetimeSubtraction(variables=["var1"], reference=["var1"], output_unit=output) - transformer = RelativeFeatures( - variables=["Age", "Marks"], - reference=["Age", "Marks"], - func=["sub"], - missing_values="ignore", - ) - X = transformer.fit_transform(df_na) +@pytest.mark.parametrize("output", [ ["D"], "J", True, 1, 1.5]) +def test_output_unit_raises_error_when_not_valid(output): + with pytest.raises(ValueError): + DatetimeSubtraction(variables=["var1"], reference=["var1"], output_unit=output) - ref = pd.DataFrame.from_dict( - { - "Name": ["tom", "nick", "krish", "jack"], - "City": ["London", "Manchester", "Liverpool", "Bristol"], - "Age": [20, np.nan, 19, 18], - "Marks": [0.9, 0.8, 0.7, 0.6], - "dob": pd.date_range("2020-02-24", periods=4, freq="T"), - "Age_sub_Age": [0, np.nan, 0, 0], - "Marks_sub_Age": [-19.1, np.nan, -18.3, -17.4], - "Age_sub_Marks": [19.1, np.nan, 18.3, 17.4], - "Marks_sub_Marks": [0.0, 0.0, 0.0, 0.0], - } - ) - pd.testing.assert_frame_equal(X, ref) +def test_raises_error_when_variables_not_datetime(df_datetime): + with pytest.raises(TypeError): + DatetimeSubtraction(variables="Age", reference="date_obj1").fit(df_datetime) + with pytest.raises(TypeError): + DatetimeSubtraction(variables=["date_obj1"], reference=["Age"]).fit(df_datetime) -def test_error_when_null_values_in_variable(df_vartypes): +def test_sets_variables_if_datetime(df_datetime): + tr = DatetimeSubtraction(variables="date_obj1", reference="date_obj1").fit(df_datetime) + assert tr.variables_ == ["date_obj1"] + assert tr.reference_ == ["date_obj1"] - df_na = df_vartypes.copy() - df_na.loc[1, "Age"] = np.nan - transformer = RelativeFeatures( - variables=["Age", "Marks"], - reference=["Age", "Marks"], - func=["add", "mul"], - missing_values="raise", - ) +def test_raises_error_when_nan_in_fit(): + df = pd.DataFrame({ + "dates_na": ["Feb-2010", np.nan, "Jun-1922", np.nan], + "dates_full":["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], + }) + tr = DatetimeSubtraction(variables="dates_na", reference="dates_full", missing_values="raise") with pytest.raises(ValueError): - transformer.fit(df_na) + tr.fit(df) - transformer.fit(df_vartypes) + tr = DatetimeSubtraction(variables="dates_full", reference="dates_na", missing_values="raise") with pytest.raises(ValueError): - transformer.transform(df_na) + tr.fit(df) -def test_when_df_cols_are_integers(df_vartypes): - df = df_vartypes.copy() - df.columns = [0, 1, 2, 3, 4] +def test_raises_error_when_nan_in_transform(): + df_fit = pd.DataFrame({ + "dates_na": ["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], + "dates_full":["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], + }) + df_transform = pd.DataFrame({ + "dates_na": ["Feb-2010", np.nan, "Jun-1922", np.nan], + "dates_full":["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], + }) - transformer = RelativeFeatures( - variables=[2, 3], - reference=[2, 3], - func=["sub", "add"], - ) - - X = transformer.fit_transform(df) - - ref = pd.DataFrame.from_dict( - { - 0: ["tom", "nick", "krish", "jack"], - 1: ["London", "Manchester", "Liverpool", "Bristol"], - 2: [20, 21, 19, 18], - 3: [0.9, 0.8, 0.7, 0.6], - 4: pd.date_range("2020-02-24", periods=4, freq="T"), - "2_sub_2": [0, 0, 0, 0], - "3_sub_2": [-19.1, -20.2, -18.3, -17.4], - "2_sub_3": [19.1, 20.2, 18.3, 17.4], - "3_sub_3": [0.0, 0.0, 0.0, 0.0], - "2_add_2": [40, 42, 38, 36], - "3_add_2": [20.9, 21.8, 19.7, 18.6], - "2_add_3": [20.9, 21.8, 19.7, 18.6], - "3_add_3": [1.8, 1.6, 1.4, 1.2], - } - ) - - pd.testing.assert_frame_equal(X, ref) - - -@pytest.mark.parametrize("_func", [["div"], ["truediv"], ["floordiv"], ["mod"]]) -def test_error_when_division_by_zero(_func, df_vartypes): - - df_zero = df_vartypes.copy() - df_zero.loc[1, "Marks"] = 0 - - transformer = RelativeFeatures( - variables=["Age"], - reference=["Marks"], - func=_func, - ) - transformer.fit(df_vartypes) + tr = DatetimeSubtraction(variables="dates_na", reference="dates_full", missing_values="raise") + tr.fit(df_fit) with pytest.raises(ValueError): - transformer.transform(df_zero) - + tr.fit(df_transform) -@pytest.mark.parametrize("_drop", [True, False]) -def test_get_feature_names_out(_drop, df_vartypes): - transformer = RelativeFeatures( - variables=["Age", "Marks"], - reference=["Age", "Marks"], - func=["add", "sub"], - drop_original=_drop, - ) - varnames = [ - "Age_add_Age", - "Marks_add_Age", - "Age_add_Marks", - "Marks_add_Marks", - "Age_sub_Age", - "Marks_sub_Age", - "Age_sub_Marks", - "Marks_sub_Marks", - ] - - X = transformer.fit_transform(df_vartypes) - assert list(X.columns) == transformer.get_feature_names_out(input_features=None) - assert list(X.columns) == transformer.get_feature_names_out(input_features=False) - assert varnames == transformer.get_feature_names_out(input_features=True) - - -@pytest.mark.parametrize("_drop", [True, False]) -def test_get_feature_names_out_from_pipeline(_drop, df_vartypes): - transformer = RelativeFeatures( - variables=["Age", "Marks"], - reference=["Age", "Marks"], - func=["add", "sub"], - drop_original=_drop, - ) - - pipe = Pipeline([("transformer", transformer)]) - - varnames = [ - "Age_add_Age", - "Marks_add_Age", - "Age_add_Marks", - "Marks_add_Marks", - "Age_sub_Age", - "Marks_sub_Age", - "Age_sub_Marks", - "Marks_sub_Marks", - ] - - X = pipe.fit_transform(df_vartypes) - assert list(X.columns) == pipe.get_feature_names_out(input_features=None) - assert list(X.columns) == pipe.get_feature_names_out(input_features=False) - assert varnames == pipe.get_feature_names_out(input_features=True) + tr = DatetimeSubtraction(variables="dates_full", reference="dates_na", missing_values="raise") + tr.fit(df_fit) + with pytest.raises(ValueError): + tr.fit(df_transform) -@pytest.mark.parametrize("_input_features", ["hola", ["Age", "Marks"]]) -def test_get_feature_names_out_raises_error_when_wrong_param( - _input_features, df_vartypes -): - transformer = RelativeFeatures( - variables=["Age", "Marks"], - reference=["Age", "Marks"], - func=["add", "sub"], - ) - transformer.fit(df_vartypes) - with pytest.raises(ValueError): - transformer.get_feature_names_out(input_features=_input_features) From 1a3471123214b8a65bc32a55f1bd53eca2773e1d Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sat, 11 Mar 2023 22:59:43 +0100 Subject: [PATCH 14/19] adds majority of tests --- .../test_check_estimator_datetime.py | 2 +- .../test_datetime_subtraction.py | 171 +++++++++++++++--- 2 files changed, 148 insertions(+), 25 deletions(-) diff --git a/tests/test_datetime/test_check_estimator_datetime.py b/tests/test_datetime/test_check_estimator_datetime.py index 4b081e946..84be35612 100644 --- a/tests/test_datetime/test_check_estimator_datetime.py +++ b/tests/test_datetime/test_check_estimator_datetime.py @@ -3,7 +3,7 @@ from feature_engine.datetime import DatetimeFeatures, DatetimeSubtraction from tests.estimator_checks.estimator_checks import check_feature_engine_estimator -_estimators = [DatetimeFeatures(), DatetimeSubtraction()] +_estimators = [DatetimeFeatures()]#, DatetimeSubtraction(variables=["var_1", "var_2"], reference=["var_3"])] @pytest.mark.parametrize("estimator", _estimators) diff --git a/tests/test_datetime/test_datetime_subtraction.py b/tests/test_datetime/test_datetime_subtraction.py index db7f6f9d3..654603679 100644 --- a/tests/test_datetime/test_datetime_subtraction.py +++ b/tests/test_datetime/test_datetime_subtraction.py @@ -1,7 +1,6 @@ import numpy as np import pandas as pd import pytest - from sklearn.pipeline import Pipeline from feature_engine.datetime import DatetimeSubtraction @@ -38,19 +37,39 @@ def test_mandatory_init_parameters(_input_vars): DatetimeSubtraction(variables=["var1"]) -@pytest.mark.parametrize("output", [ "D", "Y", "M", "W", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as",]) +@pytest.mark.parametrize( + "output", + [ + "D", + "Y", + "M", + "W", + "h", + "m", + "s", + "ms", + "us", + "μs", + "ns", + "ps", + "fs", + "as", + ], +) def test_valid_output_unit_param(output): - transformer = DatetimeSubtraction(variables=["var1"], reference=["var1"], output_unit=output) + transformer = DatetimeSubtraction( + variables=["var1"], reference=["var1"], output_unit=output + ) assert transformer.output_unit == output -@pytest.mark.parametrize("output", [ ["D"], "J", True, 1, 1.5]) +@pytest.mark.parametrize("output", [["D"], "J", True, 1, 1.5]) def test_output_unit_raises_error_when_not_valid(output): with pytest.raises(ValueError): DatetimeSubtraction(variables=["var1"], reference=["var1"], output_unit=output) -@pytest.mark.parametrize("output", [ ["D"], "J", True, 1, 1.5]) +@pytest.mark.parametrize("output", [["D"], "J", True, 1, 1.5]) def test_output_unit_raises_error_when_not_valid(output): with pytest.raises(ValueError): DatetimeSubtraction(variables=["var1"], reference=["var1"], output_unit=output) @@ -64,45 +83,149 @@ def test_raises_error_when_variables_not_datetime(df_datetime): def test_sets_variables_if_datetime(df_datetime): - tr = DatetimeSubtraction(variables="date_obj1", reference="date_obj1").fit(df_datetime) + tr = DatetimeSubtraction(variables="date_obj1", reference="date_obj1").fit( + df_datetime + ) assert tr.variables_ == ["date_obj1"] assert tr.reference_ == ["date_obj1"] def test_raises_error_when_nan_in_fit(): - df = pd.DataFrame({ - "dates_na": ["Feb-2010", np.nan, "Jun-1922", np.nan], - "dates_full":["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], - }) - - tr = DatetimeSubtraction(variables="dates_na", reference="dates_full", missing_values="raise") + df = pd.DataFrame( + { + "dates_na": ["Feb-2010", np.nan, "Jun-1922", np.nan], + "dates_full": ["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], + } + ) + + tr = DatetimeSubtraction( + variables="dates_na", reference="dates_full", missing_values="raise" + ) with pytest.raises(ValueError): tr.fit(df) - tr = DatetimeSubtraction(variables="dates_full", reference="dates_na", missing_values="raise") + tr = DatetimeSubtraction( + variables="dates_full", reference="dates_na", missing_values="raise" + ) with pytest.raises(ValueError): tr.fit(df) def test_raises_error_when_nan_in_transform(): - df_fit = pd.DataFrame({ - "dates_na": ["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], - "dates_full":["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], - }) - df_transform = pd.DataFrame({ - "dates_na": ["Feb-2010", np.nan, "Jun-1922", np.nan], - "dates_full":["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], - }) - - tr = DatetimeSubtraction(variables="dates_na", reference="dates_full", missing_values="raise") + df_fit = pd.DataFrame( + { + "dates_na": ["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], + "dates_full": ["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], + } + ) + df_transform = pd.DataFrame( + { + "dates_na": ["Feb-2010", np.nan, "Jun-1922", np.nan], + "dates_full": ["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], + } + ) + + tr = DatetimeSubtraction( + variables="dates_na", reference="dates_full", missing_values="raise" + ) tr.fit(df_fit) with pytest.raises(ValueError): tr.fit(df_transform) - tr = DatetimeSubtraction(variables="dates_full", reference="dates_na", missing_values="raise") + tr = DatetimeSubtraction( + variables="dates_full", reference="dates_na", missing_values="raise" + ) tr.fit(df_fit) with pytest.raises(ValueError): tr.fit(df_transform) +def test_get_feature_names_out(): + df = pd.DataFrame( + { + "d1": ["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], + "d2": ["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], + "d3": ["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], + "d4": ["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], + } + ) + input_vars = df.columns.to_list() + + tr = DatetimeSubtraction(variables="d1", reference="d2") + tr.fit(df) + assert tr.get_feature_names_out() == input_vars + ["d1_sub_d2"] + + tr = DatetimeSubtraction(variables=["d1", "d2"], reference="d3") + tr.fit(df) + assert tr.get_feature_names_out() == input_vars + ["d1_sub_d3", "d2_sub_d3"] + tr = DatetimeSubtraction(variables="d3", reference=["d1", "d2"]) + tr.fit(df) + assert tr.get_feature_names_out() == input_vars + ["d3_sub_d1", "d3_sub_d2"] + + tr = DatetimeSubtraction(variables=["d1", "d2"], reference=["d3", "d4"]) + tr.fit(df) + assert tr.get_feature_names_out() == input_vars + [ + "d1_sub_d3", + "d2_sub_d3", + "d1_sub_d4", + "d2_sub_d4", + ] + + +@pytest.mark.parametrize( + "unit, expected", + [ + ("D", [31, 61, 183]), + ("M", [1.018501, 2.004148, 6.012444]), + ("h", [744.0, 1464.0, 4392.0]), + ], +) +def test_subtraction_units(unit, expected): + df_input = pd.DataFrame( + { + "date1": ["2022-09-18", "2022-10-27", "2022-12-24"], + "date2": ["2022-08-18", "2022-08-27", "2022-06-24"], + } + ) + df_expected = pd.DataFrame( + { + "date1": ["2022-09-18", "2022-10-27", "2022-12-24"], + "date2": ["2022-08-18", "2022-08-27", "2022-06-24"], + "date1_sub_date2": expected, + } + ) + + dtf = DatetimeSubtraction( + variables=["date1"], reference=["date2"], output_unit=unit + ) + df_output = dtf.fit_transform(df_input) + pd.testing.assert_frame_equal(df_output, df_expected, check_dtype=False) + + +def test_multiple_subtractions(): + df_input = pd.DataFrame( + { + "date1": ["2022-09-01", "2022-10-01", "2022-12-01"], + "date2": ["2022-09-15", "2022-10-15", "2022-12-15"], + "date3": ["2022-08-01", "2022-09-01", "2022-11-01"], + "date4": ["2022-08-15", "2022-09-15", "2022-11-15"], + } + ) + df_expected = pd.DataFrame( + { + "date1": ["2022-09-01", "2022-10-01", "2022-12-01"], + "date2": ["2022-09-15", "2022-10-15", "2022-12-15"], + "date3": ["2022-08-01", "2022-09-01", "2022-11-01"], + "date4": ["2022-08-15", "2022-09-15", "2022-11-15"], + "date1_sub_date3": [31, 30, 30], + "date2_sub_date3": [45, 44, 44], + "date1_sub_date4": [17, 16, 16], + "date2_sub_date4": [31, 30, 30], + } + ) + dtf = DatetimeSubtraction( + variables=["date1", "date2"], reference=["date3", "date4"] + ) + df_output = dtf.fit_transform(df_input) + pd.testing.assert_frame_equal(df_output, df_expected, check_dtype=False) From e7ccd4c0f87b1c20358ce1bbd6b8b3636fca8a49 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sun, 12 Mar 2023 07:55:17 +0100 Subject: [PATCH 15/19] add final tests --- .../test_check_estimator_datetime.py | 4 +- .../test_datetime_subtraction.py | 130 ++++++++++++------ 2 files changed, 92 insertions(+), 42 deletions(-) diff --git a/tests/test_datetime/test_check_estimator_datetime.py b/tests/test_datetime/test_check_estimator_datetime.py index 84be35612..6fe6d1986 100644 --- a/tests/test_datetime/test_check_estimator_datetime.py +++ b/tests/test_datetime/test_check_estimator_datetime.py @@ -1,9 +1,9 @@ import pytest -from feature_engine.datetime import DatetimeFeatures, DatetimeSubtraction +from feature_engine.datetime import DatetimeFeatures from tests.estimator_checks.estimator_checks import check_feature_engine_estimator -_estimators = [DatetimeFeatures()]#, DatetimeSubtraction(variables=["var_1", "var_2"], reference=["var_3"])] +_estimators = [DatetimeFeatures()] @pytest.mark.parametrize("estimator", _estimators) diff --git a/tests/test_datetime/test_datetime_subtraction.py b/tests/test_datetime/test_datetime_subtraction.py index 654603679..c490d82a5 100644 --- a/tests/test_datetime/test_datetime_subtraction.py +++ b/tests/test_datetime/test_datetime_subtraction.py @@ -1,9 +1,18 @@ import numpy as np import pandas as pd import pytest -from sklearn.pipeline import Pipeline from feature_engine.datetime import DatetimeSubtraction +from tests.estimator_checks.estimator_checks import ( + check_raises_error_when_input_not_a_df, +) +from tests.estimator_checks.fit_functionality_checks import check_feature_names_in +from tests.estimator_checks.init_params_triggered_functionality_checks import ( + check_drop_original_variables, +) +from tests.estimator_checks.non_fitted_error_checks import check_raises_non_fitted_error + +# ========= init functionality tests @pytest.mark.parametrize( @@ -15,7 +24,7 @@ [0, 1, 1, 2], ], ) -def test_init_parameters_variables_and_reference_raises_errors(_input_vars): +def test_init_parameters_variables_and_reference_raise_error(_input_vars): with pytest.raises(ValueError): assert DatetimeSubtraction(variables=_input_vars, reference=["var1"]) with pytest.raises(ValueError): @@ -23,14 +32,14 @@ def test_init_parameters_variables_and_reference_raises_errors(_input_vars): @pytest.mark.parametrize("_input_vars", ["var1", ["var1"], ["var1", "var2"]]) -def test_init_parameters_variables_and_reference(_input_vars): +def test_init_parameters_variables_and_reference_correct_assignment(_input_vars): transformer = DatetimeSubtraction(variables=_input_vars, reference=_input_vars) assert transformer.variables == _input_vars assert transformer.reference == _input_vars @pytest.mark.parametrize("_input_vars", ["var1", ["var1"], ["var1", "var2"]]) -def test_mandatory_init_parameters(_input_vars): +def test_init_parameters_variables_and_reference_are_mandatory(_input_vars): with pytest.raises(TypeError): DatetimeSubtraction(reference=["var1"]) with pytest.raises(TypeError): @@ -69,12 +78,37 @@ def test_output_unit_raises_error_when_not_valid(output): DatetimeSubtraction(variables=["var1"], reference=["var1"], output_unit=output) -@pytest.mark.parametrize("output", [["D"], "J", True, 1, 1.5]) -def test_output_unit_raises_error_when_not_valid(output): +@pytest.mark.parametrize("param", [True, False]) +def test_drop_original_correct_assignment(param): + transformer = DatetimeSubtraction( + variables=["var1"], reference=["var1"], drop_original=param + ) + assert transformer.drop_original is param + + +@pytest.mark.parametrize("param", [["D"], "J", 10, 1.5]) +def test_drop_original_raises_error_when_not_valid(param): with pytest.raises(ValueError): - DatetimeSubtraction(variables=["var1"], reference=["var1"], output_unit=output) + DatetimeSubtraction(variables=["var1"], reference=["var1"], drop_original=param) +@pytest.mark.parametrize("param", ["ignore", "raise"]) +def test_missing_values_correct_assignment(param): + transformer = DatetimeSubtraction( + variables=["var1"], reference=["var1"], missing_values=param + ) + assert transformer.missing_values is param + + +@pytest.mark.parametrize("param", [["D"], "J", 10, 1.5]) +def test_missing_values_raises_error_when_not_valid(param): + with pytest.raises(ValueError): + DatetimeSubtraction( + variables=["var1"], reference=["var1"], missing_values=param + ) + + +# ==== fit functionality def test_raises_error_when_variables_not_datetime(df_datetime): with pytest.raises(TypeError): DatetimeSubtraction(variables="Age", reference="date_obj1").fit(df_datetime) @@ -111,6 +145,7 @@ def test_raises_error_when_nan_in_fit(): tr.fit(df) +# transform tests def test_raises_error_when_nan_in_transform(): df_fit = pd.DataFrame( { @@ -140,39 +175,6 @@ def test_raises_error_when_nan_in_transform(): tr.fit(df_transform) -def test_get_feature_names_out(): - df = pd.DataFrame( - { - "d1": ["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], - "d2": ["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], - "d3": ["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], - "d4": ["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], - } - ) - input_vars = df.columns.to_list() - - tr = DatetimeSubtraction(variables="d1", reference="d2") - tr.fit(df) - assert tr.get_feature_names_out() == input_vars + ["d1_sub_d2"] - - tr = DatetimeSubtraction(variables=["d1", "d2"], reference="d3") - tr.fit(df) - assert tr.get_feature_names_out() == input_vars + ["d1_sub_d3", "d2_sub_d3"] - - tr = DatetimeSubtraction(variables="d3", reference=["d1", "d2"]) - tr.fit(df) - assert tr.get_feature_names_out() == input_vars + ["d3_sub_d1", "d3_sub_d2"] - - tr = DatetimeSubtraction(variables=["d1", "d2"], reference=["d3", "d4"]) - tr.fit(df) - assert tr.get_feature_names_out() == input_vars + [ - "d1_sub_d3", - "d2_sub_d3", - "d1_sub_d4", - "d2_sub_d4", - ] - - @pytest.mark.parametrize( "unit, expected", [ @@ -229,3 +231,51 @@ def test_multiple_subtractions(): ) df_output = dtf.fit_transform(df_input) pd.testing.assert_frame_equal(df_output, df_expected, check_dtype=False) + + +# additional methods + + +def test_get_feature_names_out(): + df = pd.DataFrame( + { + "d1": ["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], + "d2": ["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], + "d3": ["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], + "d4": ["Feb-2010", "Mar-2010", "Jun-1922", "Feb-2011"], + } + ) + input_vars = df.columns.to_list() + + tr = DatetimeSubtraction(variables="d1", reference="d2") + tr.fit(df) + assert tr.get_feature_names_out() == input_vars + ["d1_sub_d2"] + + tr = DatetimeSubtraction(variables=["d1", "d2"], reference="d3") + tr.fit(df) + assert tr.get_feature_names_out() == input_vars + ["d1_sub_d3", "d2_sub_d3"] + + tr = DatetimeSubtraction(variables="d3", reference=["d1", "d2"]) + tr.fit(df) + assert tr.get_feature_names_out() == input_vars + ["d3_sub_d1", "d3_sub_d2"] + + tr = DatetimeSubtraction(variables=["d1", "d2"], reference=["d3", "d4"]) + tr.fit(df) + assert tr.get_feature_names_out() == input_vars + [ + "d1_sub_d3", + "d2_sub_d3", + "d1_sub_d4", + "d2_sub_d4", + ] + + +# common tests +estimator = [DatetimeSubtraction(variables=["date1"], reference=["date2"])] + + +@pytest.mark.parametrize("estimator", estimator) +def test_common_tests(estimator): + check_raises_non_fitted_error(estimator) + check_raises_error_when_input_not_a_df(estimator) + check_feature_names_in(estimator) + check_drop_original_variables(estimator) From 84e89a20f681970919da9d229f3baf6e737a00e4 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sun, 12 Mar 2023 08:12:28 +0100 Subject: [PATCH 16/19] polish docstrings --- docs/index.rst | 12 +++--- .../datetime/datetime_subtraction.py | 40 ++++++++++++------- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 5f7685a90..156480f9b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -164,6 +164,12 @@ Feature Creation: - :doc:`api_doc/creation/RelativeFeatures`: combines variables with reference features - :doc:`api_doc/creation/CyclicalFeatures`: creates variables using sine and cosine, suitable for cyclical features +Datetime: +--------- + +- :doc:`api_doc/datetime/DatetimeFeatures`: extract features from datetime variables +- :doc:`api_doc/datetime/DatetimeSubtraction`: computes subtractions between datetime variables + Feature Selection: ------------------ @@ -180,12 +186,6 @@ Feature Selection: - :doc:`api_doc/selection/RecursiveFeatureElimination`: selects features recursively, by evaluating model performance - :doc:`api_doc/selection/RecursiveFeatureAddition`: selects features recursively, by evaluating model performance -Datetime: ---------- - -- :doc:`api_doc/datetime/DatetimeFeatures`: extract features from datetime variables -- :doc:`api_doc/datetime/DatetimeSubtraction`: computes subtractions between datetime variables - Forecasting: ------------ diff --git a/feature_engine/datetime/datetime_subtraction.py b/feature_engine/datetime/datetime_subtraction.py index 6d5954362..eff2fd245 100644 --- a/feature_engine/datetime/datetime_subtraction.py +++ b/feature_engine/datetime/datetime_subtraction.py @@ -10,7 +10,6 @@ _n_features_in_docstring, ) from feature_engine._docstrings.init_parameters.all_trasnformers import ( - _drop_original_docstring, _missing_values_docstring, ) from feature_engine._docstrings.methods import ( @@ -30,22 +29,30 @@ _check_init_parameter_variables, ) -_demo_df = """ +_example = """ + >>> import pandas as pd + >>> from feature_engine.datetime import DatetimeSubtraction >>> X = pd.DataFrame({ >>> "date1": ["2022-09-18", "2022-10-27", "2022-12-24"], >>> "date2": ["2022-08-18", "2022-08-27", "2022-06-24"]}) + >>> dtf = DatetimeSubtraction(variables=["date1"], reference=["date2"]) + >>> dtf.fit(X) + >>> dtf.transform(X) + date1 date2 date1_sub_date2 + 0 2022-09-18 2022-08-18 31.0 + 1 2022-10-27 2022-08-27 61.0 + 2 2022-12-24 2022-06-24 183.0 """.rstrip() @Substitution( missing_values=_missing_values_docstring, - drop_original=_drop_original_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, fit=_fit_not_learn_docstring, transform=_transform_creation_docstring, fit_transform=_fit_transform_docstring, - demo_df=_demo_df, + example=_example, ) class DatetimeSubtraction(BaseCreation): """ @@ -79,7 +86,9 @@ class DatetimeSubtraction(BaseCreation): {missing_values} - {drop_original} + drop_original: bool, default="False" + If `True`, the variables listed in `variables` and `reference` will be dropped + from the dataframe after the computation of the new features. dayfirst: bool, default="False" Specify a date parse order if arg is str or is list-like. If True, parses @@ -100,6 +109,16 @@ class DatetimeSubtraction(BaseCreation): Attributes ---------- + variables_: + The list with datetime variables from which the variables in `reference` will + be substracted. It is created after the transformer corroborates that the + variables in `variables` are, or can be parsed to datetime. + + reference_: + The list with the datetime variables that will be subtracted from `variables_`. + It is created after the transformer corroborates that the variables in + `reference` are, or can be parsed to datetime. + {feature_names_in_} {n_features_in_} @@ -115,16 +134,7 @@ class DatetimeSubtraction(BaseCreation): Examples -------- - >>> import pandas as pd - >>> from feature_engine.datetime import DatetimeSubtraction - {demo_df} - >>> dtf = DatetimeSubtraction(variables=["date1"], reference=["date2"]) - >>> dtf.fit(X) - >>> dtf.transform(X) - date1 date2 date1_sub_date2 - 0 2022-09-18 2022-08-18 31.0 - 1 2022-10-27 2022-08-27 61.0 - 2 2022-12-24 2022-06-24 183.0 + {example} """ def __init__( From 75cac5aa95a8820a919a1eaec51cc1910e835be8 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sun, 12 Mar 2023 08:51:09 +0100 Subject: [PATCH 17/19] initial draft user guide --- .../datetime/DatetimeSubtraction.rst | 836 ++++-------------- 1 file changed, 178 insertions(+), 658 deletions(-) diff --git a/docs/user_guide/datetime/DatetimeSubtraction.rst b/docs/user_guide/datetime/DatetimeSubtraction.rst index 5efc9a191..64349dfca 100644 --- a/docs/user_guide/datetime/DatetimeSubtraction.rst +++ b/docs/user_guide/datetime/DatetimeSubtraction.rst @@ -5,785 +5,305 @@ DatetimeSubtraction =================== -In datasets commonly used in data science and machine learning projects, the variables very -often contain information about date and time. **Date of birth** and **time of purchase** are two -examples of these variables. They are commonly referred to as “datetime features”, that is, -data whose data type is date and time. - -We don’t normally use datetime variables in their raw format to train machine learning models, -like those for regression, classification, or clustering. Instead, we can extract a lot of information -from these variables by extracting the different date and time components of the datetime -variable. - -Examples of date and time components are the year, the month, the week_of_year, the day -of the week, the hour, the minutes, and the seconds. - -Datetime features with pandas ------------------------------ - -In Python, we can extract date and time components through the `dt` module of the open-source -library pandas. For example, by executing the following: - -.. code:: python - - data = pd.DataFrame({"date": pd.date_range("2019-03-05", periods=20, freq="D")}) - - data["year"] = data["date"].dt.year - data["quarter"] = data["date"].dt.quarter - data["month"] = data["date"].dt.month - -In the former code block we created 3 features from the timestamp variable: the *year*, the -*quarter* and the *month*. - - -Datetime features with Feature-engine -------------------------------------- - -:class:`DatetimeFeatures()` automatically extracts several date and time features from -datetime variables. It works with variables whose dtype is datetime, as well as with -object-like and categorical variables, provided that they can be parsed into datetime -format. It *cannot* extract features from numerical variables. - -:class:`DatetimeFeatures()` uses the pandas `dt` module under the hood, therefore automating -datetime feature engineering. In two lines of code and by specifying which features we -want to create with :class:`DatetimeFeatures()`, we can create multiple date and time variables -from various variables simultaneously. - -:class:`DatetimeFeatures()` can automatically create all features supported by pandas `dt` -and a few more, like, for example, a binary feature indicating if the event occurred on -a weekend and also the semester. - -With :class:`DatetimeFeatures()` we can choose which date and time features to extract -from the datetime variables. We can also extract date and time features from one or more -datetime variables at the same time. - -Through the following examples we highlight the functionality and versatility of :class:`DatetimeFeatures()` -for tabular data. - -Extract date features -~~~~~~~~~~~~~~~~~~~~~ - -In this example, we are going to extract three **date features** from a -specific variable in the dataframe. In particular, we are interested -in the *month*, the *day of the year*, and whether that day was the *last -day the month*. +Very often we have datetime variables in our datasets and we want to determine the +time elapsed between them. For example, if we work with financial data, we may have the +variable `date_of_loan_application` with the date and time when the customer applied +for a loan, and also the variable `date_of_birth`, with the customers' date of birth. +With those 2 variables, we want to infer the **age** of the customer at the time of application. +In order to do this, we can compute the difference in years between `date_of_loan_application` +and `date_of_birth` and capture it in a new variable. + +In a different example, if we are trying to predict the price of the house and we have +information about the year in which the house was built, we can infer the age of the house +at the point of sale. Generally, older houses cost less. + +Subtracting datetime features with pandas +----------------------------------------- -First, we will create a toy dataframe with 2 date variables: +In Python, we can subtract datetime variables with pandas. Let's create a toy dataframe +with 2 datetime variables first: .. code:: python + import numpy as np import pandas as pd - from feature_engine.datetime import DatetimeFeatures - - toy_df = pd.DataFrame({ - "var_date1": ['May-1989', 'Dec-2020', 'Jan-1999', 'Feb-2002'], - "var_date2": ['06/21/12', '02/10/98', '08/03/10', '10/31/20'], - }) - -Now, we will extract the variables month, month-end and the day of the year from the -second datetime variable in our dataset. - -.. code:: python - - dtfs = DatetimeFeatures( - variables="var_date2", - features_to_extract=["month", "month_end", "day_of_year"] - ) - - df_transf = dtfs.fit_transform(toy_df) - df_transf + data = pd.DataFrame({ + "date1": pd.date_range("2019-03-05", periods=5, freq="D"), + "date2": pd.date_range("2018-03-05", periods=5, freq="W")}) -With `transform()`, the features extracted from the datetime variable are added to the -dataframe. + print(data) -We see the new features in the following output: +This is the data that we created: .. code:: python - var_date1 var_date2_month var_date2_month_end var_date2_day_of_year - 0 May-1989 6 0 173 - 1 Dec-2020 2 0 41 - 2 Jan-1999 8 0 215 - 3 Feb-2002 10 1 305 + date1 date2 + 0 2019-03-05 2018-03-11 + 1 2019-03-06 2018-03-18 + 2 2019-03-07 2018-03-25 + 3 2019-03-08 2018-04-01 + 4 2019-03-09 2018-04-08 -By default, :class:`DatetimeFeatures()` drops the variable from which the date and time -features were extracted, in this case, *var_date2*. To keep the variable, we just need -to indicate `drop_original=False` when initializing the transformer. - -Finally, we can obtain the name of the variables in the returned data as follows: +Now, let's subtract `date2` from `date1` and capture the difference in a new variable: .. code:: python - dtfs.get_feature_names_out() - -.. code:: python - - ['var_date1', - 'var_date2_month', - 'var_date2_month_end', - 'var_date2_day_of_year'] - + data["diff"] = data["date1"].sub(data["date2"]) -Extract time features -~~~~~~~~~~~~~~~~~~~~~ + print(data) -In this example, we are going to extract the feature *minute* from the two time -variables in our dataset. - -First, let's create a toy dataset with 2 time variables and an object variable. - -.. code:: python - - import pandas as pd - from feature_engine.datetime import DatetimeFeatures - - toy_df = pd.DataFrame({ - "not_a_dt": ['not', 'a', 'date', 'time'], - "var_time1": ['12:34:45', '23:01:02', '11:59:21', '08:44:23'], - "var_time2": ['02:27:26', '10:10:55', '17:30:00', '18:11:18'], - }) - -:class:`DatetimeFeatures()` automatically finds all variables that can be parsed to -datetime. So if we want to extract time features from all our datetime variables, we -don't need to specify them. +We see the new variable at the right of the dataframe: .. code:: python - dfts = DatetimeFeatures(features_to_extract=["minute"]) - - df_transf = dfts.fit_transform(toy_df) - - df_transf + date1 date2 diff + 0 2019-03-05 2018-03-11 359 days + 1 2019-03-06 2018-03-18 353 days + 2 2019-03-07 2018-03-25 347 days + 3 2019-03-08 2018-04-01 341 days + 4 2019-03-09 2018-04-08 335 days -We see the new features in the following output: +If we want the units in something different than days, we can use `numpy`'s timedelta: .. code:: python - not_a_dt var_time1_minute var_time2_minute - 0 not 34 27 - 1 a 1 10 - 2 date 59 30 - 3 time 44 11 - - -The transformer found two variables in the dataframe that can be cast to datetime and -proceeded to extract the requested feature from them. - -The variables detected as datetime are stored in the transformer's `variables_` attribute: +data["diff"] = data["date1"].sub(data["date2"], axis=0).apply( + lambda x: x / np.timedelta64(1, "Y")) -.. code:: python +print(data) - dfts.variables_ +We see the new variable now expressing the difference in years, at the right of the dataframe: .. code:: python - ['var_time1', 'var_time2'] + date1 date2 diff + 0 2019-03-05 2018-03-11 0.982909 + 1 2019-03-06 2018-03-18 0.966481 + 2 2019-03-07 2018-03-25 0.950054 + 3 2019-03-08 2018-04-01 0.933626 + 4 2019-03-09 2018-04-08 0.917199 -The original datetime variables are dropped from the data by default. This leaves the -dataset ready to train machine learning algorithms like linear regression or random forests. +We can automate this procedure with ::class:`DatetimeSubstraction()`. -If we want to keep the datetime variables, we just need to indicate `drop_original=False` -when initializing the transformer. +Datetime subtraction with Feature-engine +---------------------------------------- -Finally, if we want to obtain the names of the variables in the output data, we can use: +:class:`DatetimeFeatures()` automatically subtracts several date and time features from +each other. You just need to indicate the features at the right of the subtraction operation +in the `variables` parameters, and those on the left in the `reference` parameter. You can +also change the output unit through the `output_unit` parameter. -.. code:: python +It works with variables whose dtype is datetime, as well as with object-like and categorical +variables, provided that they can be parsed into datetime format. - dfts.get_feature_names_out() - -.. code:: python - - ['not_a_dt', 'var_time1_minute', 'var_time2_minute'] - - -Extract date and time features -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In this example, we will combine what we have seen in the previous two examples -and extract a date feature - *year* - and time feature - *hour* - -from two variables that contain both date and time information. - -Let's go ahead and create a toy dataset with 3 datetime variables. +Following up with the former example: .. code:: python import pandas as pd - from feature_engine.datetime import DatetimeFeatures + from feature_engine.datetime import DatetimeSubtraction - toy_df = pd.DataFrame({ - "var_dt1": pd.date_range("2018-01-01", periods=3, freq="H"), - "var_dt2": ['08/31/00 12:34:45', '12/01/90 23:01:02', '04/25/01 11:59:21'], - "var_dt3": ['03/02/15 02:27:26', '02/28/97 10:10:55', '11/11/03 17:30:00'], - }) + data = pd.DataFrame({ + "date1": pd.date_range("2019-03-05", periods=5, freq="D"), + "date2": pd.date_range("2018-03-05", periods=5, freq="W")}) -Now, we set up the :class:`DatetimeFeatures()` to extract features from 2 of the datetime -variables. In this case, we do not want to drop the datetime variable after extracting -the features. + dtf = DatetimeSubtraction( + variables="date1", + reference="date2", + output_unit="Y") -.. code:: python - - dfts = DatetimeFeatures( - variables=["var_dt1", "var_dt3"], - features_to_extract=["year", "hour"], - drop_original=False, - ) - df_transf = dfts.fit_transform(toy_df) + data = dtf.fit_transform(data) - print(df_transf) + print(data) -We can see the resulting dataframe in the following output: +We see the new variable expressing the difference in years at the right of the dataframe: .. code:: python - var_dt1 var_dt2 var_dt3 var_dt1_year \ - 0 2018-01-01 00:00:00 08/31/00 12:34:45 03/02/15 02:27:26 2018 - 1 2018-01-01 01:00:00 12/01/90 23:01:02 02/28/97 10:10:55 2018 - 2 2018-01-01 02:00:00 04/25/01 11:59:21 11/11/03 17:30:00 2018 + date1 date2 date1_sub_date2 + 0 2019-03-05 2018-03-11 0.982909 + 1 2019-03-06 2018-03-18 0.966481 + 2 2019-03-07 2018-03-25 0.950054 + 3 2019-03-08 2018-04-01 0.933626 + 4 2019-03-09 2018-04-08 0.917199 - var_dt1_hour var_dt3_year var_dt3_hour - 0 0 2015 2 - 1 1 1997 10 - 2 2 2003 17 -And that is it. The new features are now added to the dataframe. - -Time series -~~~~~~~~~~~ - -Time series data consists of datapoints indexed in time order. The time is usually in -the index of the dataframe. We can extract features from the timestamp index and use them -for time series regression or classification, as well as for time series forecasting. - -With :class:`DatetimeFeatures()` we can also create date and time features from the -dataframe index. - -Let's create a toy dataframe with datetime in the index. +We can also drop the original datetime variables after the computation: .. code:: python import pandas as pd + from feature_engine.datetime import DatetimeSubtraction - X = {"ambient_temp": [31.31, 31.51, 32.15, 32.39, 32.62, 32.5, 32.52, 32.68], - "module_temp": [49.18, 49.84, 52.35, 50.63, 49.61, 47.01, 46.67, 47.52], - "irradiation": [0.51, 0.79, 0.65, 0.76, 0.42, 0.49, 0.57, 0.56], - "color": ["green"] * 4 + ["blue"] * 4, - } - - X = pd.DataFrame(X) - X.index = pd.date_range("2020-05-15 12:00:00", periods=8, freq="15min") - - X.head() + data = pd.DataFrame({ + "date1": pd.date_range("2019-03-05", periods=5, freq="D"), + "date2": pd.date_range("2018-03-05", periods=5, freq="W")}) -Below we see the output of our toy dataframe: - -.. code:: python - - ambient_temp module_temp irradiation color - 2020-05-15 12:00:00 31.31 49.18 0.51 green - 2020-05-15 12:15:00 31.51 49.84 0.79 green - 2020-05-15 12:30:00 32.15 52.35 0.65 green - 2020-05-15 12:45:00 32.39 50.63 0.76 green - 2020-05-15 13:00:00 32.62 49.61 0.42 blue - -We can extract features from the index as follows: - -.. code:: python - - from feature_engine.datetime import DatetimeFeatures - - dtf = DatetimeFeatures(variables="index") - - Xtr = dtf.fit_transform(X) - - Xtr - -We can see that the transformer created the default time features and added them at -the end of the dataframe. - -.. code:: python - - ambient_temp module_temp irradiation color month \ - 2020-05-15 12:00:00 31.31 49.18 0.51 green 5 - 2020-05-15 12:15:00 31.51 49.84 0.79 green 5 - 2020-05-15 12:30:00 32.15 52.35 0.65 green 5 - 2020-05-15 12:45:00 32.39 50.63 0.76 green 5 - 2020-05-15 13:00:00 32.62 49.61 0.42 blue 5 - 2020-05-15 13:15:00 32.50 47.01 0.49 blue 5 - 2020-05-15 13:30:00 32.52 46.67 0.57 blue 5 - 2020-05-15 13:45:00 32.68 47.52 0.56 blue 5 - - year day_of_week day_of_month hour minute second - 2020-05-15 12:00:00 2020 4 15 12 0 0 - 2020-05-15 12:15:00 2020 4 15 12 15 0 - 2020-05-15 12:30:00 2020 4 15 12 30 0 - 2020-05-15 12:45:00 2020 4 15 12 45 0 - 2020-05-15 13:00:00 2020 4 15 13 0 0 - 2020-05-15 13:15:00 2020 4 15 13 15 0 - 2020-05-15 13:30:00 2020 4 15 13 30 0 - 2020-05-15 13:45:00 2020 4 15 13 45 0 - -We can obtain the name of all the variables in the output dataframe as follows: - -.. code:: python - - dtf.get_feature_names_out() - -.. code:: python - - ['ambient_temp', - 'module_temp', - 'irradiation', - 'color', - 'month', - 'year', - 'day_of_week', - 'day_of_month', - 'hour', - 'minute', - 'second'] - - -Important ---------- - -We highly recommend specifying the date and time features that you would like to extract -from your datetime variables. - -If you have too many time variables, this might not be possible. In this case, keep in -mind that if you extract date features from variables that have only time, or time features -from variables that have only dates, your features will be meaningless. - -Let's explore the outcome with an example. We create a dataset with only time variables. - -.. code:: python - - import pandas as pd - from feature_engine.datetime import DatetimeFeatures - - toy_df = pd.DataFrame({ - "not_a_dt": ['not', 'a', 'date', 'time'], - "var_time1": ['12:34:45', '23:01:02', '11:59:21', '08:44:23'], - "var_time2": ['02:27:26', '10:10:55', '17:30:00', '18:11:18'], - }) - -And now we mistakenly extract only date features: - -.. code:: python - - dfts = DatetimeFeatures( - features_to_extract=["year", "month", "day_of_week"], + dtf = DatetimeSubtraction( + variables="date1", + reference="date2", + output_unit="M", + drop_original=True ) - df_transf = dfts.fit_transform(toy_df) - print(df_transf) + data = dtf.fit_transform(data) -.. code:: python + print(data) - not_a_dt var_time1_year var_time1_month var_time1_day_of_week var_time2_year \ - 0 not 2021 12 2 2021 - 1 a 2021 12 2 2021 - 2 date 2021 12 2 2021 - 3 time 2021 12 2 2021 +.. code:: python - var_time2_month var_time2_day_of_week - 0 12 2 - 1 12 2 - 2 12 2 - 3 12 2 + date1_sub_date2 + 0 11.794903 + 1 11.597774 + 2 11.400645 + 3 11.203515 + 4 11.006386 -The transformer will still create features derived from today's date (the date of -creating the docs). -If instead we have a dataframe with only date variables: +We can perform multiple subtractions at the same time: .. code:: python import pandas as pd - from feature_engine.datetime import DatetimeFeatures + from feature_engine.datetime import DatetimeSubtraction - toy_df = pd.DataFrame({ - "var_date1": ['May-1989', 'Dec-2020', 'Jan-1999', 'Feb-2002'], - "var_date2": ['06/21/12', '02/10/98', '08/03/10', '10/31/20'], + data = pd.DataFrame({ + "date1" : ["2022-09-01", "2022-10-01", "2022-12-01"], + "date2" : ["2022-09-15", "2022-10-15", "2022-12-15"], + "date3" : ["2022-08-01", "2022-09-01", "2022-11-01"], + "date4" : ["2022-08-15", "2022-09-15", "2022-11-15"], }) -And we mistakenly extract the hour and the minute: - -.. code:: python + dtf = DatetimeSubtraction(variables=["date1", "date2"], reference=["date3", "date4"]) - dfts = DatetimeFeatures( - features_to_extract=["hour", "minute"], - ) - df_transf = dfts.fit_transform(toy_df) + data = dtf.fit_transform(data) - print(df_transf) + print(data) .. code:: python - var_date1_hour var_date1_minute var_date2_hour var_date2_minute - 0 0 0 0 0 - 1 0 0 0 0 - 2 0 0 0 0 - 3 0 0 0 0 + date1 date2 date3 date4 date1_sub_date3 \ + 0 2022-09-01 2022-09-15 2022-08-01 2022-08-15 31.0 + 1 2022-10-01 2022-10-15 2022-09-01 2022-09-15 30.0 + 2 2022-12-01 2022-12-15 2022-11-01 2022-11-15 30.0 -The new features will contain the value 0. + date2_sub_date3 date1_sub_date4 date2_sub_date4 + 0 45.0 17.0 31.0 + 1 44.0 16.0 30.0 + 2 44.0 16.0 30.0 -Automating feature extraction ------------------------------ -We can indicate which features we want to extract from the datetime variables as we did -in the previous examples, by passing the feature names in lists. - -Alternatively, :class:`DatetimeFeatures()` has default options to extract a group of -commonly used features, or all supported features. - -Let's first create a toy dataframe: +We can work with variables with nan: .. code:: python import pandas as pd - from feature_engine.datetime import DatetimeFeatures + from feature_engine.datetime import DatetimeSubtraction - toy_df = pd.DataFrame({ - "var_dt1": pd.date_range("2018-01-01", periods=3, freq="H"), - "var_dt2": ['08/31/00 12:34:45', '12/01/90 23:01:02', '04/25/01 11:59:21'], - "var_dt3": ['03/02/15 02:27:26', '02/28/97 10:10:55', '11/11/03 17:30:00'], + data = pd.DataFrame({ + "date1" : ["2022-09-01", "2022-10-01", "2022-12-01"], + "date2" : ["2022-09-15", np.nan, "2022-12-15"], + "date3" : ["2022-08-01", "2022-09-01", "2022-11-01"], + "date4" : ["2022-08-15", "2022-09-15", np.nan], }) -Most common features -~~~~~~~~~~~~~~~~~~~~ + dtf = DatetimeSubtraction( + variables=["date1", "date2"], + reference=["date3", "date4"], + missing_values="ignore") -Now, we will extract the **most common** date and time features from one of the variables. -To do this, we leave the parameter `features_to_extract` to `None`. + data = dtf.fit_transform(data) -.. code:: python + print(data) - dfts = DatetimeFeatures( - variables=["var_dt1"], - features_to_extract=None, - drop_original=False, - ) - - df_transf = dfts.fit_transform(toy_df) - print(df_transf) .. code:: python - var_dt1 var_dt2 var_dt3 var_dt1_month \ - 0 2018-01-01 00:00:00 08/31/00 12:34:45 03/02/15 02:27:26 1 - 1 2018-01-01 01:00:00 12/01/90 23:01:02 02/28/97 10:10:55 1 - 2 2018-01-01 02:00:00 04/25/01 11:59:21 11/11/03 17:30:00 1 - - var_dt1_year var_dt1_day_of_week var_dt1_day_of_month var_dt1_hour \ - 0 2018 0 1 0 - 1 2018 0 1 - 2 2018 0 1 2 + date1 date2 date3 date4 date1_sub_date3 \ + 0 2022-09-01 2022-09-15 2022-08-01 2022-08-15 31.0 + 1 2022-10-01 NaN 2022-09-01 2022-09-15 30.0 + 2 2022-12-01 2022-12-15 2022-11-01 NaN 30.0 - var_dt1_minute var_dt1_second - 0 0 0 - 1 0 0 - 2 0 0 + date2_sub_date3 date1_sub_date4 date2_sub_date4 + 0 45.0 17.0 31.0 + 1 NaN 16.0 NaN + 2 44.0 NaN NaN -Our new dataset contains the original features plus the new variables extracted -from them. - -We can find the group of features extracted by the transformer in its attribute: - -.. code:: python - dfts.features_to_extract_ +Finally, we can extract the names of the transformed dataframe for compatibility with the +Scikit-learn pipeline: .. code:: python - ['month', - 'year', - 'day_of_week', - 'day_of_month', - 'hour', - 'minute', - 'second'] - -All supported features -~~~~~~~~~~~~~~~~~~~~~~ - -We can also extract all supported features automatically, by setting the parameter -`features_to_extract` to `"all"`: - -.. code:: python - - dfts = DatetimeFeatures( - variables=["var_dt1"], - features_to_extract='all', - drop_original=False, - ) - - df_transf = dfts.fit_transform(toy_df) - - print(df_transf) - -.. code:: python - - var_dt1 var_dt2 var_dt3 var_dt1_month \ - 0 2018-01-01 00:00:00 08/31/00 12:34:45 03/02/15 02:27:26 1 - 1 2018-01-01 01:00:00 12/01/90 23:01:02 02/28/97 10:10:55 1 - 2 2018-01-01 02:00:00 04/25/01 11:59:21 11/11/03 17:30:00 1 - - var_dt1_quarter var_dt1_semester var_dt1_year \ - 0 1 1 2018 - 1 1 1 2018 - 2 1 1 2018 - - var_dt1_week var_dt1_day_of_week ... var_dt1_month_end var_dt1_quarter_start \ - 0 1 0 ... 0 1 - 1 1 0 ... 0 1 - 2 1 0 ... 0 1 - - var_dt1_quarter_end var_dt1_year_start var_dt1_year_end \ - 0 0 1 0 - 1 0 1 0 - 2 0 1 0 - - var_dt1_leap_year var_dt1_days_in_month var_dt1_hour var_dt1_minute \ - 0 0 31 0 0 - 1 0 31 1 0 - 2 0 31 2 0 - - var_dt1_second - 0 0 - 1 0 - 2 0 - -We can find the group of features extracted by the transformer in its attribute: - -.. code:: python - - dfts.features_to_extract_ + dtf.get_feature_names_out() .. code:: python - ['month', - 'quarter', - 'semester', - 'year', - 'week', - 'day_of_week', - 'day_of_month', - 'day_of_year', - 'weekend', - 'month_start', - 'month_end', - 'quarter_start', - 'quarter_end', - 'year_start', - 'year_end', - 'leap_year', - 'days_in_month', - 'hour', - 'minute', - 'second'] - -Extract and select features automatically ------------------------------------------ + ['date1', + 'date2', + 'date3', + 'date4', + 'date1_sub_date3', + 'date2_sub_date3', + 'date1_sub_date4', + 'date2_sub_date4'] -If we have a dataframe with date variables, time variables and date and time variables, -we can extract all features, or the most common features from all the variables, and then -go ahead and remove the irrelevant features with the `DropConstantFeatures()` class. -Let's create a dataframe with a mix of datetime variables. +We can also combine the creation of numerical variables from datetime features with the +creation of new features by subtraction of datetime variables: .. code:: python import pandas as pd from sklearn.pipeline import Pipeline - from feature_engine.datetime import DatetimeFeatures - from feature_engine.selection import DropConstantFeatures + from feature_engine.datetime import DatetimeFeatures, DatetimeSubtraction - toy_df = pd.DataFrame({ - "var_date": ['06/21/12', '02/10/98', '08/03/10', '10/31/20'], - "var_time1": ['12:34:45', '23:01:02', '11:59:21', '08:44:23'], - "var_dt": ['08/31/00 12:34:45', '12/01/90 23:01:02', '04/25/01 11:59:21', '04/25/01 11:59:21'], + data = pd.DataFrame({ + "date1" : ["2022-09-01", "2022-10-01", "2022-12-01"], + "date2" : ["2022-09-15", "2022-10-15", "2022-12-15"], + "date3" : ["2022-08-01", "2022-09-01", "2022-11-01"], + "date4" : ["2022-08-15", "2022-09-15", "2022-11-15"], }) -Now, we line up in a Scikit-learn pipeline the :class:`DatetimeFeatures` and the -`DropConstantFeatures()`. The :class:`DatetimeFeatures` will create date features -derived from today for the time variable, and time features with the value 0 for the -date only variable. `DropConstantFeatures()` will identify and remove these features -from the dataset. - -.. code:: python + dtf = DatetimeFeatures(variables=["date1", "date2"], drop_original=False) + dts = DatetimeSubtraction( + variables=["date1", "date2"], + reference=["date3", "date4"], + drop_original=True, + ) pipe = Pipeline([ - ('datetime', DatetimeFeatures()), - ('drop_constant', DropConstantFeatures()), + ("features", dtf),("subtraction", dts) ]) - pipe.fit(toy_df) - -.. code:: python - - Pipeline(steps=[('datetime', DatetimeFeatures()), - ('drop_constant', DropConstantFeatures())]) - -.. code:: python - - df_transf = pipe.transform(toy_df) - - print(df_transf) - -.. code:: python - - var_date_month var_date_year var_date_day_of_week var_date_day_of_month \ - 0 6 2012 3 21 - 1 2 1998 1 10 - 2 8 2010 1 3 - 3 10 2020 5 31 - - var_time1_hour var_time1_minute var_time1_second var_dt_month \ - 0 12 34 45 8 - 1 23 1 2 12 - 2 11 59 21 4 - 3 8 44 23 4 + data = pipe.fit_transform(data) - var_dt_year var_dt_day_of_week var_dt_day_of_month var_dt_hour \ - 0 2000 3 31 12 - 1 1990 5 1 23 - 2 2001 2 25 11 - 3 2001 2 25 11 + print(data) - var_dt_minute var_dt_second - 0 34 45 - 1 1 2 - 2 59 21 - 3 59 21 - -As you can see, we do not have the constant features in the transformed dataset. - -Working with different timezones --------------------------------- - -Time-aware datetime variables can be particularly cumbersome to work with as far -as the format goes. We will briefly show how :class:`DatetimeFeatures()` deals -with such variables in three different scenarios. - -**Case 1**: our dataset contains a time-aware variable in object format, -with potentially different timezones across different observations. -We pass `utc=True` when initializing the transformer to make sure it -converts all data to UTC timezone. .. code:: python - import pandas as pd - from feature_engine.datetime import DatetimeFeatures - - toy_df = pd.DataFrame({ - "var_tz": ['12:34:45+3', '23:01:02-6', '11:59:21-8', '08:44:23Z'] - }) - - dfts = DatetimeFeatures( - features_to_extract=["hour", "minute"], - drop_original=False, - utc=True - ) - - df_transf = dfts.fit_transform(toy_df) - - print(df_transf) - -.. code:: python - - var_tz var_tz_hour var_tz_minute - 0 12:34:45+3 9 34 - 1 23:01:02-6 5 1 - 2 11:59:21-8 19 59 - 3 08:44:23Z 8 44 - - -**Case 2**: our dataset contains a variable that is cast as a localized -datetime in a particular timezone. However, we decide that we want to get all -the datetime information extracted as if it were in UTC timezone. - -.. code:: python - - import pandas as pd - from feature_engine.datetime import DatetimeFeatures - - var_tz = pd.Series(['08/31/00 12:34:45', '12/01/90 23:01:02', '04/25/01 11:59:21']) - var_tz = pd.to_datetime(var_tz) - var_tz = var_tz.dt.tz_localize("US/eastern") - var_tz - -.. code:: python - - 0 2000-08-31 12:34:45-04:00 - 1 1990-12-01 23:01:02-05:00 - 2 2001-04-25 11:59:21-04:00 - dtype: datetime64[ns, US/Eastern] - -We need to pass `utc=True` when initializing the transformer to revert back to the UTC -timezone. - -.. code:: python - - toy_df = pd.DataFrame({"var_tz": var_tz}) - - dfts = DatetimeFeatures( - features_to_extract=["day_of_month", "hour"], - drop_original=False, - utc=True, - ) - - df_transf = dfts.fit_transform(toy_df) - - print(df_transf) - -.. code:: python - - var_tz var_tz_day_of_month var_tz_hour - 0 2000-08-31 12:34:45-04:00 31 16 - 1 1990-12-01 23:01:02-05:00 2 4 - 2 2001-04-25 11:59:21-04:00 25 15 - - -**Case 3**: given a variable like *var_tz* in the example above, we now want -to extract the features keeping the original timezone localization, -therefore we pass `utc=False` or `None`. In this case, we leave it to `None` which -is the default option. - -.. code:: python - - dfts = DatetimeFeatures( - features_to_extract=["day_of_month", "hour"], - drop_original=False, - utc=None, - ) - - df_transf = dfts.fit_transform(toy_df) - - print(df_transf) - -.. code:: python - - var_tz var_tz_day_of_month var_tz_hour - 0 2000-08-31 12:34:45-04:00 31 12 - 1 1990-12-01 23:01:02-05:00 1 23 - 2 2001-04-25 11:59:21-04:00 25 11 - -Note that the hour extracted from the variable differ in this dataframe respect to the -one obtained in **Case 2**. - -Missing timestamps ------------------- - -:class:`DatetimeFeatures` has the option to ignore missing timestamps, or raise an error -when a missing value is encountered in a datetime variable. + date1_month date1_year date1_day_of_week date1_day_of_month date1_hour \ + 0 9 2022 3 1 0 + 1 10 2022 5 1 0 + 2 12 2022 3 1 0 + date1_minute date1_second date2_month date2_year date2_day_of_week \ + 0 0 0 9 2022 3 + 1 0 0 10 2022 5 + 2 0 0 12 2022 3 -Additional resources --------------------- + date2_day_of_month date2_hour date2_minute date2_second \ + 0 15 0 0 0 + 1 15 0 0 0 + 2 15 0 0 0 -You can find an example of how to use :class:`DatetimeFeatures()` with a real dataset in -the following `Jupyter notebook `_ + date1_sub_date3 date2_sub_date3 date1_sub_date4 date2_sub_date4 + 0 31.0 45.0 17.0 31.0 + 1 30.0 44.0 16.0 30.0 + 2 30.0 44.0 16.0 30.0 For tutorials on how to create and use features from datetime columns, check the following courses: From a7e7db81373bb9297f1d0579a970626d0b8c4d06 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sun, 12 Mar 2023 19:37:32 +0100 Subject: [PATCH 18/19] expand user guide, add new var names --- .../datetime/DatetimeSubtraction.rst | 136 ++++++++++++++---- .../datetime/datetime_subtraction.py | 45 +++++- .../test_datetime_subtraction.py | 74 ++++++++++ 3 files changed, 221 insertions(+), 34 deletions(-) diff --git a/docs/user_guide/datetime/DatetimeSubtraction.rst b/docs/user_guide/datetime/DatetimeSubtraction.rst index 64349dfca..189aa0db6 100644 --- a/docs/user_guide/datetime/DatetimeSubtraction.rst +++ b/docs/user_guide/datetime/DatetimeSubtraction.rst @@ -5,23 +5,33 @@ DatetimeSubtraction =================== -Very often we have datetime variables in our datasets and we want to determine the -time elapsed between them. For example, if we work with financial data, we may have the -variable `date_of_loan_application` with the date and time when the customer applied -for a loan, and also the variable `date_of_birth`, with the customers' date of birth. -With those 2 variables, we want to infer the **age** of the customer at the time of application. -In order to do this, we can compute the difference in years between `date_of_loan_application` -and `date_of_birth` and capture it in a new variable. +Very often, we have datetime variables in our datasets, and we want to determine the time +difference between them. For example, if we work with financial data, we may have the +variable **date of loan application**, with the date and time when the customer applied for +a loan, and also the variable **date of birth**, with the customer's date of birth. With those +two variables, we want to infer the **age of the customer** at the time of application. In order +to do this, we can compute the difference in years between `date_of_loan_application` and +`date_of_birth` and capture it in a new variable. In a different example, if we are trying to predict the price of the house and we have information about the year in which the house was built, we can infer the age of the house -at the point of sale. Generally, older houses cost less. +at the point of sale. Generally, older houses cost less. To calculate the age of the house, +we’d simply compute the difference in years between the sale date and the date at which +it was built. + +The Python program offers many options for making operations between datetime objects, like, +for example, the datetime module. Since most likely you will be working with Pandas dataframes, +we will focus this guide on pandas and then how we can automate the procedure with Feature-engine. Subtracting datetime features with pandas ----------------------------------------- -In Python, we can subtract datetime variables with pandas. Let's create a toy dataframe -with 2 datetime variables first: +In Python, we can subtract datetime objects with pandas. To work with datetime variables +in pandas, we need to make sure that the timestamp, which can be represented in various +formats, like strings (str), objects (`"O"`), or datetime, is cast as a datetime. If not, we +can convert strings to datetime objects by executing `pd.to_datetime(df[variable_of_interest])`. + +Let’s create a toy dataframe with 2 datetime variables for a short demo: .. code:: python @@ -34,7 +44,7 @@ with 2 datetime variables first: print(data) -This is the data that we created: +This is the data that we created, containing two datetime variables: .. code:: python @@ -45,7 +55,8 @@ This is the data that we created: 3 2019-03-08 2018-04-01 4 2019-03-09 2018-04-08 -Now, let's subtract `date2` from `date1` and capture the difference in a new variable: +Now, we can subtract `date2` from `date1` and capture the difference in a new variable by +utilizing the pandas subtraction operator: .. code:: python @@ -53,7 +64,8 @@ Now, let's subtract `date2` from `date1` and capture the difference in a new var print(data) -We see the new variable at the right of the dataframe: +The new variable, which expresses the difference in number of days, is at the right of the +dataframe: .. code:: python @@ -64,14 +76,15 @@ We see the new variable at the right of the dataframe: 3 2019-03-08 2018-04-01 341 days 4 2019-03-09 2018-04-08 335 days -If we want the units in something different than days, we can use `numpy`'s timedelta: +If we want the units in something other than days, we can use numpy’s timedelta. The following +example shows how to use this syntax: .. code:: python -data["diff"] = data["date1"].sub(data["date2"], axis=0).apply( - lambda x: x / np.timedelta64(1, "Y")) + data["diff"] = data["date1"].sub(data["date2"], axis=0).apply( + lambda x: x / np.timedelta64(1, "Y")) -print(data) + print(data) We see the new variable now expressing the difference in years, at the right of the dataframe: @@ -84,20 +97,23 @@ We see the new variable now expressing the difference in years, at the right of 3 2019-03-08 2018-04-01 0.933626 4 2019-03-09 2018-04-08 0.917199 -We can automate this procedure with ::class:`DatetimeSubstraction()`. +If you wanted to subtract various datetime variables, you would have to write lines of code +for every subtraction. Fortunately, we can automate this procedure with :class:`DatetimeSubstraction()`. Datetime subtraction with Feature-engine ---------------------------------------- -:class:`DatetimeFeatures()` automatically subtracts several date and time features from +:class:`DatetimeSubstraction()` automatically subtracts several date and time features from each other. You just need to indicate the features at the right of the subtraction operation -in the `variables` parameters, and those on the left in the `reference` parameter. You can -also change the output unit through the `output_unit` parameter. +in the `variables` parameters and those on the left in the `reference parameter`. You can also +change the output unit through the `output_unit` parameter. -It works with variables whose dtype is datetime, as well as with object-like and categorical -variables, provided that they can be parsed into datetime format. +:class:`DatetimeSubstraction()` works with variables whose `dtype` is datetime, as well as +with object-like and categorical variables, provided that they can be parsed into datetime +format. This will be done under the hood by the transformer. -Following up with the former example: +Following up with the former example, here is how we obtain the difference in number of +days using :class:`DatetimeSubstraction()`: .. code:: python @@ -117,7 +133,8 @@ Following up with the former example: print(data) -We see the new variable expressing the difference in years at the right of the dataframe: +With `transform()`, :class:`DatetimeSubstraction()` returns a new dataframe containing the +original variables and also the new variables with the time difference: .. code:: python @@ -129,7 +146,10 @@ We see the new variable expressing the difference in years at the right of the d 4 2019-03-09 2018-04-08 0.917199 -We can also drop the original datetime variables after the computation: +Drop original variables after computation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We have the option to drop the original datetime variables after the computation: .. code:: python @@ -151,6 +171,9 @@ We can also drop the original datetime variables after the computation: print(data) +In this case, the resulting dataframe contains only the time difference between the two +original variables: + .. code:: python date1_sub_date2 @@ -160,8 +183,13 @@ We can also drop the original datetime variables after the computation: 3 11.203515 4 11.006386 +Subtract multiple variables simultaneously +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -We can perform multiple subtractions at the same time: +We can perform multiple subtractions at the same time. In this example, we will add new +datetime variables to the toy dataframe as strings. The idea is to show that +:class:`DatetimeSubstraction()` will convert those strings to datetime under the hood to +carry out the subtraction operation. .. code:: python @@ -181,6 +209,9 @@ We can perform multiple subtractions at the same time: print(data) +The resulting dataframe contains the original variables plus the new variables expressing +the time difference between the date objects. + .. code:: python date1 date2 date3 date4 date1_sub_date3 \ @@ -194,7 +225,13 @@ We can perform multiple subtractions at the same time: 2 44.0 16.0 30.0 -We can work with variables with nan: +Working with missing values +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default, :class:`DatetimeSubstraction()` will raise an error if the dataframe passed +to the `fit()` or `transform()` methods contains NA in the variables to subtract. We can +override this behaviour and allow computations between variables with nan by setting the +parameter `missing_values` to `"ignore"`. Here is a code example: .. code:: python @@ -217,7 +254,8 @@ We can work with variables with nan: print(data) - +When any of the variables contains NAN, the new features with the time difference will also +display NANs: .. code:: python @@ -232,6 +270,46 @@ We can work with variables with nan: 2 44.0 NaN NaN +Working with different timezones +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If we have timestamps in different timezones or variables in different timezones, we can +still perform subtraction operations with :class:`DatetimeSubstraction()` by first setting +all timestamps to the universal central time zone. Here is a code example, were we return +the time difference in microseconds: + +.. code:: python + + import pandas as pd + from feature_engine.datetime import DatetimeSubtraction + + data = pd.DataFrame({ + "date1": ['12:34:45+3', '23:01:02-6', '11:59:21-8', '08:44:23Z'], + "date2": ['09:34:45+1', '23:01:02-6+1', '11:59:21-8-2', '08:44:23+3'] + }) + + dfts = DatetimeSubtraction( + variables="date1", + reference="date2", + utc=True, + output_unit="ms", + ) + + new = dfts.fit_transform(data) + + print(new) + +We see the resulting dataframe with the time difference in microseconds: + +.. code:: python + + date1 date2 date1_sub_date2 + 0 12:34:45+3 09:34:45+1 3600000.0 + 1 23:01:02-6 23:01:02-6+1 25200000.0 + 2 11:59:21-8 11:59:21-8-2 21600000.0 + 3 08:44:23Z 08:44:23+3 10800000.0 + + Finally, we can extract the names of the transformed dataframe for compatibility with the Scikit-learn pipeline: diff --git a/feature_engine/datetime/datetime_subtraction.py b/feature_engine/datetime/datetime_subtraction.py index eff2fd245..2a900a280 100644 --- a/feature_engine/datetime/datetime_subtraction.py +++ b/feature_engine/datetime/datetime_subtraction.py @@ -76,6 +76,11 @@ class DatetimeSubtraction(BaseCreation): The list of datetime reference variables that will be subtracted from `variables` (right side of the subtraction operation). + new_variables_names: list, default=None + Names of the new variables. You have the option to pass a list with the names + you'd like to assing to the new variables. If `None`, the transformer will + assign arbitrary names. + output_unit: string, default='D' The string representation of the output unit of the datetime differences. The default is `D` for day. This parameter is passed to `numpy.timedelta64`. @@ -141,6 +146,7 @@ def __init__( self, variables: Union[None, int, str, List[Union[str, int]]], reference: Union[None, int, str, List[Union[str, int]]], + new_variables_names: Union[None, List[str], str] = None, output_unit: str = "D", missing_values: str = "ignore", drop_original: bool = False, @@ -172,9 +178,21 @@ def __init__( f"{valid_output_units}. Got {output_unit} instead." ) + if new_variables_names is not None: + if ( + not isinstance(new_variables_names, list) + or not all(isinstance(var, str) for var in new_variables_names) + or len(set(new_variables_names)) != len(new_variables_names) + ): + raise ValueError( + "new_variable_names should be None or a list of unique strings. " + f"Got {new_variables_names} instead." + ) + super().__init__(missing_values, drop_original) self.variables = _check_init_parameter_variables(variables) self.reference = _check_init_parameter_variables(reference) + self.new_variables_names = new_variables_names self.output_unit = output_unit self.dayfirst = dayfirst self.yearfirst = yearfirst @@ -200,6 +218,17 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): self.reference_ = find_or_check_datetime_variables(X, self.reference) self.variables_ = find_or_check_datetime_variables(X, self.variables) + if self.new_variables_names is not None: + if len(self.new_variables_names) != len(self.variables_) * len( + self.reference_ + ): + raise ValueError( + f"{len(self.variables_) * len(self.reference_)} new variables will " + f"be created but only {len(self.new_variables_names)} new variable " + f"names were provided. Please check the variables list and try " + f"again." + ) + # check if dataset contains na if self.missing_values == "raise": _check_contains_na(X, self.variables_ + self.reference_) @@ -292,13 +321,19 @@ def _sub(self, dt_df: pd.DataFrame): .apply(lambda s: s / np.timedelta64(1, self.output_unit)) ) + if self.new_variables_names is not None: + new_df.columns = self.new_variables_names + return new_df def _get_new_features_name(self) -> List: """Return names of the created features.""" - feature_names = [ - f"{var}_sub_{reference}" - for reference in self.reference_ - for var in self.variables_ - ] + if self.new_variables_names is not None: + feature_names = self.new_variables_names + else: + feature_names = [ + f"{var}_sub_{reference}" + for reference in self.reference_ + for var in self.variables_ + ] return feature_names diff --git a/tests/test_datetime/test_datetime_subtraction.py b/tests/test_datetime/test_datetime_subtraction.py index c490d82a5..d256a32d9 100644 --- a/tests/test_datetime/test_datetime_subtraction.py +++ b/tests/test_datetime/test_datetime_subtraction.py @@ -46,6 +46,34 @@ def test_init_parameters_variables_and_reference_are_mandatory(_input_vars): DatetimeSubtraction(variables=["var1"]) +@pytest.mark.parametrize( + "_input_vars", + [ + ("var1", "var2"), + {"var1": 1, "var2": 2}, + "var1", + ], +) +def test_new_variable_names_raise_errors(_input_vars): + with pytest.raises(ValueError): + assert DatetimeSubtraction( + variables="var1", reference="var2", new_variables_names=_input_vars + ) + + +def test_new_variable_names_correct_assignment(): + tr = DatetimeSubtraction(variables="var1", reference="var2") + assert tr.new_variables_names is None + tr = DatetimeSubtraction( + variables="var1", reference="var2", new_variables_names=["var1"] + ) + assert tr.new_variables_names == ["var1"] + tr = DatetimeSubtraction( + variables="var1", reference="var2", new_variables_names=["var1", "var2"] + ) + assert tr.new_variables_names == ["var1", "var2"] + + @pytest.mark.parametrize( "output", [ @@ -124,6 +152,15 @@ def test_sets_variables_if_datetime(df_datetime): assert tr.reference_ == ["date_obj1"] +@pytest.mark.parametrize("new", [["new1", "new2"], ["new1", "new2", "new3"]]) +def test_new_variables_raise_error_if_not_adequate_number(df_datetime, new): + tr = DatetimeSubtraction( + variables="date_obj1", reference="date_obj1", new_variables_names=new + ) + with pytest.raises(ValueError): + tr.fit(df_datetime) + + def test_raises_error_when_nan_in_fit(): df = pd.DataFrame( { @@ -233,6 +270,36 @@ def test_multiple_subtractions(): pd.testing.assert_frame_equal(df_output, df_expected, check_dtype=False) +def test_assigns_new_variable_names(): + df_input = pd.DataFrame( + { + "date1": ["2022-09-01", "2022-10-01", "2022-12-01"], + "date2": ["2022-09-15", "2022-10-15", "2022-12-15"], + "date3": ["2022-08-01", "2022-09-01", "2022-11-01"], + "date4": ["2022-08-15", "2022-09-15", "2022-11-15"], + } + ) + df_expected = pd.DataFrame( + { + "date1": ["2022-09-01", "2022-10-01", "2022-12-01"], + "date2": ["2022-09-15", "2022-10-15", "2022-12-15"], + "date3": ["2022-08-01", "2022-09-01", "2022-11-01"], + "date4": ["2022-08-15", "2022-09-15", "2022-11-15"], + "new1": [31, 30, 30], + "new2": [45, 44, 44], + "new3": [17, 16, 16], + "new4": [31, 30, 30], + } + ) + dtf = DatetimeSubtraction( + variables=["date1", "date2"], + reference=["date3", "date4"], + new_variables_names=["new1", "new2", "new3", "new4"], + ) + df_output = dtf.fit_transform(df_input) + pd.testing.assert_frame_equal(df_output, df_expected, check_dtype=False) + + # additional methods @@ -268,6 +335,13 @@ def test_get_feature_names_out(): "d2_sub_d4", ] + new = ["new1", "new2", "new3", "new4"] + tr = DatetimeSubtraction( + variables=["d1", "d2"], reference=["d3", "d4"], new_variables_names=new + ) + tr.fit(df) + assert tr.get_feature_names_out() == input_vars + new + # common tests estimator = [DatetimeSubtraction(variables=["date1"], reference=["date2"])] From de56b15d83e29a48c9c827f856cd8ed03f126238 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sun, 12 Mar 2023 19:54:54 +0100 Subject: [PATCH 19/19] final update to user guide --- .../datetime/DatetimeSubtraction.rst | 74 ++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/docs/user_guide/datetime/DatetimeSubtraction.rst b/docs/user_guide/datetime/DatetimeSubtraction.rst index 189aa0db6..50f65322c 100644 --- a/docs/user_guide/datetime/DatetimeSubtraction.rst +++ b/docs/user_guide/datetime/DatetimeSubtraction.rst @@ -309,14 +309,73 @@ We see the resulting dataframe with the time difference in microseconds: 2 11:59:21-8 11:59:21-8-2 21600000.0 3 08:44:23Z 08:44:23+3 10800000.0 +Adding arbitrary names to the new variables +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Often, we want to compute just a few time differences. In this case, we may want as well +to assign the new variables specific names. In this code example, we do so: + +.. code:: python + + import pandas as pd + from feature_engine.datetime import DatetimeSubtraction + + data = pd.DataFrame({ + "date1": pd.date_range("2019-03-05", periods=5, freq="D"), + "date2": pd.date_range("2018-03-05", periods=5, freq="W")}) + + dtf = DatetimeSubtraction( + variables="date1", + reference="date2", + new_variables_names=["my_new_var"] + ) + + data = dtf.fit_transform(data) + + print(data) + +In the resulting dataframe, we see that the time difference was captured in a variable +called `my_new_var`: + +.. code:: python + + date1 date2 my_new_var + 0 2019-03-05 2018-03-11 359.0 + 1 2019-03-06 2018-03-18 353.0 + 2 2019-03-07 2018-03-25 347.0 + 3 2019-03-08 2018-04-01 341.0 + 4 2019-03-09 2018-04-08 335.0 + +We should be mindful to pass a list of variales containing as many names as new variables. +The number of variables that will be created is obtained by multiplying the number of variables +in the parameter `variables` by the number of variables in the parameter `reference`. + +get_feature_names_out() +~~~~~~~~~~~~~~~~~~~~~~~ Finally, we can extract the names of the transformed dataframe for compatibility with the Scikit-learn pipeline: .. code:: python + import pandas as pd + from feature_engine.datetime import DatetimeSubtraction + + data = pd.DataFrame({ + "date1" : ["2022-09-01", "2022-10-01", "2022-12-01"], + "date2" : ["2022-09-15", "2022-10-15", "2022-12-15"], + "date3" : ["2022-08-01", "2022-09-01", "2022-11-01"], + "date4" : ["2022-08-15", "2022-09-15", "2022-11-15"], + }) + + dtf = DatetimeSubtraction(variables=["date1", "date2"], reference=["date3", "date4"]) + dtf.fit(data) + dtf.get_feature_names_out() +Below the name of the variables that will appear in any dataframe resulting from applying +the `transform()` method: + .. code:: python ['date1', @@ -329,6 +388,9 @@ Scikit-learn pipeline: 'date2_sub_date4'] +Combining extraction and subtraction of datetime features +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + We can also combine the creation of numerical variables from datetime features with the creation of new features by subtraction of datetime variables: @@ -360,6 +422,9 @@ creation of new features by subtraction of datetime variables: print(data) +In the following output we see the new dataframe contaning the features that were extracted +from the different datetime variables followed by those created by capturing the time +difference: .. code:: python @@ -383,7 +448,14 @@ creation of new features by subtraction of datetime variables: 1 30.0 44.0 16.0 30.0 2 30.0 44.0 16.0 30.0 +More details +------------ + For tutorials on how to create and use features from datetime columns, check the following courses: - `Feature Engineering for Machine Learning `_. -- `Feature Engineering for Time Series Forecasting `_. \ No newline at end of file +- `Feature Engineering for Time Series Forecasting `_. + +And the following book: + +- `Python Feature Engineering Cookbook `_. \ No newline at end of file