[MRG] refactor variable handling module

feature_engine/_prediction/base_predictor.py

@@ -6,6 +6,12 @@
 from sklearn.pipeline import Pipeline
 from sklearn.utils.validation import check_is_fitted
 
+from feature_engine._variable_handling.init_parameter_checks import (
+    _check_init_parameter_variables,
+)
+from feature_engine._variable_handling.variable_type_selection import (
+    _find_categorical_and_numerical_variables,
+)
 from feature_engine.dataframe_checks import (
     _check_contains_inf,
     _check_contains_na,
@@ -19,10 +25,6 @@
 )
 from feature_engine.encoding import MeanEncoder
 from feature_engine.tags import _return_tags
-from feature_engine.variable_manipulation import (
-    _check_input_parameter_variables,
-    _find_categorical_and_numerical_variables,
-)
 
 
 class BaseTargetMeanEstimator(BaseEstimator):
@@ -96,7 +98,7 @@ def __init__(
                 f"Got {strategy} instead."
             )
 
-        self.variables = _check_input_parameter_variables(variables)
+        self.variables = _check_init_parameter_variables(variables)
         self.bins = bins
         self.strategy = strategy
 

feature_engine/_variable_handling/init_parameter_checks.py

@@ -0,0 +1,34 @@
+from typing import Any
+
+from feature_engine._variable_handling.variable_type_selection import Variables
+
+
+# set return value typehint to Any to avoid issues with the base transformer fit method
+def _check_init_parameter_variables(variables: Variables) -> Any:
+    """
+    Checks that the input is of the correct type. Allowed  values are None, int, str or
+    list of strings and ints.
+
+    Parameters
+    ----------
+    variables : string, int, list of strings, list of integers. Default=None
+
+    Returns
+    -------
+    variables: same as input
+    """
+
+    msg = "variables should be a string, an int or a list of strings or integers."
+    msg_dupes = "the list contains duplicated variable names"
+
+    if variables:
+        if isinstance(variables, list):
+            if not all(isinstance(i, (str, int)) for i in variables):
+                raise ValueError(msg)
+            if len(variables) != len(set(variables)):
+                raise ValueError(msg_dupes)
+        else:
+            if not isinstance(variables, (str, int)):
+                raise ValueError(msg)
+
+    return variables

feature_engine/_variable_handling/variable_type_checks.py

@@ -0,0 +1,47 @@
+import pandas as pd
+from pandas.core.dtypes.common import is_categorical_dtype as is_categorical
+from pandas.core.dtypes.common import is_datetime64_any_dtype as is_datetime
+from pandas.core.dtypes.common import is_numeric_dtype as is_numeric
+from pandas.core.dtypes.common import is_object_dtype as is_object
+
+
+def _is_categorical_and_is_not_datetime(column: pd.Series) -> bool:
+
+    # check for datetime only if object cannot be cast as numeric because
+    # if it could pd.to_datetime would convert it to datetime regardless
+    if is_object(column):
+        is_cat = _is_convertible_to_num(column) or not _is_convertible_to_dt(column)
+
+    # check for datetime only if the type of the categories is not numeric
+    # because pd.to_datetime throws an error when it is an integer
+    elif is_categorical(column):
+        is_cat = _is_categories_num(column) or not _is_convertible_to_dt(column)
+
+    return is_cat
+
+
+def _is_categories_num(column: pd.Series) -> bool:
+    return is_numeric(column.dtype.categories)
+
+
+def _is_convertible_to_dt(column: pd.Series) -> bool:
+    return is_datetime(pd.to_datetime(column, errors="ignore", utc=True))
+
+
+def _is_convertible_to_num(column: pd.Series) -> bool:
+    return is_numeric(pd.to_numeric(column, errors="ignore"))
+
+
+def _is_categorical_and_is_datetime(column: pd.Series) -> bool:
+
+    # check for datetime only if object cannot be cast as numeric because
+    # if it could pd.to_datetime would convert it to datetime regardless
+    if is_object(column):
+        is_dt = not _is_convertible_to_num(column) and _is_convertible_to_dt(column)
+
+    # check for datetime only if the type of the categories is not numeric
+    # because pd.to_datetime throws an error when it is an integer
+    elif is_categorical(column):
+        is_dt = not _is_categories_num(column) and _is_convertible_to_dt(column)
+
+    return is_dt

feature_engine/variable_manipulation.py → feature_engine/_variable_handling/variable_type_selection.py

@@ -1,45 +1,19 @@
 """Functions to select certain types of variables."""
 
-from typing import Any, List, Tuple, Union
+from typing import List, Tuple, Union
 
 import pandas as pd
 from pandas.api.types import is_categorical_dtype as is_categorical
 from pandas.api.types import is_datetime64_any_dtype as is_datetime
 from pandas.api.types import is_numeric_dtype as is_numeric
 from pandas.api.types import is_object_dtype as is_object
 
-Variables = Union[None, int, str, List[Union[str, int]]]
-
-
-# set return value typehint to Any to avoid issues with the base transformer fit method
-def _check_input_parameter_variables(variables: Variables) -> Any:
-    """
-    Checks that the input is of the correct type. Allowed  values are None, int, str or
-    list of strings and ints.
-
-    Parameters
-    ----------
-    variables : string, int, list of strings, list of integers. Default=None
-
-    Returns
-    -------
-    variables: same as input
-    """
-
-    msg = "variables should be a string, an int or a list of strings or integers."
-    msg_dupes = "the list contains duplicated variable names"
+from feature_engine._variable_handling.variable_type_checks import (
+    _is_categorical_and_is_datetime,
+    _is_categorical_and_is_not_datetime,
+)
 
-    if variables:
-        if isinstance(variables, list):
-            if not all(isinstance(i, (str, int)) for i in variables):
-                raise ValueError(msg)
-            if len(variables) != len(set(variables)):
-                raise ValueError(msg_dupes)
-        else:
-            if not isinstance(variables, (str, int)):
-                raise ValueError(msg)
-
-    return variables
+Variables = Union[None, int, str, List[Union[str, int]]]
 
 
 def _find_or_check_numerical_variables(
@@ -96,33 +70,6 @@ def _find_or_check_numerical_variables(
     return variables
 
 
-def _is_convertible_to_num(column: pd.Series) -> bool:
-    return is_numeric(pd.to_numeric(column, errors="ignore"))
-
-
-def _is_convertible_to_dt(column: pd.Series) -> bool:
-    return is_datetime(pd.to_datetime(column, errors="ignore", utc=True))
-
-
-def _is_categories_num(column: pd.Series) -> bool:
-    return is_numeric(column.dtype.categories)
-
-
-def _is_categorical_and_is_not_datetime(column: pd.Series) -> bool:
-
-    # check for datetime only if object cannot be cast as numeric because
-    # if it could pd.to_datetime would convert it to datetime regardless
-    if is_object(column):
-        is_cat = _is_convertible_to_num(column) or not _is_convertible_to_dt(column)
-
-    # check for datetime only if the type of the categories is not numeric
-    # because pd.to_datetime throws an error when it is an integer
-    elif is_categorical(column):
-        is_cat = _is_categories_num(column) or not _is_convertible_to_dt(column)
-
-    return is_cat
-
-
 def _find_or_check_categorical_variables(
     X: pd.DataFrame, variables: Variables = None
 ) -> List[Union[str, int]]:
@@ -181,21 +128,6 @@ def _find_or_check_categorical_variables(
     return variables
 
 
-def _is_categorical_and_is_datetime(column: pd.Series) -> bool:
-
-    # check for datetime only if object cannot be cast as numeric because
-    # if it could pd.to_datetime would convert it to datetime regardless
-    if is_object(column):
-        is_dt = not _is_convertible_to_num(column) and _is_convertible_to_dt(column)
-
-    # check for datetime only if the type of the categories is not numeric
-    # because pd.to_datetime throws an error when it is an integer
-    elif is_categorical(column):
-        is_dt = not _is_categories_num(column) and _is_convertible_to_dt(column)
-
-    return is_dt
-
-
 def _find_or_check_datetime_variables(
     X: pd.DataFrame, variables: Variables = None
 ) -> List[Union[str, int]]:
@@ -306,7 +238,7 @@ def _filter_out_variables_not_in_dataframe(X, variables):
     Function removes variables that the user defines in the argument `variables`
     but that are not present in the input dataframe.
 
-    Useful when ussing several feature selection procedures in a row. The dataframe
+    Useful when using several feature selection procedures in a row. The dataframe
     input to the first selection algorithm likely contains more variables than the
     input dataframe to subsequent selection algorithms, and it is not possible a
     priori, to say which variable will be dropped.

feature_engine/base_transformers.py

@@ -8,17 +8,19 @@
 from sklearn.base import BaseEstimator, TransformerMixin
 from sklearn.utils.validation import check_is_fitted
 
+from feature_engine._docstrings.methods import _get_feature_names_out_docstring
+from feature_engine._docstrings.substitute import Substitution
+from feature_engine._variable_handling.variable_type_selection import (
+    _find_or_check_numerical_variables,
+)
 from feature_engine.dataframe_checks import (
     _check_contains_inf,
     _check_contains_na,
     _check_X_matches_training_df,
     check_X,
 )
 from feature_engine.get_feature_names_out import _get_feature_names_out
-from feature_engine._docstrings.methods import _get_feature_names_out_docstring
-from feature_engine._docstrings.substitute import Substitution
 from feature_engine.tags import _return_tags
-from feature_engine.variable_manipulation import _find_or_check_numerical_variables
 
 
 class BaseNumericalTransformer(BaseEstimator, TransformerMixin):

feature_engine/creation/__init__.py

@@ -2,14 +2,13 @@
 The module creation includes classes to create new variables by combination of existing
 variables in the dataframe.
 """
-from .cyclical_features import CyclicalFeatures
-from .math_features import MathFeatures
-from .relative_features import RelativeFeatures
-
 # FIXME: remove in version 1.4
 from .combine_with_reference_feature import CombineWithReferenceFeature
 from .cyclical import CyclicalTransformer
+from .cyclical_features import CyclicalFeatures
+from .math_features import MathFeatures
 from .mathematical_combination import MathematicalCombination
+from .relative_features import RelativeFeatures
 
 __all__ = [
     "MathematicalCombination",

feature_engine/creation/base_creation.py

@@ -4,14 +4,16 @@
 from sklearn.base import BaseEstimator, TransformerMixin
 from sklearn.utils.validation import check_is_fitted
 
+from feature_engine._variable_handling.variable_type_selection import (
+    _find_or_check_numerical_variables,
+)
 from feature_engine.dataframe_checks import (
     _check_contains_inf,
     _check_contains_na,
     _check_X_matches_training_df,
     check_X,
 )
 from feature_engine.tags import _return_tags
-from feature_engine.variable_manipulation import _find_or_check_numerical_variables
 
 
 class BaseCreation(BaseEstimator, TransformerMixin):

feature_engine/creation/combine_with_reference_feature.py

@@ -5,27 +5,29 @@
 from sklearn.utils import deprecated
 from sklearn.utils.validation import check_is_fitted
 
-from feature_engine.dataframe_checks import (
-    _check_contains_inf,
-    _check_contains_na,
-    _check_X_matches_training_df,
-    check_X,
-)
-from feature_engine._docstrings.methods import (
-    _fit_not_learn_docstring,
-    _fit_transform_docstring,
+from feature_engine._docstrings.class_inputs import (
+    _drop_original_docstring,
+    _missing_values_docstring,
 )
 from feature_engine._docstrings.fit_attributes import (
     _feature_names_in_docstring,
     _n_features_in_docstring,
 )
-from feature_engine._docstrings.class_inputs import (
-    _drop_original_docstring,
-    _missing_values_docstring,
+from feature_engine._docstrings.methods import (
+    _fit_not_learn_docstring,
+    _fit_transform_docstring,
 )
 from feature_engine._docstrings.substitute import Substitution
+from feature_engine._variable_handling.variable_type_selection import (
+    _find_or_check_numerical_variables,
+)
+from feature_engine.dataframe_checks import (
+    _check_contains_inf,
+    _check_contains_na,
+    _check_X_matches_training_df,
+    check_X,
+)
 from feature_engine.tags import _return_tags
-from feature_engine.variable_manipulation import _find_or_check_numerical_variables
 
 
 @deprecated(

feature_engine/creation/cyclical.py

@@ -4,19 +4,21 @@
 import pandas as pd
 from sklearn.utils import deprecated
 
-from feature_engine.base_transformers import BaseNumericalTransformer
-from feature_engine._docstrings.methods import _fit_transform_docstring
+from feature_engine._docstrings.class_inputs import (
+    _drop_original_docstring,
+    _variables_numerical_docstring,
+)
 from feature_engine._docstrings.fit_attributes import (
-    _variables_attribute_docstring,
     _feature_names_in_docstring,
     _n_features_in_docstring,
+    _variables_attribute_docstring,
 )
-from feature_engine._docstrings.class_inputs import (
-    _variables_numerical_docstring,
-    _drop_original_docstring,
-)
+from feature_engine._docstrings.methods import _fit_transform_docstring
 from feature_engine._docstrings.substitute import Substitution
-from feature_engine.variable_manipulation import _check_input_parameter_variables
+from feature_engine._variable_handling.init_parameter_checks import (
+    _check_init_parameter_variables,
+)
+from feature_engine.base_transformers import BaseNumericalTransformer
 
 
 @deprecated(
@@ -109,7 +111,7 @@ def __init__(
         if not isinstance(drop_original, bool):
             raise TypeError("drop_original takes only boolean values True and False.")
 
-        self.variables = _check_input_parameter_variables(variables)
+        self.variables = _check_init_parameter_variables(variables)
         self.max_values = max_values
         self.drop_original = drop_original