From 5534ac19e4799a2581e37fd3fc23190e8739a3c4 Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+GLevV@users.noreply.github.com> Date: Fri, 28 Oct 2022 15:25:17 +0000 Subject: [PATCH 01/34] keywords support for sim encodere --- feature_engine/encoding/similarity_encoder.py | 51 +++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index 9b9c226c0..4b0f5c515 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -102,7 +102,7 @@ class StringSimilarityEncoder(CategoricalInitMixin, CategoricalMethodsMixin): categories to encode. In this case, similarity variables will be created only for those popular categories. - missing_values : str, default='impute' + missing_values: str, default='impute' Indicates if missing values should be ignored, raised or imputed. If 'raise' the transformer will return an error if the datasets to `fit` or `transform` contain missing values. If 'ignore', missing data will be ignored when learning @@ -110,6 +110,14 @@ class StringSimilarityEncoder(CategoricalInitMixin, CategoricalMethodsMixin): replace missing values with an empty string, '', and then return the similarity measures. + keywords: dict, default=None + User defined dictionary of keywords, dict(feature: [keyword1, keyword2, ...]). + Instead of finding top_k categories in features, encoder will use this keywords + to create similarity variables. Useful when someone has domain knowledge of the + problem. Could be defined only for several features; in this case for specified + features keywords will be used and most common categories will be used for + unspecified. + {variables} {ignore_format} @@ -160,7 +168,8 @@ class StringSimilarityEncoder(CategoricalInitMixin, CategoricalMethodsMixin): def __init__( self, - top_categories: Union[None, int] = None, + top_categories: Optional[int] = None, + keywords: Optional[dict[str: list[str]]] = None, missing_values: str = "impute", variables: Union[None, int, str, List[Union[str, int]]] = None, ignore_format: bool = False, @@ -174,9 +183,19 @@ def __init__( "missing_values should be one of 'raise', 'impute' or 'ignore'." f" Got {missing_values!r} instead." ) + if keywords and not isinstance(keywords, dict): + raise ValueError( + f"keywords should be dict or None. Got {keywords!r} instead." + ) + if not all(isinstance(item, list) for item in keywords): + raise ValueError( + "Items in keywords should be lists." + f" Got {keywords.items()!r} instead." + ) super().__init__(variables, ignore_format) self.top_categories = top_categories self.missing_values = missing_values + self.keywords = keywords def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ @@ -198,20 +217,19 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): X = check_X(X) self._check_or_select_variables(X) self._get_feature_names_in(X) + if self.keywords: + if not all(item in self.variables_ for item in self.keywords.keys()) + raise ValueError( + "keywords have columns that are not present in the dataset" + ) self.encoder_dict_ = {} if self.missing_values == "raise": _check_contains_na(X, self.variables_) - for var in self.variables_: - self.encoder_dict_[var] = ( - X[var] - .astype(str) - .value_counts() - .head(self.top_categories) - .index.tolist() - ) - elif self.missing_values == "impute": - for var in self.variables_: + for var in self.variables_: + if self.keywords and self.keywords.get(var): + self.encoder_dict_[var] = self.keywords[var] + else: self.encoder_dict_[var] = ( X[var] .astype(str) @@ -220,15 +238,6 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): .head(self.top_categories) .index.tolist() ) - elif self.missing_values == "ignore": - for var in self.variables_: - self.encoder_dict_[var] = ( - X[var] - .astype(str) - .value_counts(dropna=True) - .head(self.top_categories) - .index.tolist() - ) return self From 8425b3a34d5fd8d12162c6e49bd77f0760aa27ad Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+GLevV@users.noreply.github.com> Date: Fri, 28 Oct 2022 15:26:40 +0000 Subject: [PATCH 02/34] smallfix --- feature_engine/encoding/similarity_encoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index 4b0f5c515..12d944efb 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -169,7 +169,7 @@ class StringSimilarityEncoder(CategoricalInitMixin, CategoricalMethodsMixin): def __init__( self, top_categories: Optional[int] = None, - keywords: Optional[dict[str: list[str]]] = None, + keywords: Optional[dict[str: List[str]]] = None, missing_values: str = "impute", variables: Union[None, int, str, List[Union[str, int]]] = None, ignore_format: bool = False, From 56011c6c45bcc6f09480742d0871f3c66b795343 Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+GLevV@users.noreply.github.com> Date: Fri, 28 Oct 2022 15:35:42 +0000 Subject: [PATCH 03/34] added tests for keywords --- .../test_encoding/test_similarity_encoder.py | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/tests/test_encoding/test_similarity_encoder.py b/tests/test_encoding/test_similarity_encoder.py index 6f88bb6f4..adcb97970 100644 --- a/tests/test_encoding/test_similarity_encoder.py +++ b/tests/test_encoding/test_similarity_encoder.py @@ -213,3 +213,109 @@ def test_get_feature_names_out_na(df_enc_big_na): } assert tr.get_feature_names_out(input_features=None) == out assert tr.get_feature_names_out(input_features=input_features) == out + + +def test_keywords_bad_type(): + with pytest.raises(ValueError): + encoder = StringSimilarityEncoder(keywords="hola") + + +def test_keywords_bad_items(): + with pytest.raises(ValueError): + encoder = StringSimilarityEncoder(keywords={"column": "hola"}) + + +def test_keywords_dont_match(df_enc_big): + encoder = StringSimilarityEncoder(keywords={"column": "hola"}) + with pytest.raises(ValueError): + encoder.fit(df_enc_big) + + +def test_encode_top_categories_w_keywords(): + df = pd.DataFrame( + { + "var_A": ["A"] * 5 + + ["B"] * 11 + + ["C"] * 4 + + ["D"] * 9 + + ["E"] * 2 + + ["F"] * 2 + + ["G"] * 7, + "var_B": ["A"] * 11 + + ["B"] * 7 + + ["C"] * 4 + + ["D"] * 9 + + ["E"] * 2 + + ["F"] * 2 + + ["G"] * 5, + "var_C": ["A"] * 4 + + ["B"] * 5 + + ["C"] * 11 + + ["D"] * 9 + + ["E"] * 2 + + ["F"] * 2 + + ["G"] * 7, + } + ) + + encoder = StringSimilarityEncoder(top_categories=2, keywords={"var_A": ["XYZ"]}) + X = encoder.fit_transform(df) + + # test init params + assert encoder.top_categories == 2 + # test fit attr + transf = { + "var_A_D": 9, + "var_A_B": 11, + "var_B_A": 11, + "var_B_D": 9, + "var_C_D": 9, + "var_C_C": 11, + } + + # test fit attr + assert encoder.variables_ == ["var_A", "var_B", "var_C"] + assert encoder.n_features_in_ == 3 + assert encoder.encoder_dict_ == { + "var_A": ["XYZ"], + "var_B": ["A", "D", "B", "G"], + "var_C": ["C", "D", "G", "B"], + } + # test transform output + for col in transf.keys(): + assert X[col].sum() == transf[col] + assert "var_B" not in X.columns + assert "var_B_F" not in X.columns + + +def test_get_feature_names_out_w_keywords(df_enc_big): + input_features = df_enc_big_na.columns.tolist() + + tr = StringSimilarityEncoder(keywords={"var_A": ["XYZ"]}) + tr.fit(df_enc_big_na) + + out = [ + "var_A_XYZ", + "var_B_A", + "var_B_D", + "var_B_B", + "var_B_G", + "var_B_C", + "var_B_E", + "var_B_F", + "var_C_C", + "var_C_D", + "var_C_B", + "var_C_G", + "var_C_A", + "var_C_E", + "var_C_F", + ] + + assert tr.encoder_dict_ == { + "var_A": ["XYZ"], + "var_B": ["A", "D", "B", "G", "C", "E", "F"], + "var_C": ["C", "D", "B", "G", "A", "E", "F"], + } + assert tr.get_feature_names_out(input_features=None) == out + assert tr.get_feature_names_out(input_features=input_features) == out From 431678d14265bf29829938f67a429fbd4c02bca5 Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+GLevV@users.noreply.github.com> Date: Fri, 28 Oct 2022 15:40:12 +0000 Subject: [PATCH 04/34] hotfix --- feature_engine/encoding/similarity_encoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index 12d944efb..f71e416ed 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -218,7 +218,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): self._check_or_select_variables(X) self._get_feature_names_in(X) if self.keywords: - if not all(item in self.variables_ for item in self.keywords.keys()) + if not all(item in self.variables_ for item in self.keywords.keys()): raise ValueError( "keywords have columns that are not present in the dataset" ) From 33de650c4b4f9a812768ec5b799095edc62dd94c Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+GLevV@users.noreply.github.com> Date: Fri, 28 Oct 2022 15:42:42 +0000 Subject: [PATCH 05/34] small fix --- feature_engine/encoding/similarity_encoder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index f71e416ed..3092437b2 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -1,5 +1,5 @@ from difflib import SequenceMatcher -from typing import List, Optional, Union +from typing import List, Optional, Union, TypedDict import numpy as np import pandas as pd @@ -169,7 +169,7 @@ class StringSimilarityEncoder(CategoricalInitMixin, CategoricalMethodsMixin): def __init__( self, top_categories: Optional[int] = None, - keywords: Optional[dict[str: List[str]]] = None, + keywords: Optional[TypedDict[str, List[str]]] = None, missing_values: str = "impute", variables: Union[None, int, str, List[Union[str, int]]] = None, ignore_format: bool = False, From 618bda0065689d44364dd235b7bf05bc2549ae93 Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+GLevV@users.noreply.github.com> Date: Fri, 28 Oct 2022 15:48:22 +0000 Subject: [PATCH 06/34] revert --- feature_engine/encoding/similarity_encoder.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index 3092437b2..d86d5ed0a 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -1,5 +1,5 @@ from difflib import SequenceMatcher -from typing import List, Optional, Union, TypedDict +from typing import Optional, Union import numpy as np import pandas as pd @@ -169,9 +169,9 @@ class StringSimilarityEncoder(CategoricalInitMixin, CategoricalMethodsMixin): def __init__( self, top_categories: Optional[int] = None, - keywords: Optional[TypedDict[str, List[str]]] = None, + keywords: Optional[dict[str, list[str]]] = None, missing_values: str = "impute", - variables: Union[None, int, str, List[Union[str, int]]] = None, + variables: Union[None, int, str, list[Union[str, int]]] = None, ignore_format: bool = False, ): if top_categories and not isinstance(top_categories, int): @@ -282,7 +282,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: return X.drop(self.variables_, axis=1) - def _get_new_features_name(self) -> List: + def _get_new_features_name(self) -> list: """Return names of the created features.""" feature_names = [] for feature in self.variables_: @@ -294,7 +294,7 @@ def _get_new_features_name(self) -> List: return feature_names - def _add_new_feature_names(self, feature_names) -> List: + def _add_new_feature_names(self, feature_names: list) -> list: """Creates new features names and removes original categorical variables.""" feature_names = feature_names + self._get_new_features_name() feature_names = [f for f in feature_names if f not in self.variables_] From 42ab3a2273905e0bc5f369f196c08300f91323c1 Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+GLevV@users.noreply.github.com> Date: Fri, 28 Oct 2022 15:54:39 +0000 Subject: [PATCH 07/34] Update test_similarity_encoder.py --- tests/test_encoding/test_similarity_encoder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_encoding/test_similarity_encoder.py b/tests/test_encoding/test_similarity_encoder.py index adcb97970..5bfdffb1a 100644 --- a/tests/test_encoding/test_similarity_encoder.py +++ b/tests/test_encoding/test_similarity_encoder.py @@ -217,12 +217,12 @@ def test_get_feature_names_out_na(df_enc_big_na): def test_keywords_bad_type(): with pytest.raises(ValueError): - encoder = StringSimilarityEncoder(keywords="hola") + StringSimilarityEncoder(keywords="hola") def test_keywords_bad_items(): with pytest.raises(ValueError): - encoder = StringSimilarityEncoder(keywords={"column": "hola"}) + StringSimilarityEncoder(keywords={"column": "hola"}) def test_keywords_dont_match(df_enc_big): @@ -288,7 +288,7 @@ def test_encode_top_categories_w_keywords(): assert "var_B_F" not in X.columns -def test_get_feature_names_out_w_keywords(df_enc_big): +def test_get_feature_names_out_w_keywords(df_enc_big_na): input_features = df_enc_big_na.columns.tolist() tr = StringSimilarityEncoder(keywords={"var_A": ["XYZ"]}) From d5f3bf96cfdfb83be712f7585d9600ad598b94d4 Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+GLevV@users.noreply.github.com> Date: Fri, 28 Oct 2022 15:54:42 +0000 Subject: [PATCH 08/34] Update similarity_encoder.py --- feature_engine/encoding/similarity_encoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index d86d5ed0a..c1a234393 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -169,7 +169,7 @@ class StringSimilarityEncoder(CategoricalInitMixin, CategoricalMethodsMixin): def __init__( self, top_categories: Optional[int] = None, - keywords: Optional[dict[str, list[str]]] = None, + keywords: Optional[dict] = None, missing_values: str = "impute", variables: Union[None, int, str, list[Union[str, int]]] = None, ignore_format: bool = False, From b9167d09f08baa4e6a409cc118aff202365b17a0 Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+GLevV@users.noreply.github.com> Date: Fri, 28 Oct 2022 15:57:24 +0000 Subject: [PATCH 09/34] Update similarity_encoder.py --- feature_engine/encoding/similarity_encoder.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index c1a234393..0ad9652a2 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -1,5 +1,5 @@ from difflib import SequenceMatcher -from typing import Optional, Union +from typing import Optional, Union, List import numpy as np import pandas as pd @@ -171,7 +171,7 @@ def __init__( top_categories: Optional[int] = None, keywords: Optional[dict] = None, missing_values: str = "impute", - variables: Union[None, int, str, list[Union[str, int]]] = None, + variables: Union[None, int, str, List[Union[str, int]]] = None, ignore_format: bool = False, ): if top_categories and not isinstance(top_categories, int): @@ -282,7 +282,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: return X.drop(self.variables_, axis=1) - def _get_new_features_name(self) -> list: + def _get_new_features_name(self) -> List: """Return names of the created features.""" feature_names = [] for feature in self.variables_: @@ -294,7 +294,7 @@ def _get_new_features_name(self) -> list: return feature_names - def _add_new_feature_names(self, feature_names: list) -> list: + def _add_new_feature_names(self, feature_names: List) -> List: """Creates new features names and removes original categorical variables.""" feature_names = feature_names + self._get_new_features_name() feature_names = [f for f in feature_names if f not in self.variables_] From 917ea3c9dc6328afabcfe84abd27821b9af9accc Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+GLevV@users.noreply.github.com> Date: Fri, 28 Oct 2022 16:06:11 +0000 Subject: [PATCH 10/34] Update test_similarity_encoder.py --- tests/test_encoding/test_similarity_encoder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_encoding/test_similarity_encoder.py b/tests/test_encoding/test_similarity_encoder.py index 5bfdffb1a..6a6941e73 100644 --- a/tests/test_encoding/test_similarity_encoder.py +++ b/tests/test_encoding/test_similarity_encoder.py @@ -278,8 +278,8 @@ def test_encode_top_categories_w_keywords(): assert encoder.n_features_in_ == 3 assert encoder.encoder_dict_ == { "var_A": ["XYZ"], - "var_B": ["A", "D", "B", "G"], - "var_C": ["C", "D", "G", "B"], + "var_B": ["A", "D"], + "var_C": ["C", "D"], } # test transform output for col in transf.keys(): From 0858096fb7cefc7f0cea8689004428ad782dfdfa Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+GLevV@users.noreply.github.com> Date: Fri, 28 Oct 2022 16:06:13 +0000 Subject: [PATCH 11/34] Update similarity_encoder.py --- feature_engine/encoding/similarity_encoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index 0ad9652a2..bcd8d9ffa 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -187,7 +187,7 @@ def __init__( raise ValueError( f"keywords should be dict or None. Got {keywords!r} instead." ) - if not all(isinstance(item, list) for item in keywords): + if not all(isinstance(item, list) for item in keywords.items()): raise ValueError( "Items in keywords should be lists." f" Got {keywords.items()!r} instead." From 782a0870c62da5c4aedba6127f9131f28b3f4e2c Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+GLevV@users.noreply.github.com> Date: Fri, 28 Oct 2022 16:16:08 +0000 Subject: [PATCH 12/34] Update test_similarity_encoder.py --- tests/test_encoding/test_similarity_encoder.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_encoding/test_similarity_encoder.py b/tests/test_encoding/test_similarity_encoder.py index 6a6941e73..6860c22b3 100644 --- a/tests/test_encoding/test_similarity_encoder.py +++ b/tests/test_encoding/test_similarity_encoder.py @@ -226,7 +226,7 @@ def test_keywords_bad_items(): def test_keywords_dont_match(df_enc_big): - encoder = StringSimilarityEncoder(keywords={"column": "hola"}) + encoder = StringSimilarityEncoder(keywords={"column": ["hola"]}) with pytest.raises(ValueError): encoder.fit(df_enc_big) @@ -265,8 +265,7 @@ def test_encode_top_categories_w_keywords(): assert encoder.top_categories == 2 # test fit attr transf = { - "var_A_D": 9, - "var_A_B": 11, + "var_A_XYZ": 0, "var_B_A": 11, "var_B_D": 9, "var_C_D": 9, From b2c371f6b3cae34b2345482c33ebe8bf5239e9f0 Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+GLevV@users.noreply.github.com> Date: Fri, 28 Oct 2022 16:16:10 +0000 Subject: [PATCH 13/34] Update similarity_encoder.py --- feature_engine/encoding/similarity_encoder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index bcd8d9ffa..5c33819d2 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -187,10 +187,10 @@ def __init__( raise ValueError( f"keywords should be dict or None. Got {keywords!r} instead." ) - if not all(isinstance(item, list) for item in keywords.items()): + if not all(isinstance(item, list) for item in keywords.values()): raise ValueError( "Items in keywords should be lists." - f" Got {keywords.items()!r} instead." + f" Got {keywords.values()!r} instead." ) super().__init__(variables, ignore_format) self.top_categories = top_categories From 7eda2de87baf8961d4ae86ad9358e5ca8778fa6f Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+GLevV@users.noreply.github.com> Date: Fri, 28 Oct 2022 16:22:42 +0000 Subject: [PATCH 14/34] Update similarity_encoder.py --- feature_engine/encoding/similarity_encoder.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index 5c33819d2..442ec039f 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -187,11 +187,11 @@ def __init__( raise ValueError( f"keywords should be dict or None. Got {keywords!r} instead." ) - if not all(isinstance(item, list) for item in keywords.values()): - raise ValueError( - "Items in keywords should be lists." - f" Got {keywords.values()!r} instead." - ) + if not all(isinstance(item, list) for item in keywords.values()): + raise ValueError( + "Items in keywords should be lists." + f" Got {keywords.values()!r} instead." + ) super().__init__(variables, ignore_format) self.top_categories = top_categories self.missing_values = missing_values From f69775a5faae21ab65fa611f7088ff73805d1376 Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+GLevV@users.noreply.github.com> Date: Fri, 28 Oct 2022 16:24:35 +0000 Subject: [PATCH 15/34] Update similarity_encoder.py --- feature_engine/encoding/similarity_encoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index 442ec039f..034329468 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -187,7 +187,7 @@ def __init__( raise ValueError( f"keywords should be dict or None. Got {keywords!r} instead." ) - if not all(isinstance(item, list) for item in keywords.values()): + if keywords and not all(isinstance(item, list) for item in keywords.values()): raise ValueError( "Items in keywords should be lists." f" Got {keywords.values()!r} instead." From ec01da35b694d0fedd3169f52fad01795f72e92e Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+glevv@users.noreply.github.com> Date: Sat, 29 Oct 2022 08:34:59 +0000 Subject: [PATCH 16/34] typing changes --- feature_engine/encoding/similarity_encoder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index 034329468..7491ddf29 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -282,7 +282,7 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: return X.drop(self.variables_, axis=1) - def _get_new_features_name(self) -> List: + def _get_new_features_name(self) -> List[str]: """Return names of the created features.""" feature_names = [] for feature in self.variables_: @@ -294,7 +294,7 @@ def _get_new_features_name(self) -> List: return feature_names - def _add_new_feature_names(self, feature_names: List) -> List: + def _add_new_feature_names(self, feature_names: List[str]) -> List[str]: """Creates new features names and removes original categorical variables.""" feature_names = feature_names + self._get_new_features_name() feature_names = [f for f in feature_names if f not in self.variables_] From 2f89701b0c51bfd3ec637fee343abeeccda8de19 Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+glevv@users.noreply.github.com> Date: Sat, 29 Oct 2022 08:37:07 +0000 Subject: [PATCH 17/34] Update test_similarity_encoder.py --- .../test_encoding/test_similarity_encoder.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/test_encoding/test_similarity_encoder.py b/tests/test_encoding/test_similarity_encoder.py index 6860c22b3..e2c79602c 100644 --- a/tests/test_encoding/test_similarity_encoder.py +++ b/tests/test_encoding/test_similarity_encoder.py @@ -287,6 +287,58 @@ def test_encode_top_categories_w_keywords(): assert "var_B_F" not in X.columns +def test_encode_full_keywords(): + df = pd.DataFrame( + { + "var_A": ["A"] * 5 + + ["B"] * 11 + + ["C"] * 4 + + ["D"] * 9 + + ["E"] * 2 + + ["F"] * 2 + + ["G"] * 7, + "var_B": ["A"] * 11 + + ["B"] * 7 + + ["C"] * 4 + + ["D"] * 9 + + ["E"] * 2 + + ["F"] * 2 + + ["G"] * 5, + "var_C": ["A"] * 4 + + ["B"] * 5 + + ["C"] * 11 + + ["D"] * 9 + + ["E"] * 2 + + ["F"] * 2 + + ["G"] * 7, + } + ) + + encoder = StringSimilarityEncoder(keywords={"var_A": ["X"], "var_B": ["Y"], "var_C": ["Z"]}) + X = encoder.fit_transform(df) + + # test fit attr + transf = { + "var_A_X": 0, + "var_B_Y": 0, + "var_C_Z": 0, + } + + # test fit attr + assert encoder.variables_ == ["var_A", "var_B", "var_C"] + assert encoder.n_features_in_ == 3 + assert encoder.encoder_dict_ == { + "var_A": ["X"], + "var_B": ["Y"], + "var_C": ["Z"], + } + # test transform output + for col in transf.keys(): + assert X[col].sum() == transf[col] + assert "var_B" not in X.columns + assert "var_B_F" not in X.columns + + def test_get_feature_names_out_w_keywords(df_enc_big_na): input_features = df_enc_big_na.columns.tolist() From a2c315db76cd6cd1327b1c7247cd797a63cec768 Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+glevv@users.noreply.github.com> Date: Sat, 29 Oct 2022 08:42:20 +0000 Subject: [PATCH 18/34] flake --- tests/test_encoding/test_similarity_encoder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_encoding/test_similarity_encoder.py b/tests/test_encoding/test_similarity_encoder.py index e2c79602c..32e679edd 100644 --- a/tests/test_encoding/test_similarity_encoder.py +++ b/tests/test_encoding/test_similarity_encoder.py @@ -314,7 +314,9 @@ def test_encode_full_keywords(): } ) - encoder = StringSimilarityEncoder(keywords={"var_A": ["X"], "var_B": ["Y"], "var_C": ["Z"]}) + encoder = StringSimilarityEncoder( + keywords={"var_A": ["X"], "var_B": ["Y"], "var_C": ["Z"]} + ) X = encoder.fit_transform(df) # test fit attr From 858e1c578d827f87d11f8969a03d32d99777c1d9 Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+glevv@users.noreply.github.com> Date: Mon, 31 Oct 2022 17:54:09 +0000 Subject: [PATCH 19/34] revert to old logic --- feature_engine/encoding/similarity_encoder.py | 54 +++++++++++++------ 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index 7491ddf29..9dc5224e6 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -114,9 +114,8 @@ class StringSimilarityEncoder(CategoricalInitMixin, CategoricalMethodsMixin): User defined dictionary of keywords, dict(feature: [keyword1, keyword2, ...]). Instead of finding top_k categories in features, encoder will use this keywords to create similarity variables. Useful when someone has domain knowledge of the - problem. Could be defined only for several features; in this case for specified - features keywords will be used and most common categories will be used for - unspecified. + problem. Could be defined only partially, not for all features. In this case, for + features not specified in keywords, categories will be extracted from data. {variables} @@ -226,18 +225,43 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): if self.missing_values == "raise": _check_contains_na(X, self.variables_) - for var in self.variables_: - if self.keywords and self.keywords.get(var): - self.encoder_dict_[var] = self.keywords[var] - else: - self.encoder_dict_[var] = ( - X[var] - .astype(str) - .replace("nan", "") - .value_counts() - .head(self.top_categories) - .index.tolist() - ) + for var in self.variables_: + if self.keywords and self.keywords.get(var): + self.encoder_dict_[var] = self.keywords[var] + else: + self.encoder_dict_[var] = ( + X[var] + .astype(str) + .value_counts() + .head(self.top_categories) + .index.tolist() + ) + elif self.missing_values == "impute": + for var in self.variables_: + if self.keywords and self.keywords.get(var): + self.encoder_dict_[var] = self.keywords[var] + else: + self.encoder_dict_[var] = ( + X[var] + .astype(str) + .replace("nan", "") + .value_counts() + .head(self.top_categories) + .index.tolist() + ) + elif self.missing_values == "ignore": + for var in self.variables_: + if self.keywords and self.keywords.get(var): + self.encoder_dict_[var] = self.keywords[var] + else: + self.encoder_dict_[var] = ( + X[var] + .astype(str) + .drop("nan") + .value_counts(dropna=True) + .head(self.top_categories) + .index.tolist() + ) return self From bedbf6a9c74e890bc81d990f6979d37b8c2c24e9 Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+glevv@users.noreply.github.com> Date: Mon, 31 Oct 2022 17:54:22 +0000 Subject: [PATCH 20/34] add parametrize --- tests/test_encoding/test_similarity_encoder.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/test_encoding/test_similarity_encoder.py b/tests/test_encoding/test_similarity_encoder.py index 32e679edd..564892fc6 100644 --- a/tests/test_encoding/test_similarity_encoder.py +++ b/tests/test_encoding/test_similarity_encoder.py @@ -215,18 +215,21 @@ def test_get_feature_names_out_na(df_enc_big_na): assert tr.get_feature_names_out(input_features=input_features) == out -def test_keywords_bad_type(): +@pytest.mark.parametrize("keywords", ["hello", 0.5, [1]]) +def test_keywords_bad_type(keywords): with pytest.raises(ValueError): - StringSimilarityEncoder(keywords="hola") + StringSimilarityEncoder(keywords=keywords) -def test_keywords_bad_items(): +@pytest.mark.parametrize("keywords", ["hello", 0.5, 1]) +def test_keywords_bad_items(keywords): with pytest.raises(ValueError): - StringSimilarityEncoder(keywords={"column": "hola"}) + StringSimilarityEncoder(keywords={"column": keywords}) -def test_keywords_dont_match(df_enc_big): - encoder = StringSimilarityEncoder(keywords={"column": ["hola"]}) +@pytest.mark.parametrize("keywords", ["hello", 0.5, 1]) +def test_keywords_dont_match(df_enc_big, keywords): + encoder = StringSimilarityEncoder(keywords={"column": [keywords]}) with pytest.raises(ValueError): encoder.fit(df_enc_big) @@ -287,7 +290,7 @@ def test_encode_top_categories_w_keywords(): assert "var_B_F" not in X.columns -def test_encode_full_keywords(): +def test_encode_complete_keywords(): df = pd.DataFrame( { "var_A": ["A"] * 5 From 566d68595d0897819f9ae4ba1f1c3a84ff89297f Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+glevv@users.noreply.github.com> Date: Mon, 31 Oct 2022 17:55:58 +0000 Subject: [PATCH 21/34] flake fix --- feature_engine/encoding/similarity_encoder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index 9dc5224e6..ffafda2c1 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -114,8 +114,8 @@ class StringSimilarityEncoder(CategoricalInitMixin, CategoricalMethodsMixin): User defined dictionary of keywords, dict(feature: [keyword1, keyword2, ...]). Instead of finding top_k categories in features, encoder will use this keywords to create similarity variables. Useful when someone has domain knowledge of the - problem. Could be defined only partially, not for all features. In this case, for - features not specified in keywords, categories will be extracted from data. + problem. Could be defined only partially, not for all features. In this case, + for features not specified in keywords, they will be extracted from data. {variables} From bbba62227d8c3939d519601f7b1db9b011d9930e Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+glevv@users.noreply.github.com> Date: Mon, 31 Oct 2022 18:02:13 +0000 Subject: [PATCH 22/34] fix ignore case --- feature_engine/encoding/similarity_encoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index ffafda2c1..02565f202 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -257,7 +257,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): self.encoder_dict_[var] = ( X[var] .astype(str) - .drop("nan") + .drop("nan", errors="ignore") .value_counts(dropna=True) .head(self.top_categories) .index.tolist() From 87082d6e0634f8fad76a37da764fdf368064a339 Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+glevv@users.noreply.github.com> Date: Mon, 31 Oct 2022 18:10:00 +0000 Subject: [PATCH 23/34] change of logic --- feature_engine/encoding/similarity_encoder.py | 57 ++++++++----------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index 02565f202..75e326d0b 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -226,42 +226,35 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): if self.missing_values == "raise": _check_contains_na(X, self.variables_) for var in self.variables_: - if self.keywords and self.keywords.get(var): - self.encoder_dict_[var] = self.keywords[var] - else: - self.encoder_dict_[var] = ( - X[var] - .astype(str) - .value_counts() - .head(self.top_categories) - .index.tolist() - ) + self.encoder_dict_[var] = ( + X[var] + .astype(str) + .value_counts() + .head(self.top_categories) + .index.tolist() + ) elif self.missing_values == "impute": for var in self.variables_: - if self.keywords and self.keywords.get(var): - self.encoder_dict_[var] = self.keywords[var] - else: - self.encoder_dict_[var] = ( - X[var] - .astype(str) - .replace("nan", "") - .value_counts() - .head(self.top_categories) - .index.tolist() - ) + self.encoder_dict_[var] = ( + X[var] + .astype(str) + .replace("nan", "") + .value_counts() + .head(self.top_categories) + .index.tolist() + ) elif self.missing_values == "ignore": for var in self.variables_: - if self.keywords and self.keywords.get(var): - self.encoder_dict_[var] = self.keywords[var] - else: - self.encoder_dict_[var] = ( - X[var] - .astype(str) - .drop("nan", errors="ignore") - .value_counts(dropna=True) - .head(self.top_categories) - .index.tolist() - ) + self.encoder_dict_[var] = ( + X[var] + .astype(str) + .drop("nan", errors="ignore") + .value_counts(dropna=True) + .head(self.top_categories) + .index.tolist() + ) + for var in self.keywords.keys(): + self.encoder_dict_[var] = self.keywords[var] return self From 472efdf01ce6e4190211de1278f7fb0b3f0e838e Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+glevv@users.noreply.github.com> Date: Mon, 31 Oct 2022 18:11:39 +0000 Subject: [PATCH 24/34] none check --- feature_engine/encoding/similarity_encoder.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index 75e326d0b..36c52daf8 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -253,8 +253,9 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): .head(self.top_categories) .index.tolist() ) - for var in self.keywords.keys(): - self.encoder_dict_[var] = self.keywords[var] + if self.keywords: + for var in self.keywords.keys(): + self.encoder_dict_[var] = self.keywords[var] return self From 05ae18c5a6d01601de209116fe908acf1ee88635 Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+glevv@users.noreply.github.com> Date: Mon, 31 Oct 2022 18:20:45 +0000 Subject: [PATCH 25/34] test update --- tests/test_encoding/test_similarity_encoder.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_encoding/test_similarity_encoder.py b/tests/test_encoding/test_similarity_encoder.py index 564892fc6..db697f1d6 100644 --- a/tests/test_encoding/test_similarity_encoder.py +++ b/tests/test_encoding/test_similarity_encoder.py @@ -221,20 +221,20 @@ def test_keywords_bad_type(keywords): StringSimilarityEncoder(keywords=keywords) -@pytest.mark.parametrize("keywords", ["hello", 0.5, 1]) -def test_keywords_bad_items(keywords): +@pytest.mark.parametrize("item", ["hello", 0.5, 1]) +def test_keywords_bad_items(item): with pytest.raises(ValueError): - StringSimilarityEncoder(keywords={"column": keywords}) + StringSimilarityEncoder(keywords={"var_A": item}) -@pytest.mark.parametrize("keywords", ["hello", 0.5, 1]) -def test_keywords_dont_match(df_enc_big, keywords): - encoder = StringSimilarityEncoder(keywords={"column": [keywords]}) +@pytest.mark.parametrize("key", ["hello", 0.5, 1]) +def test_keywords_bad_keys(df_enc_big, key): + encoder = StringSimilarityEncoder(keywords={key: ["A"]}) with pytest.raises(ValueError): encoder.fit(df_enc_big) -def test_encode_top_categories_w_keywords(): +def test_encode_partial_keywords(): df = pd.DataFrame( { "var_A": ["A"] * 5 From aa31374cabc472bcf286582c27118c45ece48a48 Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+glevv@users.noreply.github.com> Date: Tue, 1 Nov 2022 10:44:01 +0000 Subject: [PATCH 26/34] simplify dict update --- feature_engine/encoding/similarity_encoder.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index 36c52daf8..ad50ebb71 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -254,8 +254,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): .index.tolist() ) if self.keywords: - for var in self.keywords.keys(): - self.encoder_dict_[var] = self.keywords[var] + self.encoder_dict_.update(self.keywords) return self From f23069ffb2c9a812d0ea43ecca183dd5ff5a038b Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+glevv@users.noreply.github.com> Date: Tue, 1 Nov 2022 10:44:13 +0000 Subject: [PATCH 27/34] added tests for impute and ignore --- tests/test_encoding/test_similarity_encoder.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_encoding/test_similarity_encoder.py b/tests/test_encoding/test_similarity_encoder.py index db697f1d6..1aab31103 100644 --- a/tests/test_encoding/test_similarity_encoder.py +++ b/tests/test_encoding/test_similarity_encoder.py @@ -111,12 +111,22 @@ def test_nan_behaviour_impute(df_enc_big_na): encoder = StringSimilarityEncoder(missing_values="impute") X = encoder.fit_transform(df_enc_big_na) assert (X.isna().sum() == 0).all(axis=None) + assert encoder.encoder_dict_ == { + "var_A": ["B", "D", "G", "A", "C", "E", "F", ""], + "var_B": ["A", "D", "B", "G", "C", "E", "F"], + "var_C": ["C", "D", "B", "G", "A", "E", "F"], + } def test_nan_behaviour_ignore(df_enc_big_na): encoder = StringSimilarityEncoder(missing_values="ignore") X = encoder.fit_transform(df_enc_big_na) assert (X.isna().any(1) == df_enc_big_na.isna().any(1)).all() + assert encoder.encoder_dict_ == { + "var_A": ["B", "D", "G", "A", "C", "E", "F"], + "var_B": ["A", "D", "B", "G", "C", "E", "F"], + "var_C": ["C", "D", "B", "G", "A", "E", "F"], + } def test_inverse_transform_error(df_enc_big): From 7a30c6c262d67680c2b49c7659d30a2b9cb7e76a Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+glevv@users.noreply.github.com> Date: Tue, 1 Nov 2022 10:50:10 +0000 Subject: [PATCH 28/34] ignore case fix --- feature_engine/encoding/similarity_encoder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index ad50ebb71..1a28e0ce9 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -248,8 +248,9 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): self.encoder_dict_[var] = ( X[var] .astype(str) - .drop("nan", errors="ignore") .value_counts(dropna=True) + .drop("nan", errors="ignore") + .sort_values() .head(self.top_categories) .index.tolist() ) From 69277ddc9b0935ded8f85eacfa7d43ef1884e1b0 Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+glevv@users.noreply.github.com> Date: Tue, 1 Nov 2022 10:55:39 +0000 Subject: [PATCH 29/34] Update similarity_encoder.py --- feature_engine/encoding/similarity_encoder.py | 1 - 1 file changed, 1 deletion(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index 1a28e0ce9..2f1904dde 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -250,7 +250,6 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): .astype(str) .value_counts(dropna=True) .drop("nan", errors="ignore") - .sort_values() .head(self.top_categories) .index.tolist() ) From ee21c52b159327ec5ce682b58bcbcfe16077a893 Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+glevv@users.noreply.github.com> Date: Tue, 8 Nov 2022 16:18:45 +0000 Subject: [PATCH 30/34] update logic --- feature_engine/encoding/similarity_encoder.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index 2f1904dde..d706cc212 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -223,9 +223,14 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): ) self.encoder_dict_ = {} + if self.keywords: + self.encoder_dict_.update(self.keywords) + cols_to_iterate = [x if x not in self.keywords for x in self.variables_] + else: + cols_to_iterate = self.variables_ if self.missing_values == "raise": _check_contains_na(X, self.variables_) - for var in self.variables_: + for var in cols_to_iterate: self.encoder_dict_[var] = ( X[var] .astype(str) @@ -234,7 +239,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): .index.tolist() ) elif self.missing_values == "impute": - for var in self.variables_: + for var in cols_to_iterate: self.encoder_dict_[var] = ( X[var] .astype(str) @@ -244,7 +249,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): .index.tolist() ) elif self.missing_values == "ignore": - for var in self.variables_: + for var in cols_to_iterate: self.encoder_dict_[var] = ( X[var] .astype(str) @@ -253,8 +258,6 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): .head(self.top_categories) .index.tolist() ) - if self.keywords: - self.encoder_dict_.update(self.keywords) return self From a065d438b0bd3b7bd664806fcc44c511a12c3a30 Mon Sep 17 00:00:00 2001 From: Gleb Levitski <36483986+glevv@users.noreply.github.com> Date: Tue, 8 Nov 2022 16:21:10 +0000 Subject: [PATCH 31/34] hotfix --- feature_engine/encoding/similarity_encoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index d706cc212..e942877f4 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -225,7 +225,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): if self.keywords: self.encoder_dict_.update(self.keywords) - cols_to_iterate = [x if x not in self.keywords for x in self.variables_] + cols_to_iterate = [x for x in self.variables_ if x not in self.keywords] else: cols_to_iterate = self.variables_ if self.missing_values == "raise": From 278700b8c9660e665ae89ad649078d3190880502 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Thu, 10 Nov 2022 09:46:33 +0100 Subject: [PATCH 32/34] rewords keywords docstring --- feature_engine/encoding/similarity_encoder.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index 4636582ea..936cb55e5 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -111,11 +111,12 @@ class StringSimilarityEncoder(CategoricalInitMixin, CategoricalMethodsMixin): measures. keywords: dict, default=None - User defined dictionary of keywords, dict(feature: [keyword1, keyword2, ...]). - Instead of finding top_k categories in features, encoder will use this keywords - to create similarity variables. Useful when someone has domain knowledge of the - problem. Could be defined only partially, not for all features. In this case, - for features not specified in keywords, they will be extracted from data. + Dictionary with a set of keywords to be used to create the similarity variables. + The format should be: dict(feature: [keyword1, keyword2, ...]). The encoder will + use these keywords to create the similarity variables. The dictionary can be + defined for all the features to encode, or only for a subset of them. In this + case, for the features not specified in the dictionary, the encoder will + identify the categories from the data. {variables} From 30b231b0c32a0e5177817c81899f32f561ac4d7f Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Thu, 10 Nov 2022 09:48:43 +0100 Subject: [PATCH 33/34] rewords error msgs --- feature_engine/encoding/similarity_encoder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index 936cb55e5..94db76c49 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -200,11 +200,11 @@ def __init__( ) if keywords and not isinstance(keywords, dict): raise ValueError( - f"keywords should be dict or None. Got {keywords!r} instead." + f"keywords should be a dictionary or None. Got {keywords!r} instead." ) if keywords and not all(isinstance(item, list) for item in keywords.values()): raise ValueError( - "Items in keywords should be lists." + "The items in keywords should be lists." f" Got {keywords.values()!r} instead." ) super().__init__(variables, ignore_format) From f948dc1d5ee025bce8c69736f2a9bc10d09afc26 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Thu, 10 Nov 2022 09:52:12 +0100 Subject: [PATCH 34/34] rewords error message --- feature_engine/encoding/similarity_encoder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index 94db76c49..0adae380f 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -235,7 +235,8 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): if self.keywords: if not all(item in self.variables_ for item in self.keywords.keys()): raise ValueError( - "keywords have columns that are not present in the dataset" + "There are variables in keywords that are not present " + "in the dataset." ) self.encoder_dict_ = {}