update on the auto-intent resolving (#190)

foreshadow/smart/intent_resolving/core/heuristics.py

@@ -28,7 +28,7 @@ def convert_to_numeric(series: pd.Series) -> pd.Series:
 
 
 def is_number_as_string(
-    series: pd.Series, shrinkage_threshold: float = 0.5
+    series: pd.Series, shrinkage_threshold: float = 0.7
 ) -> bool:
     """Check if string can be numerical.
 
@@ -224,7 +224,7 @@ def _maybe_zipcode(raw_s: pd.Series, threshold: float = 0.95) -> int:
     return points
 
 
-def maybe_real_as_enum(
+def maybe_real_as_categorical(
     df: pd.DataFrame, max_n_distinct: int = 20
 ) -> pd.Series:
     """Evaluate if feature column might be categorical.

foreshadow/smart/intent_resolving/core/secondary_featurizers/__init__.py

@@ -6,7 +6,6 @@
     -- MetaDataSetFeaturizerViaLambda
     -- NGramFeaturizer
     -- Chars2VecFeaturizer
-    -- FastTextFeaturizer
 
 This __init__.py file also contains defintions for the followig helper class:
     -- FeaturizerCurator

foreshadow/smart/intent_resolving/core/secondary_featurizers/raw_data_set_featurizer_via_lambda.py

@@ -324,8 +324,12 @@ def __map_features_to_feature_columns(
                 else ("attribute_name",)
             ),
         )
-
-        if merged.isnull().any(axis=None):
+        sample_cols = [
+            col
+            for col in merged.columns
+            if "sample" in col and "samples" not in col
+        ]
+        if merged.drop(sample_cols, axis=1).isnull().any(axis=None):
             raise AssertionError(
                 "Imperfect mapping from features to data set detected."
             )