[FEATURE] Introduce ParameterBuilder Evaluation Dependencies and Validation Dependencies 2022 03 23 66

great_expectations/expectations/core/expect_column_kl_divergence_to_be_less_than.py

@@ -245,7 +245,7 @@ def get_validation_dependencies(
                     metric_value_kwargs=None,
                 )
                 #
-                # NOTE 20201117 - JPC - Would prefer not to include partition_metric_configuraiton here,
+                # NOTE 20201117 - JPC - Would prefer not to include partition_metric_configuration here,
                 # since we have already evaluated it, and its result is in the kwargs for the histogram.
                 # However, currently the dependencies' configurations are not passed to the _validate method
                 #

great_expectations/rule_based_profiler/config/base.py

@@ -155,6 +155,7 @@ def __init__(
         name: str,
         class_name: str,
         module_name: Optional[str] = None,
+        evaluation_parameter_builder_configs: Optional[list] = None,
         json_serialize: bool = True,
         batch_request: Optional[Union[dict, str]] = None,
         **kwargs,
@@ -167,6 +168,11 @@ def __init__(
         if class_name is not None:
             self.class_name = class_name
 
+        if evaluation_parameter_builder_configs:
+            self.evaluation_parameter_builder_configs = (
+                evaluation_parameter_builder_configs
+            )
+
         self.json_serialize = json_serialize
 
         if batch_request is not None:
@@ -201,6 +207,15 @@ class Meta:
         required=True,
         allow_none=False,
     )
+    evaluation_parameter_builder_configs = fields.List(
+        cls_or_instance=fields.Nested(
+            lambda: ParameterBuilderConfigSchema(),
+            required=True,
+            allow_none=False,
+        ),
+        required=False,
+        allow_none=True,
+    )
     json_serialize = fields.Boolean(
         required=False,
         allow_none=True,
@@ -219,6 +234,7 @@ def __init__(
         class_name: str,
         module_name: Optional[str] = None,
         meta: Optional[dict] = None,
+        validation_parameter_builder_configs: Optional[list] = None,
         batch_request: Optional[Union[dict, str]] = None,
         **kwargs,
     ):
@@ -233,6 +249,11 @@ def __init__(
         if meta is not None:
             self.meta = meta
 
+        if validation_parameter_builder_configs:
+            self.validation_parameter_builder_configs = (
+                validation_parameter_builder_configs
+            )
+
         if batch_request is not None:
             self.batch_request = batch_request
 
@@ -275,6 +296,15 @@ class Meta:
         required=False,
         allow_none=True,
     )
+    validation_parameter_builder_configs = fields.List(
+        cls_or_instance=fields.Nested(
+            lambda: ParameterBuilderConfigSchema(),
+            required=True,
+            allow_none=False,
+        ),
+        required=False,
+        allow_none=True,
+    )
     batch_request = fields.Raw(
         required=False,
         allow_none=True,
@@ -381,12 +411,10 @@ def __init__(
         commented_map: Optional[CommentedMap] = None,
     ):
         self.name = name
+        self.module_name = module_name
+        self.class_name = class_name
         self.config_version = config_version
         self.rules = rules
-        if class_name is not None:
-            self.class_name = class_name
-        if module_name is not None:
-            self.module_name = module_name
         self.variables = variables
 
         super().__init__(commented_map=commented_map)

great_expectations/rule_based_profiler/expectation_configuration_builder/default_expectation_configuration_builder.py

@@ -67,6 +67,7 @@ def __init__(
         expectation_type: str,
         meta: Optional[Dict[str, Any]] = None,
         condition: Optional[str] = None,
+        validation_parameter_builder_configs: Optional[List[dict]] = None,
         batch_list: Optional[List[Batch]] = None,
         batch_request: Optional[
             Union[str, BatchRequest, RuntimeBatchRequest, dict]
@@ -80,6 +81,8 @@ def __init__(
             meta: the "meta" argument of "ExpectationConfiguration" object to be emitted.
             condition: Boolean statement (expressed as string and following specified grammar), which controls whether
             or not underlying logic should be executed and thus resulting "ExpectationConfiguration" emitted.
+            validation_parameter_builder_configs: ParameterBuilder configurations, having whose outputs available (as
+            fully-qualified parameter names) is pre-requisite for present ExpectationConfigurationBuilder instance.
             batch_list: explicitly passed Batch objects for parameter computation (take precedence over batch_request).
             batch_request: specified in ParameterBuilder configuration to get Batch objects for parameter computation.
             data_context: DataContext
@@ -88,6 +91,7 @@ def __init__(
 
         super().__init__(
             expectation_type=expectation_type,
+            validation_parameter_builder_configs=validation_parameter_builder_configs,
             batch_list=batch_list,
             batch_request=batch_request,
             data_context=data_context,

great_expectations/rule_based_profiler/expectation_configuration_builder/expectation_configuration_builder.py

@@ -4,6 +4,12 @@
 
 from great_expectations.core.batch import Batch, BatchRequest, RuntimeBatchRequest
 from great_expectations.core.expectation_configuration import ExpectationConfiguration
+from great_expectations.rule_based_profiler.config import ParameterBuilderConfig
+from great_expectations.rule_based_profiler.helpers.util import (
+    init_rule_parameter_builders,
+    set_batch_list_or_batch_request_on_builder,
+)
+from great_expectations.rule_based_profiler.parameter_builder import ParameterBuilder
 from great_expectations.rule_based_profiler.types import (
     Builder,
     Domain,
@@ -15,9 +21,16 @@
 
 
 class ExpectationConfigurationBuilder(Builder, ABC):
+    exclude_field_names: Set[str] = Builder.exclude_field_names | {
+        "validation_parameter_builders",
+    }
+
     def __init__(
         self,
         expectation_type: str,
+        validation_parameter_builder_configs: Optional[
+            List[ParameterBuilderConfig]
+        ] = None,
         batch_list: Optional[List[Batch]] = None,
         batch_request: Optional[
             Union[str, BatchRequest, RuntimeBatchRequest, dict]
@@ -30,6 +43,8 @@ def __init__(
 
         Args:
             expectation_type: the "expectation_type" argument of "ExpectationConfiguration" object to be emitted.
+            validation_parameter_builder_configs: ParameterBuilder configurations, having whose outputs available (as
+            fully-qualified parameter names) is pre-requisite for present ExpectationConfigurationBuilder instance.
             batch_list: explicitly passed Batch objects for parameter computation (take precedence over batch_request).
             batch_request: specified in ParameterBuilder configuration to get Batch objects for parameter computation.
             data_context: DataContext
@@ -44,6 +59,11 @@ def __init__(
 
         self._expectation_type = expectation_type
 
+        self._validation_parameter_builders = init_rule_parameter_builders(
+            parameter_builder_configs=validation_parameter_builder_configs,
+            data_context=self._data_context,
+        )
+
         """
         Since ExpectationConfigurationBuilderConfigSchema allows arbitrary fields (as ExpectationConfiguration kwargs)
         to be provided, they must be all converted to public property accessors and/or public fields in order for all
@@ -65,10 +85,43 @@ def build_expectation_configuration(
         variables: Optional[ParameterContainer] = None,
         parameters: Optional[Dict[str, ParameterContainer]] = None,
     ) -> ExpectationConfiguration:
+        self._resolve_validation_dependencies(
+            parameter_container=parameter_container,
+            domain=domain,
+            variables=variables,
+            parameters=parameters,
+        )
+
         return self._build_expectation_configuration(
             domain=domain, variables=variables, parameters=parameters
         )
 
+    def _resolve_validation_dependencies(
+        self,
+        parameter_container: ParameterContainer,
+        domain: Domain,
+        variables: Optional[ParameterContainer] = None,
+        parameters: Optional[Dict[str, ParameterContainer]] = None,
+    ) -> None:
+        validation_parameter_builders: List[ParameterBuilder] = (
+            self.validation_parameter_builders or []
+        )
+
+        validation_parameter_builder: ParameterBuilder
+        for validation_parameter_builder in validation_parameter_builders:
+            set_batch_list_or_batch_request_on_builder(
+                builder=validation_parameter_builder,
+                batch_list=self.batch_list,
+                batch_request=self.batch_request,
+                force_batch_data=False,
+            )
+            validation_parameter_builder.build_parameters(
+                parameter_container=parameter_container,
+                domain=domain,
+                variables=variables,
+                parameters=parameters,
+            )
+
     @abstractmethod
     def _build_expectation_configuration(
         self,
@@ -81,3 +134,7 @@ def _build_expectation_configuration(
     @property
     def expectation_type(self) -> str:
         return self._expectation_type
+
+    @property
+    def validation_parameter_builders(self) -> Optional[List[ParameterBuilder]]:
+        return self._validation_parameter_builders

great_expectations/rule_based_profiler/helpers/util.py

@@ -19,10 +19,12 @@
 from great_expectations.data_context.util import instantiate_class_from_config
 from great_expectations.execution_engine.execution_engine import MetricDomainTypes
 from great_expectations.rule_based_profiler.types import (
+    PARAMETER_KEY,
     VARIABLES_PREFIX,
     Builder,
     Domain,
     ParameterContainer,
+    get_fully_qualified_parameter_names,
     get_parameter_value_by_fully_qualified_parameter_name,
     is_fully_qualified_parameter_name_literal_string_format,
 )
@@ -461,6 +463,71 @@ def init_expectation_configuration_builder(
     return expectation_configuration_builder
 
 
+def resolve_evaluation_dependencies(
+    parameter_builder: "ParameterBuilder",  # noqa: F821
+    parameter_container: ParameterContainer,
+    domain: Domain,
+    variables: Optional[ParameterContainer] = None,
+    parameters: Optional[Dict[str, ParameterContainer]] = None,
+) -> None:
+    """
+    This method computes ("resolves") pre-requisite ("evaluation") dependencies (i.e., results of executing other
+    "ParameterBuilder" objects), whose output(s) are needed by specified "ParameterBuilder" object to fulfill its goals.
+    """
+
+    # Step 1: Check if any "evaluation_parameter_builders" are configured for specified "ParameterBuilder" object.
+    evaluation_parameter_builders: List[
+        "ParameterBuilder"  # noqa: F821
+    ] = parameter_builder.evaluation_parameter_builders
+    if not evaluation_parameter_builders:
+        return
+
+    # Step 2: Obtain all fully-qualified parameter names ("variables" and "parameter" keys) in namespace of "Domain"
+    # (fully-qualified parameter names are stored in "ParameterNode" objects of "ParameterContainer" of "Domain"
+    # whenever "ParameterBuilder.build_parameters()" is executed for "ParameterBuilder.fully_qualified_parameter_name").
+    fully_qualified_parameter_names: List[str] = get_fully_qualified_parameter_names(
+        domain=domain,
+        variables=variables,
+        parameters=parameters,
+    )
+
+    # Step 3: Check for presence of fully-qualified parameter names of "ParameterBuilder" objects, obtained by iterating
+    # over evaluation dependencies.  "Execute ParameterBuilder.build_parameters()" if absent from "Domain" scoped list.
+    evaluation_parameter_builder: "ParameterBuilder"  # noqa: F821
+    for evaluation_parameter_builder in evaluation_parameter_builders:
+        fully_qualified_evaluation_parameter_builder_name: str = (
+            f"{PARAMETER_KEY}{evaluation_parameter_builder.name}"
+        )
+
+        if (
+            fully_qualified_evaluation_parameter_builder_name
+            not in fully_qualified_parameter_names
+        ):
+            set_batch_list_or_batch_request_on_builder(
+                builder=evaluation_parameter_builder,
+                batch_list=parameter_builder.batch_list,
+                batch_request=parameter_builder.batch_request,
+                force_batch_data=False,
+            )
+
+            evaluation_parameter_builder.build_parameters(
+                parameter_container=parameter_container,
+                domain=domain,
+                variables=variables,
+                parameters=parameters,
+            )
+
+            # Step 4: Any "ParameterBuilder" object, including members of "evaluation_parameter_builders" list may be
+            # configured with its own "evaluation_parameter_builders" list.  Recursive call handles such situations.
+            resolve_evaluation_dependencies(
+                parameter_builder=evaluation_parameter_builder,
+                parameter_container=parameter_container,
+                domain=domain,
+                variables=variables,
+                parameters=parameters,
+            )
+
+
 def set_batch_list_or_batch_request_on_builder(
     builder: Builder,
     batch_list: Optional[List[Batch]] = None,

great_expectations/rule_based_profiler/parameter_builder/mean_unexpected_map_metric_multi_batch_parameter_builder.py

@@ -42,6 +42,7 @@ def __init__(
         null_count_parameter_builder_name: Optional[str] = None,
         metric_domain_kwargs: Optional[Union[str, dict]] = None,
         metric_value_kwargs: Optional[Union[str, dict]] = None,
+        evaluation_parameter_builder_configs: Optional[List[dict]] = None,
         json_serialize: Union[str, bool] = True,
         batch_list: Optional[List[Batch]] = None,
         batch_request: Optional[
@@ -60,6 +61,8 @@ def __init__(
             null_count_parameter_builder_name: name of parameter that computes null_count (of domain values in Batch).
             metric_domain_kwargs: used in MetricConfiguration
             metric_value_kwargs: used in MetricConfiguration
+            evaluation_parameter_builder_configs: ParameterBuilder configurations, executing and making whose respective
+            ParameterBuilder objects' outputs available (as fully-qualified parameter names) is pre-requisite.
             json_serialize: If True (default), convert computed value to JSON prior to saving results.
             batch_list: explicitly passed Batch objects for parameter computation (take precedence over batch_request).
             batch_request: specified in ParameterBuilder configuration to get Batch objects for parameter computation.
@@ -73,6 +76,7 @@ def __init__(
             enforce_numeric_metric=True,
             replace_nan_with_zero=True,
             reduce_scalar_metric=True,
+            evaluation_parameter_builder_configs=evaluation_parameter_builder_configs,
             json_serialize=json_serialize,
             batch_list=batch_list,
             batch_request=batch_request,

great_expectations/rule_based_profiler/parameter_builder/metric_multi_batch_parameter_builder.py

@@ -33,6 +33,7 @@ def __init__(
         enforce_numeric_metric: Union[str, bool] = False,
         replace_nan_with_zero: Union[str, bool] = False,
         reduce_scalar_metric: Union[str, bool] = True,
+        evaluation_parameter_builder_configs: Optional[List[dict]] = None,
         json_serialize: Union[str, bool] = True,
         batch_list: Optional[List[Batch]] = None,
         batch_request: Optional[
@@ -52,13 +53,16 @@ def __init__(
             replace_nan_with_zero: if False (default), then if the computed metric gives NaN, then exception is raised;
             otherwise, if True, then if the computed metric gives NaN, then it is converted to the 0.0 (float) value.
             reduce_scalar_metric: if True (default), then reduces computation of 1-dimensional metric to scalar value.
+            evaluation_parameter_builder_configs: ParameterBuilder configurations, executing and making whose respective
+            ParameterBuilder objects' outputs available (as fully-qualified parameter names) is pre-requisite.
             json_serialize: If True (default), convert computed value to JSON prior to saving results.
             batch_list: explicitly passed Batch objects for parameter computation (take precedence over batch_request).
             batch_request: specified in ParameterBuilder configuration to get Batch objects for parameter computation.
             data_context: DataContext
         """
         super().__init__(
             name=name,
+            evaluation_parameter_builder_configs=evaluation_parameter_builder_configs,
             json_serialize=json_serialize,
             batch_list=batch_list,
             batch_request=batch_request,

great_expectations/rule_based_profiler/parameter_builder/numeric_metric_range_multi_batch_parameter_builder.py

@@ -71,6 +71,7 @@ def __init__(
         truncate_values: Optional[
             Union[str, Dict[str, Union[Optional[int], Optional[float]]]]
         ] = None,
+        evaluation_parameter_builder_configs: Optional[List[dict]] = None,
         json_serialize: Union[str, bool] = True,
         batch_list: Optional[List[Batch]] = None,
         batch_request: Optional[
@@ -100,6 +101,8 @@ def __init__(
             output.  If omitted, then no rounding is performed, unless the computed value is already an integer.
             truncate_values: user-configured directive for whether or not to allow the computed parameter values
             (i.e., lower_bound, upper_bound) to take on values outside the specified bounds when packaged on output.
+            evaluation_parameter_builder_configs: ParameterBuilder configurations, executing and making whose respective
+            ParameterBuilder objects' outputs available (as fully-qualified parameter names) is pre-requisite.
             json_serialize: If True (default), convert computed value to JSON prior to saving results.
             batch_list: explicitly passed Batch objects for parameter computation (take precedence over batch_request).
             batch_request: specified in ParameterBuilder configuration to get Batch objects for parameter computation.
@@ -113,6 +116,7 @@ def __init__(
             enforce_numeric_metric=enforce_numeric_metric,
             replace_nan_with_zero=replace_nan_with_zero,
             reduce_scalar_metric=reduce_scalar_metric,
+            evaluation_parameter_builder_configs=evaluation_parameter_builder_configs,
             json_serialize=json_serialize,
             batch_list=batch_list,
             batch_request=batch_request,