diff --git a/pyproject.toml b/pyproject.toml index 78d9308..f98a4ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath-core" -version = "0.1.4" +version = "0.1.5" description = "UiPath Core abstractions" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/src/uipath/core/guardrails/_deterministic_guardrails_service.py b/src/uipath/core/guardrails/_deterministic_guardrails_service.py index 8996a4c..56e61bf 100644 --- a/src/uipath/core/guardrails/_deterministic_guardrails_service.py +++ b/src/uipath/core/guardrails/_deterministic_guardrails_service.py @@ -10,10 +10,14 @@ evaluate_word_rule, ) from .guardrails import ( + AllFieldsSelector, + ApplyTo, BooleanRule, DeterministicGuardrail, + FieldSource, GuardrailValidationResult, NumberRule, + SpecificFieldsSelector, UniversalRule, WordRule, ) @@ -27,6 +31,16 @@ def evaluate_pre_deterministic_guardrail( guardrail: DeterministicGuardrail, ) -> GuardrailValidationResult: """Evaluate deterministic guardrail rules against input data (pre-execution).""" + # Check if guardrail contains any output-dependent rules + has_output_rule = self._has_output_dependent_rule(guardrail, [ApplyTo.OUTPUT]) + + # If guardrail has output-dependent rules, skip evaluation in pre-execution + # Output rules will be evaluated during post-execution + if has_output_rule: + return GuardrailValidationResult( + validation_passed=True, + reason="Guardrail contains output-dependent rules that will be evaluated during post-execution", + ) return self._evaluate_deterministic_guardrail( input_data=input_data, output_data={}, @@ -41,12 +55,61 @@ def evaluate_post_deterministic_guardrail( guardrail: DeterministicGuardrail, ) -> GuardrailValidationResult: """Evaluate deterministic guardrail rules against input and output data.""" + # Check if guardrail contains any output-dependent rules + has_output_rule = self._has_output_dependent_rule( + guardrail, [ApplyTo.OUTPUT, ApplyTo.INPUT_AND_OUTPUT] + ) + + # If guardrail has no output-dependent rules, skip post-execution evaluation + # Only input rules exist and they should have been evaluated during pre-execution + if not has_output_rule: + return GuardrailValidationResult( + validation_passed=True, + reason="Guardrail contains only input-dependent rules that were evaluated during pre-execution", + ) + return self._evaluate_deterministic_guardrail( input_data=input_data, output_data=output_data, guardrail=guardrail, ) + @staticmethod + def _has_output_dependent_rule( + guardrail: DeterministicGuardrail, + universal_rules_apply_to_values: list[ApplyTo], + ) -> bool: + """Check if at least one rule EXCLUSIVELY requires output data. + + Args: + guardrail: The guardrail to check + universal_rules_apply_to_values: List of ApplyTo values to consider as output-dependent for UniversalRules. + + Returns: + True if at least one rule exclusively depends on output data, False otherwise. + """ + for rule in guardrail.rules: + # UniversalRule: only return True if it applies to values in universal_rules_apply_to_values + if isinstance(rule, UniversalRule): + if rule.apply_to in universal_rules_apply_to_values: + return True + # Rules with field_selector + elif isinstance(rule, (WordRule, NumberRule, BooleanRule)): + field_selector = rule.field_selector + # AllFieldsSelector applies to both input and output, not exclusively output + # SpecificFieldsSelector: only return True if at least one field has OUTPUT source + if isinstance(field_selector, SpecificFieldsSelector): + if field_selector.fields and any( + field.source == FieldSource.OUTPUT + for field in field_selector.fields + ): + return True + elif isinstance(field_selector, AllFieldsSelector): + if FieldSource.OUTPUT in field_selector.sources: + return True + + return False + @staticmethod def _evaluate_deterministic_guardrail( input_data: dict[str, Any], diff --git a/src/uipath/core/guardrails/_evaluators.py b/src/uipath/core/guardrails/_evaluators.py index d3b9c1b..4275705 100644 --- a/src/uipath/core/guardrails/_evaluators.py +++ b/src/uipath/core/guardrails/_evaluators.py @@ -120,23 +120,25 @@ def get_fields_from_selector( fields: list[tuple[Any, FieldReference]] = [] if isinstance(field_selector, AllFieldsSelector): - # For "all" selector, we need to collect all fields from both input and output + # For "all" selector, we need to collect all fields from the specified sources # This is a simplified implementation - in practice, you might want to # recursively collect all nested fields - for key, value in input_data.items(): - fields.append( - ( - value, - FieldReference(path=key, source=FieldSource.INPUT), + if FieldSource.INPUT in field_selector.sources: + for key, value in input_data.items(): + fields.append( + ( + value, + FieldReference(path=key, source=FieldSource.INPUT), + ) ) - ) - for key, value in output_data.items(): - fields.append( - ( - value, - FieldReference(path=key, source=FieldSource.OUTPUT), + if FieldSource.OUTPUT in field_selector.sources: + for key, value in output_data.items(): + fields.append( + ( + value, + FieldReference(path=key, source=FieldSource.OUTPUT), + ) ) - ) elif isinstance(field_selector, SpecificFieldsSelector): # For specific fields, extract values based on field references for field_ref in field_selector.fields: diff --git a/src/uipath/core/guardrails/guardrails.py b/src/uipath/core/guardrails/guardrails.py index 1cab05b..1ab4f57 100644 --- a/src/uipath/core/guardrails/guardrails.py +++ b/src/uipath/core/guardrails/guardrails.py @@ -59,6 +59,7 @@ class AllFieldsSelector(BaseModel): """All fields selector.""" selector_type: Literal["all"] = Field(alias="$selectorType") + sources: list[FieldSource] model_config = ConfigDict(populate_by_name=True, extra="allow") diff --git a/tests/guardrails/test_deterministic_guardrails_service.py b/tests/guardrails/test_deterministic_guardrails_service.py index b73e2ba..c554ffa 100644 --- a/tests/guardrails/test_deterministic_guardrails_service.py +++ b/tests/guardrails/test_deterministic_guardrails_service.py @@ -80,7 +80,10 @@ def test_evaluate_post_deterministic_guardrail_validation_passed( ) assert result.validation_passed is True - assert result.reason == "All deterministic guardrail rules passed" + assert ( + result.reason + == "Guardrail contains only input-dependent rules that were evaluated during pre-execution" + ) def test_evaluate_post_deterministic_guardrail_validation_failed_age( self, @@ -115,6 +118,16 @@ def test_evaluate_post_deterministic_guardrail_validation_failed_age( ), detects_violation=lambda b: b is not True, ), + NumberRule( + rule_type="number", + field_selector=SpecificFieldsSelector( + selector_type="specific", + fields=[ + FieldReference(path="status", source=FieldSource.OUTPUT) + ], + ), + detects_violation=lambda n: n != 200.0, + ), ], ) @@ -171,6 +184,16 @@ def test_evaluate_post_deterministic_guardrail_validation_failed_is_active( ), detects_violation=lambda b: b is not True, ), + NumberRule( + rule_type="number", + field_selector=SpecificFieldsSelector( + selector_type="specific", + fields=[ + FieldReference(path="status", source=FieldSource.OUTPUT) + ], + ), + detects_violation=lambda n: n != 200.0, + ), ], ) @@ -180,7 +203,9 @@ def test_evaluate_post_deterministic_guardrail_validation_failed_is_active( "age": 25, "isActive": False, } - output_data: dict[str, Any] = {} + output_data = { + "status": 200, + } result = service.evaluate_post_deterministic_guardrail( input_data=input_data, @@ -235,7 +260,10 @@ def test_evaluate_post_deterministic_guardrail_matches_regex_positive( ) assert result.validation_passed is True - assert result.reason == "All deterministic guardrail rules passed" + assert ( + result.reason + == "Guardrail contains only input-dependent rules that were evaluated during pre-execution" + ) def test_evaluate_post_deterministic_guardrail_matches_regex_negative( self, @@ -262,6 +290,16 @@ def test_evaluate_post_deterministic_guardrail_matches_regex_negative( ), detects_violation=lambda s: not bool(re.search(".*te.*3.*", s)), ), + NumberRule( + rule_type="number", + field_selector=SpecificFieldsSelector( + selector_type="specific", + fields=[ + FieldReference(path="status", source=FieldSource.OUTPUT) + ], + ), + detects_violation=lambda n: n != 200.0, + ), ], ) @@ -269,7 +307,9 @@ def test_evaluate_post_deterministic_guardrail_matches_regex_negative( input_data = { "userName": "test", } - output_data: dict[str, Any] = {} + output_data = { + "status": 200, + } result = service.evaluate_post_deterministic_guardrail( input_data=input_data, @@ -324,7 +364,10 @@ def test_evaluate_post_deterministic_guardrail_word_func_positive( ) assert result.validation_passed is True - assert result.reason == "All deterministic guardrail rules passed" + assert ( + result.reason + == "Guardrail contains only input-dependent rules that were evaluated during pre-execution" + ) def test_evaluate_post_deterministic_guardrail_word_func_negative( self, @@ -351,6 +394,16 @@ def test_evaluate_post_deterministic_guardrail_word_func_negative( ), detects_violation=lambda s: len(s) <= 5, ), + NumberRule( + rule_type="number", + field_selector=SpecificFieldsSelector( + selector_type="specific", + fields=[ + FieldReference(path="status", source=FieldSource.OUTPUT) + ], + ), + detects_violation=lambda n: n != 200.0, + ), ], ) @@ -358,7 +411,9 @@ def test_evaluate_post_deterministic_guardrail_word_func_negative( input_data = { "userName": "test", } - output_data: dict[str, Any] = {} + output_data = { + "status": 200, + } result = service.evaluate_post_deterministic_guardrail( input_data=input_data, @@ -393,6 +448,16 @@ def test_evaluate_post_deterministic_guardrail_word_contains_substring_detects_v ), detects_violation=lambda s: "dre" in s, ), + NumberRule( + rule_type="number", + field_selector=SpecificFieldsSelector( + selector_type="specific", + fields=[ + FieldReference(path="status", source=FieldSource.OUTPUT) + ], + ), + detects_violation=lambda n: n != 200.0, + ), ], ) @@ -400,7 +465,9 @@ def test_evaluate_post_deterministic_guardrail_word_contains_substring_detects_v input_data = { "userName": "andrei", } - output_data: dict[str, Any] = {} + output_data = { + "status": 200, + } result = service.evaluate_post_deterministic_guardrail( input_data=input_data, @@ -453,7 +520,10 @@ def test_evaluate_post_deterministic_guardrail_number_func_positive( ) assert result.validation_passed is True - assert result.reason == "All deterministic guardrail rules passed" + assert ( + result.reason + == "Guardrail contains only input-dependent rules that were evaluated during pre-execution" + ) def test_evaluate_post_deterministic_guardrail_number_func_negative( self, @@ -478,6 +548,16 @@ def test_evaluate_post_deterministic_guardrail_number_func_negative( ), detects_violation=lambda n: n < 18 or n > 65, ), + NumberRule( + rule_type="number", + field_selector=SpecificFieldsSelector( + selector_type="specific", + fields=[ + FieldReference(path="status", source=FieldSource.OUTPUT) + ], + ), + detects_violation=lambda n: n != 200.0, + ), ], ) @@ -485,7 +565,9 @@ def test_evaluate_post_deterministic_guardrail_number_func_negative( input_data = { "age": 70, } - output_data: dict[str, Any] = {} + output_data = { + "status": 200, + } result = service.evaluate_post_deterministic_guardrail( input_data=input_data, @@ -506,7 +588,9 @@ def test_should_trigger_policy_pre_execution_only_some_rules_not_met_returns_fal "age": 18, # Less than 21 "isActive": True, } - output_data: dict[str, Any] = {} + output_data = { + "status": 200, + } result = service.evaluate_post_deterministic_guardrail( input_data=input_data, @@ -731,7 +815,9 @@ def test_should_trigger_policy_post_execution_with_all_fields_selector_output_sc rules=[ NumberRule( rule_type="number", - field_selector=AllFieldsSelector(selector_type="all"), + field_selector=AllFieldsSelector( + selector_type="all", sources=[FieldSource.OUTPUT] + ), detects_violation=lambda n: n != 25.0, ), ], @@ -756,7 +842,7 @@ def test_should_trigger_policy_post_execution_with_all_fields_selector_output_sc assert result.validation_passed is True - def test_should_trigger_policy_post_execution_with_all_fields_selector_empty_output_schema_returns_false( + def test_should_trigger_policy_post_execution_with_all_fields_selector_empty_output_schema_returns_true( self, service: DeterministicGuardrailsService, ) -> None: @@ -773,7 +859,9 @@ def test_should_trigger_policy_post_execution_with_all_fields_selector_empty_out rules=[ NumberRule( rule_type="number", - field_selector=AllFieldsSelector(selector_type="all"), + field_selector=AllFieldsSelector( + selector_type="all", sources=[FieldSource.INPUT] + ), detects_violation=lambda n: n != 200.0, ), ], @@ -792,7 +880,7 @@ def test_should_trigger_policy_post_execution_with_all_fields_selector_empty_out guardrail=guardrail, ) - assert result.validation_passed is False + assert result.validation_passed is True def test_should_trigger_policy_pre_execution_always_rule_with_input_apply_to_returns_true( self, @@ -805,11 +893,9 @@ def test_should_trigger_policy_pre_execution_always_rule_with_input_apply_to_ret "age": 25, "isActive": True, } - output_data: dict[str, Any] = {} - result = service.evaluate_post_deterministic_guardrail( + result = service.evaluate_pre_deterministic_guardrail( input_data=input_data, - output_data=output_data, guardrail=guardrail, ) @@ -964,6 +1050,16 @@ def _create_guardrail_for_pre_execution(self) -> DeterministicGuardrail: ), detects_violation=lambda b: b is not True, ), + NumberRule( + rule_type="number", + field_selector=SpecificFieldsSelector( + selector_type="specific", + fields=[ + FieldReference(path="status", source=FieldSource.OUTPUT) + ], + ), + detects_violation=lambda n: n != 200.0, + ), ], ) @@ -1088,6 +1184,258 @@ def _create_guardrail_with_rule_having_multiple_conditions( ], ) + def test_evaluate_post_deterministic_guardrail_word_contains_operator_passes( + self, + service: DeterministicGuardrailsService, + ) -> None: + """Test deterministic guardrail with word contains operator passes for pre-execution.""" + deterministic_guardrail = DeterministicGuardrail( + id="b4283bd4-5ce0-49de-a918-2604d830460c", + name="Before", + description="", + enabled_for_evals=True, + guardrail_type="custom", + selector=GuardrailSelector( + scopes=[GuardrailScope.TOOL], match_names=["ConverterToStringAgent"] + ), + rules=[ + WordRule( + rule_type="word", + field_selector=SpecificFieldsSelector( + selector_type="specific", + fields=[ + FieldReference( + path="input_string", source=FieldSource.INPUT + ) + ], + ), + detects_violation=lambda s: "cti" in s, + ), + ], + ) + + # Input data without "cti" in input_string - should pass + input_data = { + "input_string": "test value", + } + output_data: dict[str, Any] = {} + + result = service.evaluate_post_deterministic_guardrail( + input_data=input_data, + output_data=output_data, + guardrail=deterministic_guardrail, + ) + + assert result.validation_passed is True + assert ( + result.reason + == "Guardrail contains only input-dependent rules that were evaluated during pre-execution" + ) + + def test_evaluate_post_deterministic_guardrail_only_output_rules_passes( + self, + service: DeterministicGuardrailsService, + ) -> None: + """Test post guardrail with only output rules passes when conditions are met.""" + deterministic_guardrail = DeterministicGuardrail( + id="test-only-output-id", + name="Output Only Guardrail", + description="Test guardrail with only output rules", + enabled_for_evals=True, + guardrail_type="custom", + selector=GuardrailSelector( + scopes=[GuardrailScope.TOOL], match_names=["test"] + ), + rules=[ + NumberRule( + rule_type="number", + field_selector=SpecificFieldsSelector( + selector_type="specific", + fields=[ + FieldReference(path="status", source=FieldSource.OUTPUT) + ], + ), + detects_violation=lambda n: n != 200.0, + ), + WordRule( + rule_type="word", + field_selector=SpecificFieldsSelector( + selector_type="specific", + fields=[ + FieldReference(path="result", source=FieldSource.OUTPUT) + ], + ), + detects_violation=lambda s: s != "Success", + ), + ], + ) + + input_data = { + "userName": "John", + } + output_data = { + "status": 200, + "result": "Success", + } + + result = service.evaluate_post_deterministic_guardrail( + input_data=input_data, + output_data=output_data, + guardrail=deterministic_guardrail, + ) + + assert result.validation_passed is True + assert result.reason == "All deterministic guardrail rules passed" + + def test_evaluate_post_deterministic_guardrail_only_always_rule_fails( + self, + service: DeterministicGuardrailsService, + ) -> None: + """Test post guardrail with only UniversalRule always fails.""" + deterministic_guardrail = DeterministicGuardrail( + id="test-only-always-id", + name="Always Rule Only Guardrail", + description="Test guardrail with only always rule", + enabled_for_evals=True, + guardrail_type="custom", + selector=GuardrailSelector( + scopes=[GuardrailScope.TOOL], match_names=["test"] + ), + rules=[ + UniversalRule( + rule_type="always", + apply_to=ApplyTo.OUTPUT, + ), + ], + ) + + input_data = { + "userName": "John", + } + output_data = { + "status": 200, + } + + result = service.evaluate_post_deterministic_guardrail( + input_data=input_data, + output_data=output_data, + guardrail=deterministic_guardrail, + ) + + assert result.validation_passed is False + + def test_evaluate_post_deterministic_guardrail_only_input_rules_passes( + self, + service: DeterministicGuardrailsService, + ) -> None: + """Test post guardrail passes when only input rules exist (no output data required).""" + deterministic_guardrail = DeterministicGuardrail( + id="test-only-input-id", + name="Input Only Guardrail", + description="Test guardrail with only input rules", + enabled_for_evals=True, + guardrail_type="custom", + selector=GuardrailSelector( + scopes=[GuardrailScope.TOOL], match_names=["test"] + ), + rules=[ + NumberRule( + rule_type="number", + field_selector=SpecificFieldsSelector( + selector_type="specific", + fields=[FieldReference(path="age", source=FieldSource.INPUT)], + ), + detects_violation=lambda n: n < 18, + ), + WordRule( + rule_type="word", + field_selector=SpecificFieldsSelector( + selector_type="specific", + fields=[FieldReference(path="name", source=FieldSource.INPUT)], + ), + detects_violation=lambda s: len(s) < 2, + ), + ], + ) + + input_data = { + "age": 25, + "name": "John", + } + output_data: dict[str, Any] = {} + + result = service.evaluate_post_deterministic_guardrail( + input_data=input_data, + output_data=output_data, + guardrail=deterministic_guardrail, + ) + + assert result.validation_passed is True + assert ( + result.reason + == "Guardrail contains only input-dependent rules that were evaluated during pre-execution" + ) + + def test_evaluate_pre_deterministic_guardrail_with_input_and_output_rules_input_true( + self, + service: DeterministicGuardrailsService, + ) -> None: + """Test pre-execution guardrail with input rule and output rules, should pass because is ignored.""" + deterministic_guardrail = DeterministicGuardrail( + id="test-pre-mixed-rules-id", + name="Pre Execution Mixed Rules Guardrail", + description="Test pre-execution with both input and output rules", + enabled_for_evals=True, + guardrail_type="custom", + selector=GuardrailSelector( + scopes=[GuardrailScope.TOOL], match_names=["test"] + ), + rules=[ + NumberRule( + rule_type="number", + field_selector=SpecificFieldsSelector( + selector_type="specific", + fields=[FieldReference(path="age", source=FieldSource.INPUT)], + ), + detects_violation=lambda n: n < 21.0, + ), + BooleanRule( + rule_type="boolean", + field_selector=SpecificFieldsSelector( + selector_type="specific", + fields=[ + FieldReference(path="isActive", source=FieldSource.INPUT) + ], + ), + detects_violation=lambda b: b is not True, + ), + # Output rule - should be ignored in pre-execution + NumberRule( + rule_type="number", + field_selector=SpecificFieldsSelector( + selector_type="specific", + fields=[ + FieldReference(path="status", source=FieldSource.OUTPUT) + ], + ), + detects_violation=lambda n: n != 200.0, + ), + ], + ) + + input_data = { + "userName": "John", + "age": 18, + "isActive": True, + } + + result = service.evaluate_pre_deterministic_guardrail( + input_data=input_data, + guardrail=deterministic_guardrail, + ) + + assert result.validation_passed is True + def _create_guardrail_with_always_rule( self, apply_to: ApplyTo ) -> DeterministicGuardrail: diff --git a/uv.lock b/uv.lock index 62233fc..f7cac35 100644 --- a/uv.lock +++ b/uv.lock @@ -991,7 +991,7 @@ wheels = [ [[package]] name = "uipath-core" -version = "0.1.4" +version = "0.1.5" source = { editable = "." } dependencies = [ { name = "opentelemetry-instrumentation" },