[uss_qualifier] Remove and deprecate misleading instances of "success"

NEXT_RELEASE_NOTES.md

@@ -40,3 +40,5 @@ The release notes should contain at least the following sections:
 ## Optional migration tasks
 
 ## Important information
+
+`successful` attributes in test report objects are deprecated and will be removed in the future (#1428).

monitoring/uss_qualifier/common_data_definitions.py

@@ -30,10 +30,9 @@ class Severity(str, Enum):
     """
 
     Low = "Low"
-    """The system meets requirements but could be improved.
+    """The system does not fail to meet a requirement, but could be improved.
 
-    Further test steps can be executed without impact.  A test run with only
-    Low-Severity issues will be considered successful.
+    Further test steps can be executed without impact.
     """
 
     def __eq__(self, other):

monitoring/uss_qualifier/main.py

@@ -148,10 +148,7 @@ def execute_test_run(
     action = TestSuiteAction(config.action, resources)
     logger.info("Running top-level test suite action")
     report = action.run(context)
-    if report.successful():
-        logger.info("Final result: SUCCESS")
-    else:
-        logger.warning("Final result: FAILURE")
+    logger.info("Top-level test suite action complete")
 
     return TestRunReport(
         codebase_version=description.codebase_version,

monitoring/uss_qualifier/reports/report.py

@@ -4,6 +4,7 @@
 from datetime import UTC, datetime
 from typing import Any
 
+import deprecation
 from implicitdict import (
     ImplicitDict,
     Optional,
@@ -132,6 +133,7 @@ class TestStepReport(ImplicitDict):
     def has_critical_problem(self) -> bool:
         return any(fc.severity == Severity.Critical for fc in self.failed_checks)
 
+    @deprecation.deprecated()
     def successful(self) -> bool:
         return False if self.failed_checks else True
 
@@ -304,7 +306,7 @@ class TestScenarioReport(ImplicitDict):
     """Time at which the test scenario completed or encountered an error"""
 
     successful: bool = False
-    """True iff test scenario completed normally with no failed checks"""
+    """DEPRECATED.  True iff test scenario completed normally with no failed checks."""
 
     cases: list[TestCaseReport]
     """Reports for each of the test cases in this test scenario, in chronological order."""
@@ -414,7 +416,7 @@ class ActionGeneratorReport(ImplicitDict):
     """Time at which the action generator completed or encountered an error"""
 
     successful: bool = False
-    """True iff all actions completed normally with no failed checks"""
+    """DEPRECATED.  True iff all actions completed normally with no failed checks"""
 
     def has_critical_problem(self) -> bool:
         return any(a.has_critical_problem() for a in self.actions)
@@ -507,6 +509,7 @@ def _conditional(
         else:
             raise self.invalid_type_error
 
+    @deprecation.deprecated()
     def successful(self) -> bool:
         return self._conditional(lambda report: report.successful)
 
@@ -726,6 +729,7 @@ class SkippedActionReport(ImplicitDict):
     """Full declaration of the action that was skipped."""
 
     @property
+    @deprecation.deprecated()
     def successful(self) -> bool:
         return True
 
@@ -770,7 +774,7 @@ class TestSuiteReport(ImplicitDict):
     """Time at which the test suite completed"""
 
     successful: bool = False
-    """True iff test suite completed normally with no failed checks"""
+    """DEPRECATED.  True iff test suite completed normally with no failed checks"""
 
     capability_evaluations: list[ParticipantCapabilityEvaluationReport]
     """List of capabilities defined in this test suite, evaluated for each participant."""

monitoring/uss_qualifier/reports/sequence_view/events.py

@@ -83,7 +83,7 @@ def _step_events(
         )
         for pid in participants:
             p = scenario_participants.get(pid, TestedParticipant())
-            p.has_successes = True
+            p.has_passes = True
             scenario_participants[pid] = p
 
     # Create events for this step's queries

monitoring/uss_qualifier/reports/sequence_view/summary_types.py

@@ -92,7 +92,7 @@ def rows(self) -> int:
 class TestedParticipant:
     has_failures: bool = False
     has_infos: bool = False
-    has_successes: bool = False
+    has_passes: bool = False
     has_queries: bool = False
 
 

monitoring/uss_qualifier/reports/tested_requirements/breakdown.py

@@ -89,7 +89,7 @@ def make_breakdown(
     if participant_reqs is not None:
         _populate_breakdown_with_req_set(participant_breakdown, participant_reqs)
         if REQ_RUN_TO_COMPLETION in participant_reqs:
-            # Add a success to REQ_RUN_TO_COMPLETION if nothing caused it to fail
+            # Add a passing check to REQ_RUN_TO_COMPLETION if nothing caused it to fail
             tested_requirement = _tested_requirement_for(
                 REQ_RUN_TO_COMPLETION, participant_breakdown
             )
@@ -113,7 +113,7 @@ def make_breakdown(
                                                 url="",
                                                 has_todo=False,
                                                 is_finding_acceptable=False,
-                                                successes=1,
+                                                passes=1,
                                             )
                                         ],
                                     )
@@ -405,7 +405,7 @@ def _add_check_to_breakdown_for_req(
             tested_check.url = check.documentation_url
         tested_step.checks.append(tested_check)
     if isinstance(check, PassedCheck):
-        tested_check.successes += 1
+        tested_check.passes += 1
     elif isinstance(check, FailedCheck):
         if check.severity == Severity.Low:
             tested_check.findings += 1

monitoring/uss_qualifier/reports/tested_requirements/data_types.py

@@ -22,27 +22,27 @@ class TestedCheck(ImplicitDict):
     url: str
     has_todo: bool
     is_finding_acceptable: bool
-    successes: int = 0
+    passes: int = 0
     findings: int = 0
     failures: int = 0
 
     @property
     def result(self) -> str:
         if self.failures > 0:
             return "Fail"
-        if self.findings > 0 and self.successes == 0:
+        if self.findings > 0 and self.passes == 0:
             return "Findings"
-        if self.findings == 0 and self.successes > 0:
+        if self.findings == 0 and self.passes > 0:
             return "Pass"
-        if self.findings > 0 and self.successes > 0:
+        if self.findings > 0 and self.passes > 0:
             return "Pass (with findings)"
         return "Not tested"
 
     @property
     def check_classname(self) -> str:
         if self.failures > 0:
             return ACCEPTED_FINDINGS_CLASS if self.is_finding_acceptable else FAIL_CLASS
-        if self.successes + self.failures == 0:
+        if self.passes + self.failures == 0:
             if self.has_todo:
                 return HAS_TODO_CLASS
             else:
@@ -53,7 +53,7 @@ def check_classname(self) -> str:
     @property
     def result_classname(self) -> str:
         if self.is_finding_acceptable:
-            if self.successes > 0:
+            if self.passes > 0:
                 return PASS_CLASS
             elif self.failures > 0 or self.findings > 0:
                 return ACCEPTED_FINDINGS_CLASS
@@ -62,7 +62,7 @@ def result_classname(self) -> str:
         else:
             if self.failures > 0:
                 return FAIL_CLASS
-            if self.successes + self.failures + self.findings == 0:
+            if self.passes + self.failures + self.findings == 0:
                 return NOT_TESTED_CLASS
             if self.findings > 0:
                 return FINDINGS_CLASS
@@ -139,15 +139,15 @@ def checks(self) -> Iterable[TestedCheck]:
     def status(self) -> TestedRequirementStatus:
         if any((c.failures > 0 and not c.is_finding_acceptable) for c in self.checks):
             return TestedRequirementStatus.Fail
-        if all(c.successes == 0 for c in self.checks) and any(
+        if all(c.passes == 0 for c in self.checks) and any(
             c.findings > 0 for c in self.checks
         ):
             return TestedRequirementStatus.Findings
-        if any(c.successes > 0 for c in self.checks) and any(
+        if any(c.passes > 0 for c in self.checks) and any(
             (c.findings > 0 and not c.is_finding_acceptable) for c in self.checks
         ):
             return TestedRequirementStatus.PassWithFindings
-        if any(c.successes > 0 for c in self.checks):
+        if any(c.passes > 0 for c in self.checks):
             return TestedRequirementStatus.Pass
         return TestedRequirementStatus.NotTested
 

monitoring/uss_qualifier/scenarios/documentation/requirements.py

@@ -27,7 +27,7 @@
 
 
 class ParticipantRequirementPerformance(ImplicitDict):
-    successes: list[JSONPath]
+    passes: list[JSONPath]
     """List of passed checks involving the requirement"""
 
     failures: list[JSONPath]
@@ -74,7 +74,7 @@ def _add_check(
                 )
             performance = requirement.participant_performance[participant_id]
             if isinstance(check, PassedCheck):
-                performance.successes.append(path)
+                performance.passes.append(path)
             elif isinstance(check, FailedCheck):
                 performance.failures.append(path)
             else:

monitoring/uss_qualifier/suites/suite.py

@@ -177,15 +177,15 @@ def _run_test_scenario(self, context: ExecutionContext) -> TestScenarioReport:
         except Exception as e:
             scenario.record_execution_error(e)
         report = scenario.get_report()
-        if report.successful:
-            logger.info(f'SUCCESS for "{scenario.documentation.name}" scenario')
+        if "execution_error" in report and report.execution_error:
+            lines = report.execution_error.stacktrace.split("\n")
+            logger.error(
+                'Execution error in scenario "{}":\n{}',
+                scenario.documentation.name,
+                "\n".join("  " + line for line in lines),
+            )
         else:
-            if "execution_error" in report and report.execution_error:
-                lines = report.execution_error.stacktrace.split("\n")
-                logger.error(
-                    "Execution error:\n{}", "\n".join("  " + line for line in lines)
-                )
-            logger.warning(f'FAILURE for "{scenario.documentation.name}" scenario')
+            logger.info(f'"{scenario.documentation.name}" scenario completed')
         return report
 
     def _run_test_suite(self, context: ExecutionContext) -> TestSuiteReport:

schemas/monitoring/uss_qualifier/reports/report/ActionGeneratorReport.json

@@ -32,7 +32,7 @@
       "type": "string"
     },
     "successful": {
-      "description": "True iff all actions completed normally with no failed checks",
+      "description": "DEPRECATED.  True iff all actions completed normally with no failed checks",
       "type": "boolean"
     }
   },

schemas/monitoring/uss_qualifier/reports/report/TestScenarioReport.json

@@ -104,7 +104,7 @@
       "type": "string"
     },
     "successful": {
-      "description": "True iff test scenario completed normally with no failed checks",
+      "description": "DEPRECATED.  True iff test scenario completed normally with no failed checks.",
       "type": "boolean"
     }
   },

schemas/monitoring/uss_qualifier/reports/report/TestSuiteReport.json

@@ -43,7 +43,7 @@
       "type": "string"
     },
     "successful": {
-      "description": "True iff test suite completed normally with no failed checks",
+      "description": "DEPRECATED.  True iff test suite completed normally with no failed checks",
       "type": "boolean"
     },
     "suite_type": {