diff --git a/tools/sbom-diff-and-risk/docs/report-schema.md b/tools/sbom-diff-and-risk/docs/report-schema.md index 837c11d..e748463 100644 --- a/tools/sbom-diff-and-risk/docs/report-schema.md +++ b/tools/sbom-diff-and-risk/docs/report-schema.md @@ -28,6 +28,22 @@ JSON reports currently use this top-level structure: When provenance policy fields are relevant, reports may also include `provenance_policy` and `provenance_policy_impact`. Consumers should treat unrecognized top-level fields as additive report data. +## Policy finding explanation fields + +Policy findings in `policy_evaluation.blocking_violations`, `policy_evaluation.warning_violations`, `policy_evaluation.suppressed_violations`, `blocking_findings`, `warning_findings`, `suppressed_findings`, and provenance policy impact sections include stable explainability metadata. + +These fields describe why a local policy rule produced a block, warning, or suppression. They are policy-decision metadata only; they are not dependency safety verdicts, CVE results, or proof that a package is safe or unsafe. + +| Field | Meaning | +| --- | --- | +| `decision_reason` | Stable reason code for the policy decision, such as `risk_finding_matched_policy_rule`, `added_package_count_exceeded_threshold`, or `scorecard_score_below_threshold`. | +| `policy_rule` | Policy rule id that produced the decision. This mirrors `rule_id` for consumers that group explanation data separately. | +| `severity_source` | Source of the active severity, such as `block_on`, `warn_on`, `default_block`, or `default_warn`; `null` when a policy finding has no active severity. | +| `matched_threshold` | Configured threshold or allowlist value involved in the decision, when applicable. | +| `observed_value` | Observed local value that was compared to the policy rule, when applicable. | + +Explanation fields appear only on policy finding objects. Risk findings in `risks` remain the analyzer's local heuristic findings and do not receive policy-decision metadata unless a policy evaluation maps them into policy findings. + ## Summary contract `summary` is the stable, compact entry point for automation that needs counts without walking the full report. The `--summary-json PATH` CLI option writes only this stable `report.json["summary"]` object. The checked-in [../examples/sample-summary.json](../examples/sample-summary.json) artifact is the summary-only output for the default CycloneDX example and matches the `summary` object in [../examples/sample-report.json](../examples/sample-report.json). For CI consumption examples, see [summary-json-ci-cookbook.md](summary-json-ci-cookbook.md). diff --git a/tools/sbom-diff-and-risk/examples/sample-policy-fail-report.json b/tools/sbom-diff-and-risk/examples/sample-policy-fail-report.json index 6e81cf4..e212d43 100644 --- a/tools/sbom-diff-and-risk/examples/sample-policy-fail-report.json +++ b/tools/sbom-diff-and-risk/examples/sample-policy-fail-report.json @@ -335,6 +335,11 @@ "rule_id": "max_added_packages", "level": "block", "message": "Added package count 1 exceeds max_added_packages=0.", + "decision_reason": "added_package_count_exceeded_threshold", + "policy_rule": "max_added_packages", + "severity_source": "block_on", + "matched_threshold": 0, + "observed_value": 1, "component_key": null, "component_name": null, "finding_bucket": null, @@ -344,6 +349,11 @@ "rule_id": "stale_package", "level": "block", "message": "stale_package was not evaluated because enrichment mode is disabled.", + "decision_reason": "risk_finding_matched_policy_rule", + "policy_rule": "stale_package", + "severity_source": "block_on", + "matched_threshold": null, + "observed_value": "not_evaluated", "component_key": "purl:pkg:pypi/requests", "component_name": "requests", "finding_bucket": "not_evaluated", @@ -353,6 +363,11 @@ "rule_id": "stale_package", "level": "block", "message": "stale_package was not evaluated because enrichment mode is disabled.", + "decision_reason": "risk_finding_matched_policy_rule", + "policy_rule": "stale_package", + "severity_source": "block_on", + "matched_threshold": null, + "observed_value": "not_evaluated", "component_key": "purl:pkg:pypi/urllib3", "component_name": "urllib3", "finding_bucket": "not_evaluated", @@ -364,6 +379,11 @@ "rule_id": "new_package", "level": "warn", "message": "Component was not present in the before input.", + "decision_reason": "risk_finding_matched_policy_rule", + "policy_rule": "new_package", + "severity_source": "warn_on", + "matched_threshold": null, + "observed_value": "new_package", "component_key": "purl:pkg:pypi/urllib3", "component_name": "urllib3", "finding_bucket": "new_package", @@ -384,6 +404,11 @@ "rule_id": "max_added_packages", "level": "block", "message": "Added package count 1 exceeds max_added_packages=0.", + "decision_reason": "added_package_count_exceeded_threshold", + "policy_rule": "max_added_packages", + "severity_source": "block_on", + "matched_threshold": 0, + "observed_value": 1, "component_key": null, "component_name": null, "finding_bucket": null, @@ -393,6 +418,11 @@ "rule_id": "stale_package", "level": "block", "message": "stale_package was not evaluated because enrichment mode is disabled.", + "decision_reason": "risk_finding_matched_policy_rule", + "policy_rule": "stale_package", + "severity_source": "block_on", + "matched_threshold": null, + "observed_value": "not_evaluated", "component_key": "purl:pkg:pypi/requests", "component_name": "requests", "finding_bucket": "not_evaluated", @@ -402,6 +432,11 @@ "rule_id": "stale_package", "level": "block", "message": "stale_package was not evaluated because enrichment mode is disabled.", + "decision_reason": "risk_finding_matched_policy_rule", + "policy_rule": "stale_package", + "severity_source": "block_on", + "matched_threshold": null, + "observed_value": "not_evaluated", "component_key": "purl:pkg:pypi/urllib3", "component_name": "urllib3", "finding_bucket": "not_evaluated", @@ -413,6 +448,11 @@ "rule_id": "new_package", "level": "warn", "message": "Component was not present in the before input.", + "decision_reason": "risk_finding_matched_policy_rule", + "policy_rule": "new_package", + "severity_source": "warn_on", + "matched_threshold": null, + "observed_value": "new_package", "component_key": "purl:pkg:pypi/urllib3", "component_name": "urllib3", "finding_bucket": "new_package", @@ -596,6 +636,11 @@ "rule_id": "max_added_packages", "level": "block", "message": "Added package count 1 exceeds max_added_packages=0.", + "decision_reason": "added_package_count_exceeded_threshold", + "policy_rule": "max_added_packages", + "severity_source": "block_on", + "matched_threshold": 0, + "observed_value": 1, "component_key": null, "component_name": null, "finding_bucket": null, @@ -605,6 +650,11 @@ "rule_id": "stale_package", "level": "block", "message": "stale_package was not evaluated because enrichment mode is disabled.", + "decision_reason": "risk_finding_matched_policy_rule", + "policy_rule": "stale_package", + "severity_source": "block_on", + "matched_threshold": null, + "observed_value": "not_evaluated", "component_key": "purl:pkg:pypi/requests", "component_name": "requests", "finding_bucket": "not_evaluated", @@ -614,6 +664,11 @@ "rule_id": "stale_package", "level": "block", "message": "stale_package was not evaluated because enrichment mode is disabled.", + "decision_reason": "risk_finding_matched_policy_rule", + "policy_rule": "stale_package", + "severity_source": "block_on", + "matched_threshold": null, + "observed_value": "not_evaluated", "component_key": "purl:pkg:pypi/urllib3", "component_name": "urllib3", "finding_bucket": "not_evaluated", @@ -625,6 +680,11 @@ "rule_id": "new_package", "level": "warn", "message": "Component was not present in the before input.", + "decision_reason": "risk_finding_matched_policy_rule", + "policy_rule": "new_package", + "severity_source": "warn_on", + "matched_threshold": null, + "observed_value": "new_package", "component_key": "purl:pkg:pypi/urllib3", "component_name": "urllib3", "finding_bucket": "new_package", diff --git a/tools/sbom-diff-and-risk/examples/sample-policy-warn-report.json b/tools/sbom-diff-and-risk/examples/sample-policy-warn-report.json index 94f525d..f487337 100644 --- a/tools/sbom-diff-and-risk/examples/sample-policy-warn-report.json +++ b/tools/sbom-diff-and-risk/examples/sample-policy-warn-report.json @@ -327,6 +327,11 @@ "rule_id": "new_package", "level": "warn", "message": "Component was not present in the before input.", + "decision_reason": "risk_finding_matched_policy_rule", + "policy_rule": "new_package", + "severity_source": "warn_on", + "matched_threshold": null, + "observed_value": "new_package", "component_key": "purl:pkg:pypi/urllib3", "component_name": "urllib3", "finding_bucket": "new_package", @@ -348,6 +353,11 @@ "rule_id": "new_package", "level": "warn", "message": "Component was not present in the before input.", + "decision_reason": "risk_finding_matched_policy_rule", + "policy_rule": "new_package", + "severity_source": "warn_on", + "matched_threshold": null, + "observed_value": "new_package", "component_key": "purl:pkg:pypi/urllib3", "component_name": "urllib3", "finding_bucket": "new_package", @@ -523,6 +533,11 @@ "rule_id": "new_package", "level": "warn", "message": "Component was not present in the before input.", + "decision_reason": "risk_finding_matched_policy_rule", + "policy_rule": "new_package", + "severity_source": "warn_on", + "matched_threshold": null, + "observed_value": "new_package", "component_key": "purl:pkg:pypi/urllib3", "component_name": "urllib3", "finding_bucket": "new_package", diff --git a/tools/sbom-diff-and-risk/examples/sample-provenance-report.json b/tools/sbom-diff-and-risk/examples/sample-provenance-report.json index 7d3a11c..7b0c92b 100644 --- a/tools/sbom-diff-and-risk/examples/sample-provenance-report.json +++ b/tools/sbom-diff-and-risk/examples/sample-provenance-report.json @@ -393,6 +393,11 @@ "rule_id": "provenance_required", "level": "block", "message": "Provenance is required for new package, but no attestations were published for this PyPI package.", + "decision_reason": "required_provenance_not_satisfied", + "policy_rule": "provenance_required", + "severity_source": "block_on", + "matched_threshold": null, + "observed_value": "attestation_missing", "component_key": "purl:pkg:pypi/mystery-lib", "component_name": "mystery-lib", "finding_bucket": null, @@ -402,6 +407,15 @@ "rule_id": "unverified_provenance", "level": "block", "message": "PyPI attestations were present, but publisher kinds manual upload did not match allow_provenance_publishers=github actions.", + "decision_reason": "provenance_publisher_not_verified", + "policy_rule": "unverified_provenance", + "severity_source": "block_on", + "matched_threshold": [ + "github actions" + ], + "observed_value": [ + "manual upload" + ], "component_key": "purl:pkg:pypi/legacy-lib", "component_name": "legacy-lib", "finding_bucket": null, @@ -413,6 +427,11 @@ "rule_id": "missing_attestation", "level": "warn", "message": "PyPI release metadata was fetched, but no attestations were published for this package release.", + "decision_reason": "attestation_not_published", + "policy_rule": "missing_attestation", + "severity_source": "warn_on", + "matched_threshold": null, + "observed_value": false, "component_key": "purl:pkg:pypi/mystery-lib", "component_name": "mystery-lib", "finding_bucket": null, @@ -433,6 +452,11 @@ "rule_id": "provenance_required", "level": "block", "message": "Provenance is required for new package, but no attestations were published for this PyPI package.", + "decision_reason": "required_provenance_not_satisfied", + "policy_rule": "provenance_required", + "severity_source": "block_on", + "matched_threshold": null, + "observed_value": "attestation_missing", "component_key": "purl:pkg:pypi/mystery-lib", "component_name": "mystery-lib", "finding_bucket": null, @@ -442,6 +466,15 @@ "rule_id": "unverified_provenance", "level": "block", "message": "PyPI attestations were present, but publisher kinds manual upload did not match allow_provenance_publishers=github actions.", + "decision_reason": "provenance_publisher_not_verified", + "policy_rule": "unverified_provenance", + "severity_source": "block_on", + "matched_threshold": [ + "github actions" + ], + "observed_value": [ + "manual upload" + ], "component_key": "purl:pkg:pypi/legacy-lib", "component_name": "legacy-lib", "finding_bucket": null, @@ -453,6 +486,11 @@ "rule_id": "missing_attestation", "level": "warn", "message": "PyPI release metadata was fetched, but no attestations were published for this package release.", + "decision_reason": "attestation_not_published", + "policy_rule": "missing_attestation", + "severity_source": "warn_on", + "matched_threshold": null, + "observed_value": false, "component_key": "purl:pkg:pypi/mystery-lib", "component_name": "mystery-lib", "finding_bucket": null, @@ -652,6 +690,11 @@ "rule_id": "provenance_required", "level": "block", "message": "Provenance is required for new package, but no attestations were published for this PyPI package.", + "decision_reason": "required_provenance_not_satisfied", + "policy_rule": "provenance_required", + "severity_source": "block_on", + "matched_threshold": null, + "observed_value": "attestation_missing", "component_key": "purl:pkg:pypi/mystery-lib", "component_name": "mystery-lib", "finding_bucket": null, @@ -661,6 +704,15 @@ "rule_id": "unverified_provenance", "level": "block", "message": "PyPI attestations were present, but publisher kinds manual upload did not match allow_provenance_publishers=github actions.", + "decision_reason": "provenance_publisher_not_verified", + "policy_rule": "unverified_provenance", + "severity_source": "block_on", + "matched_threshold": [ + "github actions" + ], + "observed_value": [ + "manual upload" + ], "component_key": "purl:pkg:pypi/legacy-lib", "component_name": "legacy-lib", "finding_bucket": null, @@ -672,6 +724,11 @@ "rule_id": "missing_attestation", "level": "warn", "message": "PyPI release metadata was fetched, but no attestations were published for this package release.", + "decision_reason": "attestation_not_published", + "policy_rule": "missing_attestation", + "severity_source": "warn_on", + "matched_threshold": null, + "observed_value": false, "component_key": "purl:pkg:pypi/mystery-lib", "component_name": "mystery-lib", "finding_bucket": null, @@ -732,6 +789,11 @@ "rule_id": "provenance_required", "level": "block", "message": "Provenance is required for new package, but no attestations were published for this PyPI package.", + "decision_reason": "required_provenance_not_satisfied", + "policy_rule": "provenance_required", + "severity_source": "block_on", + "matched_threshold": null, + "observed_value": "attestation_missing", "component_key": "purl:pkg:pypi/mystery-lib", "component_name": "mystery-lib", "finding_bucket": null, @@ -741,6 +803,15 @@ "rule_id": "unverified_provenance", "level": "block", "message": "PyPI attestations were present, but publisher kinds manual upload did not match allow_provenance_publishers=github actions.", + "decision_reason": "provenance_publisher_not_verified", + "policy_rule": "unverified_provenance", + "severity_source": "block_on", + "matched_threshold": [ + "github actions" + ], + "observed_value": [ + "manual upload" + ], "component_key": "purl:pkg:pypi/legacy-lib", "component_name": "legacy-lib", "finding_bucket": null, @@ -752,6 +823,11 @@ "rule_id": "missing_attestation", "level": "warn", "message": "PyPI release metadata was fetched, but no attestations were published for this package release.", + "decision_reason": "attestation_not_published", + "policy_rule": "missing_attestation", + "severity_source": "warn_on", + "matched_threshold": null, + "observed_value": false, "component_key": "purl:pkg:pypi/mystery-lib", "component_name": "mystery-lib", "finding_bucket": null, @@ -780,6 +856,11 @@ "rule_id": "provenance_required", "level": "block", "message": "Provenance is required for new package, but no attestations were published for this PyPI package.", + "decision_reason": "required_provenance_not_satisfied", + "policy_rule": "provenance_required", + "severity_source": "block_on", + "matched_threshold": null, + "observed_value": "attestation_missing", "component_key": "purl:pkg:pypi/mystery-lib", "component_name": "mystery-lib", "finding_bucket": null, @@ -789,6 +870,15 @@ "rule_id": "unverified_provenance", "level": "block", "message": "PyPI attestations were present, but publisher kinds manual upload did not match allow_provenance_publishers=github actions.", + "decision_reason": "provenance_publisher_not_verified", + "policy_rule": "unverified_provenance", + "severity_source": "block_on", + "matched_threshold": [ + "github actions" + ], + "observed_value": [ + "manual upload" + ], "component_key": "purl:pkg:pypi/legacy-lib", "component_name": "legacy-lib", "finding_bucket": null, @@ -800,6 +890,11 @@ "rule_id": "missing_attestation", "level": "warn", "message": "PyPI release metadata was fetched, but no attestations were published for this package release.", + "decision_reason": "attestation_not_published", + "policy_rule": "missing_attestation", + "severity_source": "warn_on", + "matched_threshold": null, + "observed_value": false, "component_key": "purl:pkg:pypi/mystery-lib", "component_name": "mystery-lib", "finding_bucket": null, diff --git a/tools/sbom-diff-and-risk/examples/sample-scorecard-report.json b/tools/sbom-diff-and-risk/examples/sample-scorecard-report.json index 56e3c88..a77a88a 100644 --- a/tools/sbom-diff-and-risk/examples/sample-scorecard-report.json +++ b/tools/sbom-diff-and-risk/examples/sample-scorecard-report.json @@ -304,6 +304,11 @@ "rule_id": "scorecard_below_threshold", "level": "warn", "message": "Scorecard score 6.0 is below minimum_scorecard_score=7.0 for repository github.com/psf/requests.", + "decision_reason": "scorecard_score_below_threshold", + "policy_rule": "scorecard_below_threshold", + "severity_source": "warn_on", + "matched_threshold": 7.0, + "observed_value": 6.0, "component_key": "purl:pkg:pypi/requests", "component_name": "requests", "finding_bucket": null, @@ -325,6 +330,11 @@ "rule_id": "scorecard_below_threshold", "level": "warn", "message": "Scorecard score 6.0 is below minimum_scorecard_score=7.0 for repository github.com/psf/requests.", + "decision_reason": "scorecard_score_below_threshold", + "policy_rule": "scorecard_below_threshold", + "severity_source": "warn_on", + "matched_threshold": 7.0, + "observed_value": 6.0, "component_key": "purl:pkg:pypi/requests", "component_name": "requests", "finding_bucket": null, @@ -543,6 +553,11 @@ "rule_id": "scorecard_below_threshold", "level": "warn", "message": "Scorecard score 6.0 is below minimum_scorecard_score=7.0 for repository github.com/psf/requests.", + "decision_reason": "scorecard_score_below_threshold", + "policy_rule": "scorecard_below_threshold", + "severity_source": "warn_on", + "matched_threshold": 7.0, + "observed_value": 6.0, "component_key": "purl:pkg:pypi/requests", "component_name": "requests", "finding_bucket": null, diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/policy_evaluator.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/policy_evaluator.py index 72d329a..9f5cd99 100644 --- a/tools/sbom-diff-and-risk/src/sbom_diff_risk/policy_evaluator.py +++ b/tools/sbom-diff-and-risk/src/sbom_diff_risk/policy_evaluator.py @@ -18,6 +18,12 @@ class ProvenanceAssessment: unverified_message: str | None = None +@dataclass(slots=True, frozen=True) +class PolicyRuleDecision: + severity: PolicyLevel | None + severity_source: str | None + + def evaluate_policy( policy: PolicyConfig | None, *, @@ -36,14 +42,16 @@ def evaluate_policy( for finding in findings: rule_id = finding_rule_id(finding) - severity = _severity_for_rule(policy, rule_id) + decision = _decision_for_rule(policy, rule_id) ignored_checks += _record_violation( policy, - severity=severity, - violation=PolicyViolation( + severity=decision.severity, + violation=_policy_violation( rule_id=rule_id, - level=severity, + decision=decision, message=finding.rationale, + decision_reason="risk_finding_matched_policy_rule", + observed_value=finding.bucket.value, component_key=finding.component_key, component_name=finding.component.name, finding_bucket=finding.bucket.value, @@ -73,14 +81,16 @@ def evaluate_policy( # Keep allow_unattested_packages narrow and explicit: it waives only # missing-attestation checks, not complete provenance unavailability. if assessment.provenance_unavailable: - severity = _severity_for_rule(policy, "provenance_unavailable") + decision = _decision_for_rule(policy, "provenance_unavailable") ignored_checks += _record_violation( policy, - severity=severity, - violation=PolicyViolation( + severity=decision.severity, + violation=_policy_violation( rule_id="provenance_unavailable", - level=severity, + decision=decision, message=assessment.unavailable_message or "PyPI provenance evidence is unavailable.", + decision_reason="provenance_evidence_unavailable", + observed_value="unavailable", component_key=component_key(component), component_name=component.name, ), @@ -89,14 +99,16 @@ def evaluate_policy( suppressed_violations=suppressed_violations, ) elif not assessment.attestation_available and not package_is_unattested_allowed: - severity = _severity_for_rule(policy, "missing_attestation") + decision = _decision_for_rule(policy, "missing_attestation") ignored_checks += _record_violation( policy, - severity=severity, - violation=PolicyViolation( + severity=decision.severity, + violation=_policy_violation( rule_id="missing_attestation", - level=severity, + decision=decision, message="PyPI release metadata was fetched, but no attestations were published for this package release.", + decision_reason="attestation_not_published", + observed_value=False, component_key=component_key(component), component_name=component.name, ), @@ -106,14 +118,17 @@ def evaluate_policy( ) if assessment.attestation_available and not assessment.verified: - severity = _severity_for_rule(policy, "unverified_provenance") + decision = _decision_for_rule(policy, "unverified_provenance") ignored_checks += _record_violation( policy, - severity=severity, - violation=PolicyViolation( + severity=decision.severity, + violation=_policy_violation( rule_id="unverified_provenance", - level=severity, + decision=decision, message=assessment.unverified_message or "PyPI provenance could not be verified.", + decision_reason="provenance_publisher_not_verified", + matched_threshold=list(policy.allow_provenance_publishers) or None, + observed_value=list(assessment.publisher_kinds), component_key=component_key(component), component_name=component.name, ), @@ -138,14 +153,16 @@ def evaluate_policy( package_is_unattested_allowed=package_is_unattested_allowed, ) if requirement_message is not None: - severity = _severity_for_rule(policy, "provenance_required", default=PolicyLevel.BLOCK) + decision = _decision_for_rule(policy, "provenance_required", default=PolicyLevel.BLOCK) ignored_checks += _record_violation( policy, - severity=severity, - violation=PolicyViolation( + severity=decision.severity, + violation=_policy_violation( rule_id="provenance_required", - level=severity, + decision=decision, message=requirement_message, + decision_reason="required_provenance_not_satisfied", + observed_value=_provenance_observed_value(assessment), component_key=component_key(component), component_name=component.name, ), @@ -156,14 +173,17 @@ def evaluate_policy( if policy.max_added_packages is not None and len(added) > policy.max_added_packages: rule_id = "max_added_packages" - severity = _severity_for_rule(policy, rule_id, default=PolicyLevel.BLOCK) + decision = _decision_for_rule(policy, rule_id, default=PolicyLevel.BLOCK) ignored_checks += _record_violation( policy, - severity=severity, - violation=PolicyViolation( + severity=decision.severity, + violation=_policy_violation( rule_id=rule_id, - level=severity, + decision=decision, message=f"Added package count {len(added)} exceeds max_added_packages={policy.max_added_packages}.", + decision_reason="added_package_count_exceeded_threshold", + matched_threshold=policy.max_added_packages, + observed_value=len(added), ), blocking_violations=blocking_violations, warning_violations=warning_violations, @@ -177,25 +197,28 @@ def evaluate_policy( continue rule_id = "allow_sources" - severity = _severity_for_rule(policy, rule_id, default=PolicyLevel.BLOCK) + decision = _decision_for_rule(policy, rule_id, default=PolicyLevel.BLOCK) ignored_checks += _record_violation( policy, - severity=severity, - violation=PolicyViolation( + severity=decision.severity, + violation=_policy_violation( rule_id=rule_id, - level=severity, + decision=decision, message=f"Source host {host or 'missing'} is not present in allow_sources.", + decision_reason="source_host_not_allowed", + matched_threshold=list(policy.allow_sources), + observed_value=host or "missing", component_key=component_key(component), component_name=component.name, ), blocking_violations=blocking_violations, warning_violations=warning_violations, - suppressed_violations=suppressed_violations, - ) + suppressed_violations=suppressed_violations, + ) if policy.minimum_scorecard_score is not None: rule_id = "scorecard_below_threshold" - severity = _severity_for_rule(policy, rule_id) + decision = _decision_for_rule(policy, rule_id) for component in provenance_components: scorecard_score = _scorecard_score(component) if scorecard_score is None or scorecard_score >= policy.minimum_scorecard_score: @@ -207,14 +230,17 @@ def evaluate_policy( ) ignored_checks += _record_violation( policy, - severity=severity, - violation=PolicyViolation( + severity=decision.severity, + violation=_policy_violation( rule_id=rule_id, - level=severity, + decision=decision, message=( f"Scorecard score {scorecard_score:.1f} is below minimum_scorecard_score=" f"{policy.minimum_scorecard_score:.1f} for repository {repository_name}." ), + decision_reason="scorecard_score_below_threshold", + matched_threshold=policy.minimum_scorecard_score, + observed_value=scorecard_score, component_key=component_key(component), component_name=component.name, ), @@ -246,17 +272,48 @@ def finding_rule_id(finding: RiskFinding) -> str: return finding.bucket.value -def _severity_for_rule( +def _decision_for_rule( policy: PolicyConfig, rule_id: str, *, default: PolicyLevel | None = None, -) -> PolicyLevel | None: +) -> PolicyRuleDecision: if rule_id in policy.block_on: - return PolicyLevel.BLOCK + return PolicyRuleDecision(severity=PolicyLevel.BLOCK, severity_source="block_on") if rule_id in policy.warn_on: - return PolicyLevel.WARN - return default + return PolicyRuleDecision(severity=PolicyLevel.WARN, severity_source="warn_on") + if default is PolicyLevel.BLOCK: + return PolicyRuleDecision(severity=PolicyLevel.BLOCK, severity_source="default_block") + if default is PolicyLevel.WARN: + return PolicyRuleDecision(severity=PolicyLevel.WARN, severity_source="default_warn") + return PolicyRuleDecision(severity=default, severity_source=None) + + +def _policy_violation( + *, + rule_id: str, + decision: PolicyRuleDecision, + message: str, + decision_reason: str, + matched_threshold: object | None = None, + observed_value: object | None = None, + component_key: str | None = None, + component_name: str | None = None, + finding_bucket: str | None = None, +) -> PolicyViolation: + return PolicyViolation( + rule_id=rule_id, + level=decision.severity, + message=message, + decision_reason=decision_reason, + policy_rule=rule_id, + severity_source=decision.severity_source, + matched_threshold=matched_threshold, + observed_value=observed_value, + component_key=component_key, + component_name=component_name, + finding_bucket=finding_bucket, + ) def _record_violation( @@ -274,6 +331,11 @@ def _record_violation( rule_id=violation.rule_id, level=severity, message=violation.message, + decision_reason=violation.decision_reason, + policy_rule=violation.policy_rule, + severity_source=violation.severity_source, + matched_threshold=violation.matched_threshold, + observed_value=violation.observed_value, component_key=violation.component_key, component_name=violation.component_name, finding_bucket=violation.finding_bucket, @@ -289,6 +351,11 @@ def _record_violation( rule_id=violation.rule_id, level=severity, message=violation.message, + decision_reason=violation.decision_reason, + policy_rule=violation.policy_rule, + severity_source=violation.severity_source, + matched_threshold=violation.matched_threshold, + observed_value=violation.observed_value, component_key=violation.component_key, component_name=violation.component_name, finding_bucket=violation.finding_bucket, @@ -426,6 +493,16 @@ def _provenance_requirement_message( return None +def _provenance_observed_value(assessment: ProvenanceAssessment) -> str: + if assessment.provenance_unavailable: + return "provenance_unavailable" + if not assessment.attestation_available: + return "attestation_missing" + if not assessment.verified: + return "provenance_unverified" + return "provenance_verified" + + def _source_host(source_url: str | None) -> str | None: if not source_url: return None diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/policy_models.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/policy_models.py index 4b1105a..bc4a955 100644 --- a/tools/sbom-diff-and-risk/src/sbom_diff_risk/policy_models.py +++ b/tools/sbom-diff-and-risk/src/sbom_diff_risk/policy_models.py @@ -2,6 +2,7 @@ from dataclasses import dataclass, field from enum import StrEnum +from typing import Any class PolicyLevel(StrEnum): @@ -58,6 +59,11 @@ class PolicyViolation: rule_id: str level: PolicyLevel | None message: str + decision_reason: str | None = None + policy_rule: str | None = None + severity_source: str | None = None + matched_threshold: Any | None = None + observed_value: Any | None = None component_key: str | None = None component_name: str | None = None finding_bucket: str | None = None diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/presentation.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/presentation.py index d41fcb7..7200391 100644 --- a/tools/sbom-diff-and-risk/src/sbom_diff_risk/presentation.py +++ b/tools/sbom-diff-and-risk/src/sbom_diff_risk/presentation.py @@ -178,6 +178,11 @@ def policy_violation_to_dict(violation: PolicyViolation) -> dict[str, Any]: "rule_id": violation.rule_id, "level": violation.level.value if violation.level is not None else None, "message": violation.message, + "decision_reason": violation.decision_reason, + "policy_rule": violation.policy_rule, + "severity_source": violation.severity_source, + "matched_threshold": violation.matched_threshold, + "observed_value": violation.observed_value, "component_key": violation.component_key, "component_name": violation.component_name, "finding_bucket": violation.finding_bucket, diff --git a/tools/sbom-diff-and-risk/tests/test_policy.py b/tools/sbom-diff-and-risk/tests/test_policy.py index b64abb7..5aa933f 100644 --- a/tools/sbom-diff-and-risk/tests/test_policy.py +++ b/tools/sbom-diff-and-risk/tests/test_policy.py @@ -82,6 +82,11 @@ def test_policy_evaluator_blocks_on_finding_bucket() -> None: assert len(evaluation.blocking_violations) == 1 assert evaluation.blocking_violations[0].rule_id == "unknown_license" assert evaluation.blocking_violations[0].level is PolicyLevel.BLOCK + assert evaluation.blocking_violations[0].decision_reason == "risk_finding_matched_policy_rule" + assert evaluation.blocking_violations[0].policy_rule == "unknown_license" + assert evaluation.blocking_violations[0].severity_source == "block_on" + assert evaluation.blocking_violations[0].matched_threshold is None + assert evaluation.blocking_violations[0].observed_value == "unknown_license" def test_policy_evaluator_warns_on_rule_when_configured() -> None: @@ -110,7 +115,12 @@ def test_policy_evaluator_max_added_packages_blocks() -> None: evaluation = evaluate_policy(policy, policy_path="policy.yml", added=added, changed=[], findings=[]) assert evaluation.exit_code == 1 - assert any(violation.rule_id == "max_added_packages" for violation in evaluation.blocking_violations) + violation = next(item for item in evaluation.blocking_violations if item.rule_id == "max_added_packages") + assert violation.decision_reason == "added_package_count_exceeded_threshold" + assert violation.policy_rule == "max_added_packages" + assert violation.severity_source == "default_block" + assert violation.matched_threshold == 0 + assert violation.observed_value == 1 def test_policy_evaluator_allow_sources_blocks_unknown_hosts() -> None: @@ -145,6 +155,9 @@ def test_policy_ignore_rules_suppresses_violations() -> None: assert evaluation.ignored_checks == 1 assert len(evaluation.suppressed_violations) == 1 assert evaluation.suppressed_violations[0].suppression_reason == "ignored_by_policy" + assert evaluation.suppressed_violations[0].decision_reason == "risk_finding_matched_policy_rule" + assert evaluation.suppressed_violations[0].policy_rule == "unknown_license" + assert evaluation.suppressed_violations[0].severity_source == "block_on" def _example_path(name: str) -> Path: diff --git a/tools/sbom-diff-and-risk/tests/test_reports.py b/tools/sbom-diff-and-risk/tests/test_reports.py index 5edb223..3e066af 100644 --- a/tools/sbom-diff-and-risk/tests/test_reports.py +++ b/tools/sbom-diff-and-risk/tests/test_reports.py @@ -197,6 +197,60 @@ def test_report_json_summary_includes_policy_status_when_policy_is_used() -> Non assert "enrichment" not in payload["summary"] +def test_report_json_policy_findings_include_explanation_fields() -> None: + report = _build_report("cdx_before.json", "cdx_after.json", policy_name="policy-minimal.yml") + + payload = json.loads(render_report_json(report)) + finding = payload["warning_findings"][0] + + assert finding["rule_id"] == "new_package" + assert finding["decision_reason"] == "risk_finding_matched_policy_rule" + assert finding["policy_rule"] == "new_package" + assert finding["severity_source"] == "warn_on" + assert finding["matched_threshold"] is None + assert finding["observed_value"] == "new_package" + assert payload["metadata"]["policy_evaluation"] == payload["policy_evaluation"] + assert payload["summary"]["policy"] == { + "status": "warn", + "blocking": 0, + "warning": 1, + "suppressed": 0, + } + + +def test_report_json_explanation_fields_are_policy_only() -> None: + report = _build_report("cdx_before.json", "cdx_after.json") + + payload = json.loads(render_report_json(report)) + + assert payload["blocking_findings"] == [] + assert payload["warning_findings"] == [] + assert payload["suppressed_findings"] == [] + assert all("decision_reason" not in finding for finding in payload["risks"]) + assert "policy" not in payload["summary"] + + +def test_report_json_policy_output_ordering_is_deterministic() -> None: + report = _build_report("cdx_before.json", "cdx_after.json", policy_name="policy-strict.yml") + + first = render_report_json(report) + second = render_report_json(report) + payload = json.loads(first) + sort_keys = [ + (item["rule_id"], item["component_key"] or "", item["component_name"] or "") + for item in payload["blocking_findings"] + ] + + assert first == second + assert sort_keys == sorted(sort_keys) + assert payload["summary"]["policy"] == { + "status": "fail", + "blocking": 3, + "warning": 1, + "suppressed": 0, + } + + def test_reports_render_suppressions_when_policy_ignores_findings() -> None: policy = PolicyConfig( version=1,