diff --git a/tools/sbom-diff-and-risk/README.md b/tools/sbom-diff-and-risk/README.md index b150491..6d96b3a 100644 --- a/tools/sbom-diff-and-risk/README.md +++ b/tools/sbom-diff-and-risk/README.md @@ -28,6 +28,8 @@ For CI consumption of summary-only output, see [docs/summary-json-ci-cookbook.md](docs/summary-json-ci-cookbook.md). For a consumer-facing GitHub Actions example, see [docs/github-actions-consumer-example.md](docs/github-actions-consumer-example.md). +For regenerating checked-in local example outputs, see +[docs/example-artifact-regeneration.md](docs/example-artifact-regeneration.md). 1. If you want to verify `sbom-diff-and-risk` itself, start with [docs/verification.md](docs/verification.md). @@ -316,6 +318,14 @@ The [examples/](examples/) directory includes: - provenance-aware sample reports at [sample-provenance-report.json](examples/sample-provenance-report.json), [sample-provenance-report.md](examples/sample-provenance-report.md), and [sample-provenance-report.sarif](examples/sample-provenance-report.sarif) - Scorecard-aware sample reports at [sample-scorecard-report.json](examples/sample-scorecard-report.json), [sample-scorecard-report.md](examples/sample-scorecard-report.md), and [sample-scorecard-report.sarif](examples/sample-scorecard-report.sarif) - requirements-based sample reports at [sample-requirements-report.json](examples/sample-requirements-report.json) and [sample-requirements-report.md](examples/sample-requirements-report.md) + +After changing local example inputs, regenerate checked-in deterministic +examples with: + +```bash +python scripts/regenerate-example-artifacts.py +python scripts/regenerate-example-artifacts.py --check +``` ## Enforcement Mode diff --git a/tools/sbom-diff-and-risk/docs/example-artifact-regeneration.md b/tools/sbom-diff-and-risk/docs/example-artifact-regeneration.md new file mode 100644 index 0000000..77c2c5a --- /dev/null +++ b/tools/sbom-diff-and-risk/docs/example-artifact-regeneration.md @@ -0,0 +1,57 @@ +# Example Artifact Regeneration + +This page documents how to regenerate the checked-in no-network example +artifacts for `sbom-diff-and-risk`. + +Use this when an example input changes, such as +`examples/requirements_before.txt` or `examples/requirements_after.txt`. +The generated sample reports are intentionally committed so reviewers can +compare deterministic output without running enrichment services. + +## Regenerate + +From `tools/sbom-diff-and-risk`: + +```powershell +python scripts/regenerate-example-artifacts.py +``` + +The script regenerates these local, deterministic artifacts: + +- `examples/sample-report.json` +- `examples/sample-summary.json` +- `examples/sample-report.md` +- `examples/sample-policy-warn-report.json` +- `examples/sample-policy-warn-report.md` +- `examples/sample-policy-fail-report.json` +- `examples/sample-policy.json` +- `examples/sample-policy-fail-report.md` +- `examples/sample-requirements-report.json` +- `examples/sample-requirements-report.md` + +The strict-policy example intentionally exits with code `1` because it produces +blocking local policy findings. The script treats that as expected while still +capturing the generated reports. + +## Check Mode + +Use `--check` to verify that generated output matches the checked-in artifacts +without modifying the repository: + +```powershell +python scripts/regenerate-example-artifacts.py --check +``` + +The test suite runs this check mode so stale local JSON, Markdown, summary, or +policy-sidecar examples fail predictably. + +## Boundaries + +The regeneration script covers no-network JSON, Markdown, summary, and policy +sidecar examples produced through the public CLI. + +It does not perform PyPI or Scorecard enrichment, does not call external +services, and does not make dependency safety claims. Provenance-aware, +Scorecard-aware, and SARIF sample artifacts remain covered by their focused +golden tests because those examples include mocked evidence or normalized SARIF +metadata. diff --git a/tools/sbom-diff-and-risk/docs/reviewer-evidence-pack.md b/tools/sbom-diff-and-risk/docs/reviewer-evidence-pack.md index 2882e8a..df2c8fd 100644 --- a/tools/sbom-diff-and-risk/docs/reviewer-evidence-pack.md +++ b/tools/sbom-diff-and-risk/docs/reviewer-evidence-pack.md @@ -56,6 +56,16 @@ No differences means the sample path reproduced the committed example output. `examples/sample-summary.json` is the summary-only artifact for the same run and is expected to match `examples/sample-report.json`'s `summary` object. +Maintainers can also verify checked-in no-network JSON, Markdown, summary, and +policy sidecar examples in one pass: + +```powershell +python scripts/regenerate-example-artifacts.py --check +``` + +For the exact regeneration scope, see +[example-artifact-regeneration.md](example-artifact-regeneration.md). + Generate the strict-policy JSON sidecar: ```powershell diff --git a/tools/sbom-diff-and-risk/scripts/regenerate-example-artifacts.py b/tools/sbom-diff-and-risk/scripts/regenerate-example-artifacts.py new file mode 100644 index 0000000..cd36338 --- /dev/null +++ b/tools/sbom-diff-and-risk/scripts/regenerate-example-artifacts.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Sequence + + +@dataclass(frozen=True) +class ExampleArtifactSet: + name: str + base_args: tuple[str, ...] + outputs: tuple[tuple[str, str], ...] + expected_exit_codes: tuple[int, ...] = (0,) + + +ARTIFACT_SETS: tuple[ExampleArtifactSet, ...] = ( + ExampleArtifactSet( + name="cyclonedx report, summary, and markdown", + base_args=( + "--before", + "examples/cdx_before.json", + "--after", + "examples/cdx_after.json", + "--format", + "auto", + ), + outputs=( + ("--out-json", "sample-report.json"), + ("--summary-json", "sample-summary.json"), + ("--out-md", "sample-report.md"), + ), + ), + ExampleArtifactSet( + name="warn-only policy report", + base_args=( + "--before", + "examples/cdx_before.json", + "--after", + "examples/cdx_after.json", + "--policy", + "examples/policy-minimal.yml", + ), + outputs=( + ("--out-json", "sample-policy-warn-report.json"), + ("--out-md", "sample-policy-warn-report.md"), + ), + ), + ExampleArtifactSet( + name="blocking policy report and sidecar", + base_args=( + "--before", + "examples/cdx_before.json", + "--after", + "examples/cdx_after.json", + "--policy", + "examples/policy-strict.yml", + ), + outputs=( + ("--out-json", "sample-policy-fail-report.json"), + ("--policy-json", "sample-policy.json"), + ("--out-md", "sample-policy-fail-report.md"), + ), + expected_exit_codes=(1,), + ), + ExampleArtifactSet( + name="requirements report", + base_args=( + "--before", + "examples/requirements_before.txt", + "--after", + "examples/requirements_after.txt", + "--format", + "auto", + ), + outputs=( + ("--out-json", "sample-requirements-report.json"), + ("--out-md", "sample-requirements-report.md"), + ), + ), +) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Regenerate checked-in no-network example report artifacts.", + ) + parser.add_argument( + "--check", + action="store_true", + help="Generate artifacts into a temporary directory and fail if checked-in examples are stale.", + ) + args = parser.parse_args(argv) + + project_root = Path(__file__).resolve().parents[1] + if args.check: + with tempfile.TemporaryDirectory(prefix="sbom-diff-risk-examples-") as temp_dir: + return _check_artifacts(project_root, Path(temp_dir)) + return _write_artifacts(project_root, project_root / "examples") + + +def _write_artifacts(project_root: Path, output_root: Path) -> int: + for artifact_set in ARTIFACT_SETS: + _run_artifact_set(project_root, output_root, artifact_set) + print(f"generated: {artifact_set.name}") + return 0 + + +def _check_artifacts(project_root: Path, output_root: Path) -> int: + _write_artifacts(project_root, output_root) + + examples_dir = project_root / "examples" + stale_files: list[str] = [] + for artifact_set in ARTIFACT_SETS: + for _, output_name in artifact_set.outputs: + expected = (examples_dir / output_name).read_text(encoding="utf-8") + generated = (output_root / output_name).read_text(encoding="utf-8") + if generated != expected: + stale_files.append(output_name) + + if stale_files: + print("stale example artifacts detected:", file=sys.stderr) + for name in stale_files: + print(f" {name}", file=sys.stderr) + print("run scripts/regenerate-example-artifacts.py and commit the updated files.", file=sys.stderr) + return 1 + + print("all checked example artifacts are up to date") + return 0 + + +def _run_artifact_set(project_root: Path, output_root: Path, artifact_set: ExampleArtifactSet) -> None: + output_root.mkdir(parents=True, exist_ok=True) + command = [sys.executable, "-m", "sbom_diff_risk.cli", "compare", *artifact_set.base_args] + for flag, output_name in artifact_set.outputs: + command.extend([flag, str(output_root / output_name)]) + + env = dict(os.environ) + src_path = str(project_root / "src") + env["PYTHONPATH"] = src_path if not env.get("PYTHONPATH") else f"{src_path}{os.pathsep}{env['PYTHONPATH']}" + + result = subprocess.run( + command, + cwd=project_root, + text=True, + capture_output=True, + env=env, + ) + if result.returncode not in artifact_set.expected_exit_codes: + detail = result.stderr.strip() or result.stdout.strip() + raise RuntimeError( + f"{artifact_set.name} exited with {result.returncode}; " + f"expected {artifact_set.expected_exit_codes}: {detail}" + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/sbom-diff-and-risk/tests/test_example_artifacts.py b/tools/sbom-diff-and-risk/tests/test_example_artifacts.py new file mode 100644 index 0000000..50900ea --- /dev/null +++ b/tools/sbom-diff-and-risk/tests/test_example_artifacts.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + + +def test_regenerate_example_artifacts_check_mode_passes() -> None: + project_root = Path(__file__).resolve().parents[1] + + result = subprocess.run( + [ + sys.executable, + str(project_root / "scripts" / "regenerate-example-artifacts.py"), + "--check", + ], + cwd=project_root, + text=True, + capture_output=True, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "all checked example artifacts are up to date" in result.stdout