From 59e00489e96a43ca1c83fe0f9b1d1d468b765a77 Mon Sep 17 00:00:00 2001 From: Ralf Anton Beier Date: Wed, 22 Apr 2026 06:49:31 +0200 Subject: [PATCH 1/4] feat(cli): add --fail-on flag to validate New flag on `rivet validate`: --fail-on error (default, current behavior), --fail-on warning, --fail-on info. Exit code 1 when any diagnostic at or above the given severity is emitted. Lets CI tighten the traceability gate over time without forcing every warning to be promoted in the schema. Tests cover all three outcomes: default --fail-on error on a warning-only project exits 0, --fail-on warning on the same project exits 1, and an invalid value is rejected with a clear error message. Implements: REQ-007 Co-Authored-By: Claude Opus 4.7 (1M context) --- rivet-cli/src/main.rs | 40 ++++++++- rivet-cli/tests/cli_commands.rs | 138 ++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+), 1 deletion(-) diff --git a/rivet-cli/src/main.rs b/rivet-cli/src/main.rs index 8344706..f96440b 100644 --- a/rivet-cli/src/main.rs +++ b/rivet-cli/src/main.rs @@ -233,6 +233,12 @@ enum Command { /// Path to feature-to-artifact binding YAML file #[arg(long)] binding: Option, + + /// Minimum severity that causes exit code 1. Values: "error" (default), + /// "warning", "info". E.g. --fail-on warning tightens the gate so any + /// warning (or error) fails the run. + #[arg(long, default_value = "error")] + fail_on: String, }, /// Show a single artifact by ID @@ -927,6 +933,7 @@ fn run(cli: Cli) -> Result { model, variant, binding, + fail_on, } => cmd_validate( &cli, format, @@ -937,6 +944,7 @@ fn run(cli: Cli) -> Result { model.as_deref(), variant.as_deref(), binding.as_deref(), + fail_on, ), Command::List { r#type, @@ -3091,8 +3099,10 @@ fn cmd_validate( model_path: Option<&std::path::Path>, variant_path: Option<&std::path::Path>, binding_path: Option<&std::path::Path>, + fail_on: &str, ) -> Result { validate_format(format, &["text", "json"])?; + let fail_on_threshold = parse_fail_on(fail_on)?; check_for_updates(); let ctx = ProjectContext::load_with_docs(cli)?; @@ -3511,7 +3521,33 @@ fn cmd_validate( } } - Ok(errors == 0 && cross_errors == 0) + // Exit-code gate: fail if any diagnostic at or above the configured + // severity threshold is present. Cross-repo broken refs are always + // treated as errors for this purpose (they aren't classified by + // severity today). + let has_threshold_hit = match fail_on_threshold { + Severity::Error => errors > 0 || cross_errors > 0, + Severity::Warning => errors > 0 || cross_errors > 0 || warnings > 0, + Severity::Info => { + errors > 0 || cross_errors > 0 || warnings > 0 || infos > 0 + } + }; + Ok(!has_threshold_hit) +} + +/// Parse the `--fail-on` flag into a `Severity` threshold. +/// +/// Accepts `error` (default), `warning`, `info` (case-insensitive). +fn parse_fail_on(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "error" => Ok(Severity::Error), + "warning" | "warn" => Ok(Severity::Warning), + "info" => Ok(Severity::Info), + other => anyhow::bail!( + "invalid --fail-on value '{}' — valid options: error, warning, info", + other + ), + } } /// Run core validation via the salsa incremental database. @@ -5154,6 +5190,7 @@ fn cmd_diff( model: None, variant: None, binding: None, + fail_on: "error".to_string(), }, }; let head_cli = Cli { @@ -5169,6 +5206,7 @@ fn cmd_diff( model: None, variant: None, binding: None, + fail_on: "error".to_string(), }, }; let bc = ProjectContext::load(&base_cli)?; diff --git a/rivet-cli/tests/cli_commands.rs b/rivet-cli/tests/cli_commands.rs index 904fdab..dc3c487 100644 --- a/rivet-cli/tests/cli_commands.rs +++ b/rivet-cli/tests/cli_commands.rs @@ -1322,3 +1322,141 @@ fn all_json_outputs_are_valid() { ); } } + +// ── rivet validate --fail-on ───────────────────────────────── + +/// Build a small project with a single requirement that has no backlink +/// from a feature. The dev schema's `requirement-coverage` rule is a +/// warning — so validation emits 0 errors and 1 warning. This is the +/// fixture used by the `--fail-on` tests. +/// +/// Returns the tempdir so the caller controls its lifetime. +fn warning_only_project() -> tempfile::TempDir { + let tmp = tempfile::tempdir().expect("create temp dir"); + let dir = tmp.path(); + + // Init the dev preset (which seeds REQ-001 satisfied by FEAT-001), + // then overwrite the sample with a requirement that has no + // satisfying feature so the coverage warning fires. + let init = Command::new(rivet_bin()) + .args(["init", "--preset", "dev", "--dir", dir.to_str().unwrap()]) + .output() + .expect("init"); + assert!( + init.status.success(), + "init must succeed: {}", + String::from_utf8_lossy(&init.stderr) + ); + + let artifacts = dir.join("artifacts").join("requirements.yaml"); + // `active` status keeps rule severity at its declared level + // (warning for `requirement-coverage`). Draft would downgrade to info. + std::fs::write( + &artifacts, + "artifacts:\n - id: REQ-001\n type: requirement\n \ + title: Orphan requirement\n status: active\n \ + description: >\n Unsatisfied — triggers \ + requirement-coverage warning.\n tags: [core]\n \ + fields:\n priority: must\n category: functional\n", + ) + .expect("write fixture"); + + tmp +} + +/// `rivet validate --fail-on error` (the default) must exit 0 on a +/// project that only emits warnings. +#[test] +fn validate_fail_on_error_ignores_warnings() { + let tmp = warning_only_project(); + let out = Command::new(rivet_bin()) + .args([ + "--project", + tmp.path().to_str().unwrap(), + "validate", + "--format", + "json", + "--fail-on", + "error", + ]) + .output() + .expect("validate"); + + let stdout = String::from_utf8_lossy(&out.stdout); + let stderr = String::from_utf8_lossy(&out.stderr); + let parsed: serde_json::Value = + serde_json::from_str(&stdout).expect("validate JSON"); + + // Sanity: 0 errors, at least 1 warning. + assert_eq!( + parsed.get("errors").and_then(|v| v.as_u64()).unwrap_or(99), + 0, + "expected 0 errors, got:\n{stdout}" + ); + assert!( + parsed + .get("warnings") + .and_then(|v| v.as_u64()) + .unwrap_or(0) + >= 1, + "expected >=1 warning, got:\n{stdout}" + ); + + assert!( + out.status.success(), + "--fail-on error must exit 0 when there are only warnings.\n\ + stdout: {stdout}\nstderr: {stderr}" + ); +} + +/// `rivet validate --fail-on warning` must exit 1 on the same project +/// (warnings promote to failures). +#[test] +fn validate_fail_on_warning_fails_on_warnings() { + let tmp = warning_only_project(); + let out = Command::new(rivet_bin()) + .args([ + "--project", + tmp.path().to_str().unwrap(), + "validate", + "--format", + "json", + "--fail-on", + "warning", + ]) + .output() + .expect("validate"); + + let stdout = String::from_utf8_lossy(&out.stdout); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + !out.status.success(), + "--fail-on warning must exit non-zero when warnings are present.\n\ + stdout: {stdout}\nstderr: {stderr}" + ); +} + +/// An invalid `--fail-on` value is rejected up-front. +#[test] +fn validate_fail_on_invalid_value_rejected() { + let out = Command::new(rivet_bin()) + .args([ + "--project", + project_root().to_str().unwrap(), + "validate", + "--fail-on", + "bogus", + ]) + .output() + .expect("validate"); + + assert!( + !out.status.success(), + "--fail-on bogus must fail" + ); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains("bogus") || stderr.contains("fail-on"), + "error must mention the bad value, got: {stderr}" + ); +} From cbe22a1b522bc4e7a537497f43644321496515b8 Mon Sep 17 00:00:00 2001 From: Ralf Anton Beier Date: Wed, 22 Apr 2026 06:51:36 +0200 Subject: [PATCH 2/4] feat(cli): include errors/warnings/infos in stats JSON output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `rivet stats --format json` now exposes the same severity breakdown as `rivet validate --format json` (new fields: errors, warnings, infos). Removes the need for consumers to make a second validate call just to get diagnostic counts when rendering a dashboard or CI summary. Existing fields (total, types, orphans, broken_links) are unchanged — additive only, backward-compatible. The text output gains a trailing "Diagnostics: N error(s), ..." summary line so the human-readable form agrees with JSON. Tests: one asserting the new fields are present and numeric; a cross-command test asserting stats and validate agree on the counts for the current project. Implements: REQ-007 Co-Authored-By: Claude Opus 4.7 (1M context) --- rivet-cli/src/main.rs | 30 +++++++++++++ rivet-cli/tests/cli_commands.rs | 77 +++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/rivet-cli/src/main.rs b/rivet-cli/src/main.rs index f96440b..1c5e6b3 100644 --- a/rivet-cli/src/main.rs +++ b/rivet-cli/src/main.rs @@ -3869,6 +3869,27 @@ fn cmd_stats( // Compute stats once — both formats share the same data. let stats = compute_stats(&store, &graph); + // Diagnostic counts (errors/warnings/infos) — same shape as + // `rivet validate --format json` emits, so consumers don't need a + // second call to get the severity breakdown. + // + // We use the direct validator on the (already scoped) store so the + // counts line up with the visible artifact set when --filter or + // --baseline is in effect. + let diagnostics = validate::validate(&store, &ctx.schema, &graph); + let errors = diagnostics + .iter() + .filter(|d| d.severity == Severity::Error) + .count(); + let warnings = diagnostics + .iter() + .filter(|d| d.severity == Severity::Warning) + .count(); + let infos = diagnostics + .iter() + .filter(|d| d.severity == Severity::Info) + .count(); + if format == "json" { let mut types = serde_json::Map::new(); for (name, count) in &stats.type_counts { @@ -3880,6 +3901,9 @@ fn cmd_stats( "types": types, "orphans": stats.orphans, "broken_links": stats.broken_links, + "errors": errors, + "warnings": warnings, + "infos": infos, }); println!("{}", serde_json::to_string_pretty(&output).unwrap()); } else { @@ -3899,6 +3923,12 @@ fn cmd_stats( if stats.broken_links > 0 { println!("\nBroken links: {}", stats.broken_links); } + + // Diagnostic summary — same numbers as the JSON output. + println!( + "\nDiagnostics: {} error(s), {} warning(s), {} info(s)", + errors, warnings, infos + ); } Ok(true) diff --git a/rivet-cli/tests/cli_commands.rs b/rivet-cli/tests/cli_commands.rs index dc3c487..70adf7d 100644 --- a/rivet-cli/tests/cli_commands.rs +++ b/rivet-cli/tests/cli_commands.rs @@ -1436,6 +1436,83 @@ fn validate_fail_on_warning_fails_on_warnings() { ); } +/// `rivet stats --format json` exposes diagnostic counts so consumers +/// don't need a second `rivet validate --format json` call just to +/// get the severity breakdown. +#[test] +fn stats_json_includes_diagnostic_counts() { + let output = Command::new(rivet_bin()) + .args([ + "--project", + project_root().to_str().unwrap(), + "stats", + "--format", + "json", + ]) + .output() + .expect("stats"); + + assert!( + output.status.success(), + "rivet stats must exit 0: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let stdout = String::from_utf8_lossy(&output.stdout); + let parsed: serde_json::Value = + serde_json::from_str(&stdout).expect("stats JSON must be valid"); + + // Backward-compat: existing fields still present. + assert!(parsed.get("total").is_some(), "'total' still present"); + assert!(parsed.get("types").is_some(), "'types' still present"); + + // New fields, numeric, >=0. + for field in ["errors", "warnings", "infos"] { + let v = parsed.get(field); + assert!( + v.is_some(), + "stats JSON must include '{field}' count, got: {stdout}" + ); + assert!( + v.unwrap().is_u64(), + "'{field}' must be a number, got: {}", + v.unwrap() + ); + } +} + +/// Counts in `rivet stats --format json` must match what +/// `rivet validate --format json` reports for the same project. +#[test] +fn stats_json_counts_match_validate() { + let root = project_root(); + let root_str = root.to_str().unwrap(); + + let stats = Command::new(rivet_bin()) + .args(["--project", root_str, "stats", "--format", "json"]) + .output() + .expect("stats"); + assert!(stats.status.success()); + let stats_json: serde_json::Value = + serde_json::from_slice(&stats.stdout).expect("stats JSON"); + + let validate = Command::new(rivet_bin()) + .args(["--project", root_str, "validate", "--format", "json"]) + .output() + .expect("validate"); + let validate_json: serde_json::Value = + serde_json::from_slice(&validate.stdout).expect("validate JSON"); + + for field in ["errors", "warnings", "infos"] { + let s = stats_json.get(field).and_then(|v| v.as_u64()); + let v = validate_json.get(field).and_then(|v| v.as_u64()); + assert_eq!( + s, v, + "stats vs validate disagree on '{field}': stats={s:?} validate={v:?}" + ); + } +} + /// An invalid `--fail-on` value is rejected up-front. #[test] fn validate_fail_on_invalid_value_rejected() { From 2e361575ec43d945062fa8ab9fead7315b6155ed Mon Sep 17 00:00:00 2001 From: Ralf Anton Beier Date: Wed, 22 Apr 2026 06:53:45 +0200 Subject: [PATCH 3/4] feat(cli): polish coverage --fail-under gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The --fail-under flag already gated exit code on overall coverage. This commit hardens and documents the CI-gate use case: - JSON output echoes a new `threshold: { fail_under, passed }` block when the flag is set, so consumers can distinguish a clean run from a gated failure without parsing stderr. - Text output prints a "✔ coverage N.N% meets threshold M.M%" line on success to match the existing failure message. - JSON output now carries `"command": "coverage"` for consistency with the rest of the --format json envelopes. Tests: --fail-under 0 passes, --fail-under 101 fails, no flag is report-only, and JSON carries the threshold echo. Implements: REQ-007 Co-Authored-By: Claude Opus 4.7 (1M context) --- rivet-cli/src/main.rs | 20 +++++- rivet-cli/tests/cli_commands.rs | 107 ++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 2 deletions(-) diff --git a/rivet-cli/src/main.rs b/rivet-cli/src/main.rs index 1c5e6b3..44a77b4 100644 --- a/rivet-cli/src/main.rs +++ b/rivet-cli/src/main.rs @@ -4020,7 +4020,8 @@ fn cmd_coverage( let total: usize = report.entries.iter().map(|e| e.total).sum(); let covered: usize = report.entries.iter().map(|e| e.covered).sum(); let overall_pct = (report.overall_coverage() * 10.0).round() / 10.0; - let output = serde_json::json!({ + let mut output = serde_json::json!({ + "command": "coverage", "rules": rules_json, "overall": { "covered": covered, @@ -4028,6 +4029,16 @@ fn cmd_coverage( "percentage": overall_pct, }, }); + // Echo the threshold + pass/fail result when --fail-under is in + // effect so CI consumers can programmatically distinguish a + // clean run from a gated failure without parsing stderr. + if let Some(&threshold) = fail_under { + let passed = report.overall_coverage() >= threshold; + output["threshold"] = serde_json::json!({ + "fail_under": threshold, + "passed": passed, + }); + } println!("{}", serde_json::to_string_pretty(&output).unwrap()); } else { println!("Traceability Coverage Report\n"); @@ -4071,10 +4082,15 @@ fn cmd_coverage( let overall = report.overall_coverage(); if overall < threshold { eprintln!( - "\nerror: overall coverage {:.1}% is below threshold {:.1}%", + "\nerror: overall coverage {:.1}% is below threshold {:.1}% (--fail-under)", overall, threshold ); return Ok(false); + } else if format != "json" { + println!( + "\n\u{2714} coverage {:.1}% meets threshold {:.1}%", + overall, threshold + ); } } diff --git a/rivet-cli/tests/cli_commands.rs b/rivet-cli/tests/cli_commands.rs index 70adf7d..2b4f274 100644 --- a/rivet-cli/tests/cli_commands.rs +++ b/rivet-cli/tests/cli_commands.rs @@ -1436,6 +1436,113 @@ fn validate_fail_on_warning_fails_on_warnings() { ); } +// ── rivet coverage --fail-under ───────────────────────────────────────── + +/// `rivet coverage --format json` echoes the threshold block when +/// `--fail-under` is set. Consumers can check `threshold.passed` to +/// distinguish a clean run from a gated failure without parsing stderr. +#[test] +fn coverage_json_echoes_threshold() { + let output = Command::new(rivet_bin()) + .args([ + "--project", + project_root().to_str().unwrap(), + "coverage", + "--format", + "json", + "--fail-under", + "0", + ]) + .output() + .expect("coverage"); + assert!(output.status.success()); + let parsed: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("coverage JSON"); + let threshold = parsed + .get("threshold") + .expect("threshold block present when --fail-under set"); + assert_eq!( + threshold + .get("fail_under") + .and_then(|v| v.as_f64()) + .unwrap_or(-1.0), + 0.0 + ); + assert_eq!( + threshold.get("passed").and_then(|v| v.as_bool()), + Some(true) + ); +} + +/// `rivet coverage --fail-under 0` always succeeds (any coverage ≥ 0%). +#[test] +fn coverage_fail_under_zero_passes() { + let output = Command::new(rivet_bin()) + .args([ + "--project", + project_root().to_str().unwrap(), + "coverage", + "--fail-under", + "0", + ]) + .output() + .expect("coverage"); + + assert!( + output.status.success(), + "--fail-under 0 must always pass. stderr:\n{}", + String::from_utf8_lossy(&output.stderr) + ); +} + +/// `rivet coverage --fail-under 101` always fails (no project has > 100%). +#[test] +fn coverage_fail_under_above_100_fails() { + let output = Command::new(rivet_bin()) + .args([ + "--project", + project_root().to_str().unwrap(), + "coverage", + "--fail-under", + "101", + ]) + .output() + .expect("coverage"); + + assert!( + !output.status.success(), + "--fail-under 101 must fail. stdout:\n{}", + String::from_utf8_lossy(&output.stdout) + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("below threshold") || stderr.contains("coverage"), + "error message should mention threshold, got:\n{stderr}" + ); +} + +/// Without `--fail-under`, coverage is report-only — a low-coverage +/// project still exits 0. +#[test] +fn coverage_without_fail_under_is_report_only() { + let output = Command::new(rivet_bin()) + .args([ + "--project", + project_root().to_str().unwrap(), + "coverage", + "--format", + "json", + ]) + .output() + .expect("coverage"); + + assert!( + output.status.success(), + "coverage without --fail-under must exit 0. stderr:\n{}", + String::from_utf8_lossy(&output.stderr) + ); +} + /// `rivet stats --format json` exposes diagnostic counts so consumers /// don't need a second `rivet validate --format json` call just to /// get the severity breakdown. From 88c7f37517d27042810504042995b5749e424169 Mon Sep 17 00:00:00 2001 From: Ralf Anton Beier Date: Wed, 22 Apr 2026 07:05:24 +0200 Subject: [PATCH 4/4] feat(cli): publish JSON schemas for --format json outputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds draft-2020-12 JSON Schemas describing every `--format json` CLI output so downstream consumers can validate against a machine-readable contract instead of reverse-engineering field layouts: schemas/json/validate-output.schema.json schemas/json/stats-output.schema.json schemas/json/coverage-output.schema.json schemas/json/list-output.schema.json Schemas are hand-written (rivet has no `schemars` dependency today — grep of workspace Cargo.toml files returned zero hits — and pulling it in just for four small schemas is heavier than the schemas themselves). Two new subcommands under `rivet schema` surface the schemas: rivet schema list-json # enumerate shipped schemas + paths rivet schema get-json # print path for one rivet schema get-json --content # print schema content Tests cover: - every shipped schema file is valid JSON with required metadata - `schema list-json --format json` lists all four, all files exist - `schema get-json ` round-trips path-and-content for all four - an unknown name is rejected with a helpful error - the actual `rivet validate` / `rivet stats` JSON output contains every `required` field declared in the corresponding schema — so future field drift fails CI instead of silently breaking consumers Implements: REQ-007 Co-Authored-By: Claude Opus 4.7 (1M context) --- rivet-cli/src/main.rs | 150 ++++++++++++++ rivet-cli/tests/cli_commands.rs | 251 +++++++++++++++++++++++ schemas/json/coverage-output.schema.json | 74 +++++++ schemas/json/list-output.schema.json | 41 ++++ schemas/json/stats-output.schema.json | 62 ++++++ schemas/json/validate-output.schema.json | 162 +++++++++++++++ 6 files changed, 740 insertions(+) create mode 100644 schemas/json/coverage-output.schema.json create mode 100644 schemas/json/list-output.schema.json create mode 100644 schemas/json/stats-output.schema.json create mode 100644 schemas/json/validate-output.schema.json diff --git a/rivet-cli/src/main.rs b/rivet-cli/src/main.rs index 44a77b4..2647c8d 100644 --- a/rivet-cli/src/main.rs +++ b/rivet-cli/src/main.rs @@ -759,6 +759,27 @@ enum SchemaAction { #[arg(short, long, default_value = "text")] format: String, }, + /// List JSON schemas describing `--format json` CLI outputs + /// + /// Rivet ships draft-2020-12 JSON Schemas for every `--format json` + /// output (validate, stats, coverage, list). Consumers can pipe + /// the CLI output through a JSON Schema validator to catch + /// regressions when CLI fields are added or removed. + ListJson { + /// Output format: "text" (default) or "json" + #[arg(short, long, default_value = "text")] + format: String, + }, + /// Print the path (or content) of a JSON schema for a given CLI output + /// + /// Valid output names: validate, stats, coverage, list. + GetJson { + /// Output name (validate | stats | coverage | list) + name: String, + /// Print the schema content instead of just its path + #[arg(long)] + content: bool, + }, } #[derive(Debug, Subcommand)] @@ -5834,6 +5855,20 @@ fn cmd_docs( /// Introspect loaded schemas. fn cmd_schema(cli: &Cli, action: &SchemaAction) -> Result { + // `list-json` / `get-json` don't need the project schema graph — + // they describe CLI output shapes, not artifact types. Handle them + // before the expensive load. + match action { + SchemaAction::ListJson { format } => { + validate_format(format, &["text", "json"])?; + return cmd_schema_list_json(cli, format); + } + SchemaAction::GetJson { name, content } => { + return cmd_schema_get_json(cli, name, *content); + } + _ => {} + } + let schemas_dir = resolve_schemas_dir(cli); let config_path = cli.project.join("rivet.yaml"); let schema_names = if config_path.exists() { @@ -5875,11 +5910,126 @@ fn cmd_schema(cli: &Cli, action: &SchemaAction) -> Result { }; schema_cmd::cmd_info(&schema_file, format) } + SchemaAction::ListJson { .. } | SchemaAction::GetJson { .. } => unreachable!(), }; print!("{output}"); Ok(true) } +/// The four CLI subcommands that emit machine-readable JSON along with the +/// JSON schema file that describes their output. +const JSON_SCHEMA_REGISTRY: &[(&str, &str, &str)] = &[ + ( + "validate", + "schemas/json/validate-output.schema.json", + "rivet validate --format json", + ), + ( + "stats", + "schemas/json/stats-output.schema.json", + "rivet stats --format json", + ), + ( + "coverage", + "schemas/json/coverage-output.schema.json", + "rivet coverage --format json", + ), + ( + "list", + "schemas/json/list-output.schema.json", + "rivet list --format json", + ), +]; + +/// Resolve a schema file path against `--schemas` (if set) or the +/// bundled repo-relative `schemas/` directory. We need a slightly +/// different heuristic than `resolve_schemas_dir` — JSON schemas live +/// under `schemas/json/` regardless of whether the user overrode the +/// YAML schemas path. +fn resolve_json_schema(cli: &Cli, relative: &str) -> PathBuf { + // Strip the leading "schemas/" — we'll reattach it whether we use + // the override or the default. + let sub = relative.strip_prefix("schemas/").unwrap_or(relative); + if let Some(ref s) = cli.schemas { + return s.join(sub); + } + // Prefer the sibling of the current project dir (`/schemas`) + // when the project has one; fall back to repo-local. + let project_schemas = cli.project.join("schemas").join(sub); + if project_schemas.exists() { + return project_schemas; + } + PathBuf::from(relative) +} + +fn cmd_schema_list_json(cli: &Cli, format: &str) -> Result { + let entries: Vec<(String, PathBuf, String, bool)> = JSON_SCHEMA_REGISTRY + .iter() + .map(|(name, rel, desc)| { + let path = resolve_json_schema(cli, rel); + let exists = path.exists(); + (name.to_string(), path, desc.to_string(), exists) + }) + .collect(); + + if format == "json" { + let items: Vec = entries + .iter() + .map(|(name, path, desc, exists)| { + serde_json::json!({ + "name": name, + "path": path.display().to_string(), + "describes": desc, + "exists": exists, + }) + }) + .collect(); + let output = serde_json::json!({ + "command": "schema-list-json", + "count": items.len(), + "schemas": items, + }); + println!("{}", serde_json::to_string_pretty(&output).unwrap()); + } else { + println!("JSON schemas for rivet --format json outputs:\n"); + let header = format!( + " {:<12} {:<72} {}", + "Name", "Path", "Describes" + ); + println!("{header}"); + let sep = "-".repeat(110); + println!(" {sep}"); + for (name, path, desc, exists) in &entries { + let marker = if *exists { " " } else { "!" }; + let path_str = path.display().to_string(); + println!(" {marker} {name:<10} {path_str:<72} {desc}"); + } + println!("\nUse: rivet schema get-json # print path"); + println!(" rivet schema get-json --content # print schema JSON"); + } + Ok(true) +} + +fn cmd_schema_get_json(cli: &Cli, name: &str, print_content: bool) -> Result { + let Some((_, rel, _)) = JSON_SCHEMA_REGISTRY.iter().find(|(n, _, _)| *n == name) else { + let valid: Vec<&str> = JSON_SCHEMA_REGISTRY.iter().map(|(n, _, _)| *n).collect(); + anyhow::bail!( + "unknown JSON schema '{}' — valid names: {}", + name, + valid.join(", ") + ); + }; + let path = resolve_json_schema(cli, rel); + if print_content { + let content = std::fs::read_to_string(&path) + .with_context(|| format!("reading {}", path.display()))?; + print!("{content}"); + } else { + println!("{}", path.display()); + } + Ok(true) +} + /// Generate .rivet/agent-context.md from project state. fn cmd_context(cli: &Cli) -> Result { let ctx = ProjectContext::load_with_docs(cli)?; diff --git a/rivet-cli/tests/cli_commands.rs b/rivet-cli/tests/cli_commands.rs index 2b4f274..247ac31 100644 --- a/rivet-cli/tests/cli_commands.rs +++ b/rivet-cli/tests/cli_commands.rs @@ -1620,6 +1620,257 @@ fn stats_json_counts_match_validate() { } } +// ── rivet schema list-json / get-json ─────────────────────────────────── + +/// `rivet schema list-json --format json` lists all shipped JSON +/// schemas describing `--format json` output shapes. +#[test] +fn schema_list_json_produces_valid_output() { + let output = Command::new(rivet_bin()) + .args([ + "--project", + project_root().to_str().unwrap(), + "schema", + "list-json", + "--format", + "json", + ]) + .output() + .expect("schema list-json"); + + assert!( + output.status.success(), + "schema list-json must succeed: {}", + String::from_utf8_lossy(&output.stderr) + ); + let parsed: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("valid JSON"); + assert_eq!( + parsed.get("command").and_then(|v| v.as_str()), + Some("schema-list-json"), + ); + let schemas = parsed + .get("schemas") + .and_then(|v| v.as_array()) + .expect("schemas array"); + let names: Vec<&str> = schemas + .iter() + .filter_map(|e| e.get("name").and_then(|v| v.as_str())) + .collect(); + for expected in ["validate", "stats", "coverage", "list"] { + assert!( + names.contains(&expected), + "expected '{expected}' in list, got {names:?}" + ); + } + // Every shipped schema must resolve to an existing file on disk. + for entry in schemas { + assert_eq!( + entry.get("exists").and_then(|v| v.as_bool()), + Some(true), + "schema entry must exist on disk: {entry}" + ); + } +} + +/// `rivet schema get-json ` prints the path to the schema file, +/// and `--content` reads the schema. +#[test] +fn schema_get_json_returns_path_and_content() { + let root_str = project_root(); + let root_str = root_str.to_str().unwrap(); + + for name in ["validate", "stats", "coverage", "list"] { + // Path mode + let out = Command::new(rivet_bin()) + .args([ + "--project", + root_str, + "schema", + "get-json", + name, + ]) + .output() + .expect("get-json path"); + assert!( + out.status.success(), + "get-json {name} must succeed: {}", + String::from_utf8_lossy(&out.stderr) + ); + let path_str = String::from_utf8_lossy(&out.stdout).trim().to_string(); + let path = std::path::PathBuf::from(&path_str); + assert!( + path.exists(), + "path '{path_str}' printed by get-json {name} must exist" + ); + + // Content mode — verify it's valid JSON and looks like a schema. + let out = Command::new(rivet_bin()) + .args([ + "--project", + root_str, + "schema", + "get-json", + name, + "--content", + ]) + .output() + .expect("get-json --content"); + assert!(out.status.success()); + let content: serde_json::Value = + serde_json::from_slice(&out.stdout).expect("schema JSON parseable"); + assert_eq!( + content.get("$schema").and_then(|v| v.as_str()), + Some("https://json-schema.org/draft/2020-12/schema"), + "{name} schema must declare draft-2020-12" + ); + assert!( + content.get("title").and_then(|v| v.as_str()).is_some(), + "{name} schema must have a title" + ); + } +} + +/// An unknown schema name is rejected with a helpful message. +#[test] +fn schema_get_json_unknown_name_rejected() { + let out = Command::new(rivet_bin()) + .args([ + "--project", + project_root().to_str().unwrap(), + "schema", + "get-json", + "bogus", + ]) + .output() + .expect("get-json"); + + assert!(!out.status.success()); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains("unknown") || stderr.contains("valid names"), + "error must list valid names, got: {stderr}" + ); +} + +/// Every shipped JSON schema file must itself be parseable as JSON +/// (catches hand-written typos at CI time). +#[test] +fn shipped_json_schemas_are_valid_json() { + let schemas_dir = project_root().join("schemas").join("json"); + for name in [ + "validate-output.schema.json", + "stats-output.schema.json", + "coverage-output.schema.json", + "list-output.schema.json", + ] { + let path = schemas_dir.join(name); + let content = std::fs::read_to_string(&path) + .unwrap_or_else(|e| panic!("read {}: {e}", path.display())); + let parsed: serde_json::Value = serde_json::from_str(&content) + .unwrap_or_else(|e| panic!("{} is not valid JSON: {e}", path.display())); + // Minimal well-formed JSON Schema: must be an object with $schema, + // title, type. + assert!(parsed.is_object(), "{name} must be a JSON object"); + for key in ["$schema", "title", "type"] { + assert!( + parsed.get(key).is_some(), + "{name} must declare '{key}'" + ); + } + } +} + +/// The `rivet validate --format json` output must conform to the shipped +/// schema — this catches drift between the CLI output shape and the +/// published schema. +#[test] +fn validate_json_output_matches_shipped_schema() { + let out = Command::new(rivet_bin()) + .args([ + "--project", + project_root().to_str().unwrap(), + "validate", + "--format", + "json", + ]) + .output() + .expect("validate"); + + let parsed: serde_json::Value = + serde_json::from_slice(&out.stdout).expect("validate JSON"); + + // Light-weight schema conformance (no external crate): check the + // required fields listed in validate-output.schema.json are all + // present with the expected types. + let schema_path = project_root() + .join("schemas") + .join("json") + .join("validate-output.schema.json"); + let schema: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(&schema_path).expect("read schema"), + ) + .expect("schema JSON"); + let required = schema + .get("required") + .and_then(|v| v.as_array()) + .expect("required array"); + for req in required { + let key = req.as_str().expect("required[] is string"); + assert!( + parsed.get(key).is_some(), + "validate JSON missing required field '{key}'" + ); + } + // `command` field must match the const in the schema. + assert_eq!( + parsed.get("command").and_then(|v| v.as_str()), + Some("validate"), + ); +} + +/// Same conformance check for `rivet stats --format json`. +#[test] +fn stats_json_output_matches_shipped_schema() { + let out = Command::new(rivet_bin()) + .args([ + "--project", + project_root().to_str().unwrap(), + "stats", + "--format", + "json", + ]) + .output() + .expect("stats"); + + let parsed: serde_json::Value = + serde_json::from_slice(&out.stdout).expect("stats JSON"); + + let schema_path = project_root() + .join("schemas") + .join("json") + .join("stats-output.schema.json"); + let schema: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(&schema_path).expect("read schema"), + ) + .expect("schema JSON"); + let required = schema + .get("required") + .and_then(|v| v.as_array()) + .expect("required array"); + for req in required { + let key = req.as_str().expect("required[] is string"); + assert!( + parsed.get(key).is_some(), + "stats JSON missing required field '{key}'" + ); + } + assert_eq!( + parsed.get("command").and_then(|v| v.as_str()), + Some("stats") + ); +} + /// An invalid `--fail-on` value is rejected up-front. #[test] fn validate_fail_on_invalid_value_rejected() { diff --git a/schemas/json/coverage-output.schema.json b/schemas/json/coverage-output.schema.json new file mode 100644 index 0000000..72ae6ad --- /dev/null +++ b/schemas/json/coverage-output.schema.json @@ -0,0 +1,74 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pulseengine.github.io/rivet/schemas/json/coverage-output.schema.json", + "title": "rivet coverage --format json", + "description": "Machine-readable output produced by `rivet coverage --format json`.", + "type": "object", + "required": ["command", "rules", "overall"], + "additionalProperties": true, + "properties": { + "command": { + "type": "string", + "const": "coverage" + }, + "rules": { + "type": "array", + "description": "One entry per traceability rule covered by the report.", + "items": { + "type": "object", + "required": [ + "name", + "source_type", + "covered", + "total", + "percentage" + ], + "additionalProperties": true, + "properties": { + "name": { "type": "string" }, + "description": { "type": ["string", "null"] }, + "source_type": { "type": "string" }, + "link_type": { "type": ["string", "null"] }, + "direction": { "type": ["string", "null"] }, + "covered": { "type": "integer", "minimum": 0 }, + "total": { "type": "integer", "minimum": 0 }, + "percentage": { + "type": "number", + "minimum": 0, + "maximum": 100 + }, + "uncovered_ids": { + "type": "array", + "items": { "type": "string" } + } + } + } + }, + "overall": { + "type": "object", + "required": ["covered", "total", "percentage"], + "properties": { + "covered": { "type": "integer", "minimum": 0 }, + "total": { "type": "integer", "minimum": 0 }, + "percentage": { + "type": "number", + "minimum": 0, + "maximum": 100 + } + } + }, + "threshold": { + "type": "object", + "description": "Present when --fail-under is set. Added in v0.4.1.", + "required": ["fail_under", "passed"], + "properties": { + "fail_under": { + "type": "number", + "minimum": 0, + "maximum": 100 + }, + "passed": { "type": "boolean" } + } + } + } +} diff --git a/schemas/json/list-output.schema.json b/schemas/json/list-output.schema.json new file mode 100644 index 0000000..bedb70e --- /dev/null +++ b/schemas/json/list-output.schema.json @@ -0,0 +1,41 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pulseengine.github.io/rivet/schemas/json/list-output.schema.json", + "title": "rivet list --format json", + "description": "Machine-readable output produced by `rivet list --format json`.", + "type": "object", + "required": ["command", "count", "artifacts"], + "additionalProperties": true, + "properties": { + "command": { + "type": "string", + "const": "list" + }, + "count": { + "type": "integer", + "minimum": 0 + }, + "artifacts": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "type", "title"], + "additionalProperties": true, + "properties": { + "id": { "type": "string" }, + "type": { "type": "string" }, + "title": { "type": "string" }, + "status": { + "type": "string", + "description": "Status string, or '-' if unset." + }, + "links": { + "type": "integer", + "minimum": 0, + "description": "Number of outgoing links from this artifact." + } + } + } + } + } +} diff --git a/schemas/json/stats-output.schema.json b/schemas/json/stats-output.schema.json new file mode 100644 index 0000000..d368284 --- /dev/null +++ b/schemas/json/stats-output.schema.json @@ -0,0 +1,62 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pulseengine.github.io/rivet/schemas/json/stats-output.schema.json", + "title": "rivet stats --format json", + "description": "Machine-readable output produced by `rivet stats --format json`.", + "type": "object", + "required": [ + "command", + "total", + "types", + "orphans", + "broken_links", + "errors", + "warnings", + "infos" + ], + "additionalProperties": true, + "properties": { + "command": { + "type": "string", + "const": "stats" + }, + "total": { + "type": "integer", + "minimum": 0, + "description": "Total artifacts in the scoped store (post-filter, post-baseline)." + }, + "types": { + "type": "object", + "description": "Per-artifact-type counts; keys are artifact-type names.", + "additionalProperties": { + "type": "integer", + "minimum": 0 + } + }, + "orphans": { + "type": "array", + "description": "Artifact IDs with no incoming or outgoing links.", + "items": { "type": "string" } + }, + "broken_links": { + "type": "integer", + "minimum": 0, + "description": "Count of links whose target does not exist in the store." + }, + "errors": { + "type": "integer", + "minimum": 0, + "description": "Diagnostic count, severity 'error'. Added in v0.4.1." + }, + "warnings": { + "type": "integer", + "minimum": 0, + "description": "Diagnostic count, severity 'warning'. Added in v0.4.1." + }, + "infos": { + "type": "integer", + "minimum": 0, + "description": "Diagnostic count, severity 'info'. Added in v0.4.1." + } + } +} diff --git a/schemas/json/validate-output.schema.json b/schemas/json/validate-output.schema.json new file mode 100644 index 0000000..5986cf8 --- /dev/null +++ b/schemas/json/validate-output.schema.json @@ -0,0 +1,162 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pulseengine.github.io/rivet/schemas/json/validate-output.schema.json", + "title": "rivet validate --format json", + "description": "Machine-readable output produced by `rivet validate --format json`.", + "type": "object", + "required": [ + "command", + "result", + "errors", + "warnings", + "infos", + "diagnostics" + ], + "additionalProperties": true, + "properties": { + "command": { + "type": "string", + "const": "validate" + }, + "result": { + "type": "string", + "enum": ["PASS", "FAIL"], + "description": "Overall result. FAIL iff there is at least one error or broken cross-ref." + }, + "errors": { + "type": "integer", + "minimum": 0, + "description": "Count of diagnostics with severity 'error'." + }, + "warnings": { + "type": "integer", + "minimum": 0 + }, + "infos": { + "type": "integer", + "minimum": 0 + }, + "cross_repo_broken": { + "type": "integer", + "minimum": 0 + }, + "backlinks": { + "type": "integer", + "minimum": 0 + }, + "circular_deps": { + "type": "integer", + "minimum": 0 + }, + "version_conflicts": { + "type": "integer", + "minimum": 0 + }, + "lifecycle_gaps": { + "type": "integer", + "minimum": 0 + }, + "diagnostics": { + "type": "array", + "items": { + "type": "object", + "required": ["severity", "message"], + "additionalProperties": true, + "properties": { + "severity": { + "type": "string", + "enum": ["error", "warning", "info"] + }, + "artifact_id": { + "type": ["string", "null"] + }, + "message": { + "type": "string" + } + } + } + }, + "broken_cross_refs": { + "type": "array", + "items": { + "type": "object", + "required": ["reference", "reason"], + "properties": { + "reference": { "type": "string" }, + "reason": { "type": "string" } + } + } + }, + "cross_repo_backlinks": { + "type": "array", + "items": { + "type": "object", + "required": ["source_prefix", "source_id", "target"], + "properties": { + "source_prefix": { "type": "string" }, + "source_id": { "type": "string" }, + "target": { "type": "string" } + } + } + }, + "circular_dependencies": { + "type": "array", + "items": { + "type": "object", + "required": ["chain"], + "properties": { + "chain": { + "type": "array", + "items": { "type": "string" } + } + } + } + }, + "version_conflict_details": { + "type": "array", + "items": { + "type": "object", + "required": ["repo_identifier", "versions"], + "properties": { + "repo_identifier": { "type": "string" }, + "versions": { + "type": "array", + "items": { + "type": "object", + "required": ["declared_by", "version"], + "properties": { + "declared_by": { "type": "string" }, + "version": { "type": "string" } + } + } + } + } + } + }, + "lifecycle_coverage": { + "type": "array", + "items": { + "type": "object", + "required": ["artifact_id", "artifact_type", "missing"], + "properties": { + "artifact_id": { "type": "string" }, + "artifact_type": { "type": "string" }, + "status": { "type": ["string", "null"] }, + "missing": { + "type": "array", + "items": { "type": "string" } + } + } + } + }, + "variant": { + "type": "object", + "required": ["name", "bound_artifacts", "resolved_artifacts"], + "properties": { + "name": { "type": "string" }, + "bound_artifacts": { "type": "integer", "minimum": 0 }, + "resolved_artifacts": { "type": "integer", "minimum": 0 } + } + } + } +}