diff --git a/rivet-cli/src/main.rs b/rivet-cli/src/main.rs index 8344706..2647c8d 100644 --- a/rivet-cli/src/main.rs +++ b/rivet-cli/src/main.rs @@ -233,6 +233,12 @@ enum Command { /// Path to feature-to-artifact binding YAML file #[arg(long)] binding: Option, + + /// Minimum severity that causes exit code 1. Values: "error" (default), + /// "warning", "info". E.g. --fail-on warning tightens the gate so any + /// warning (or error) fails the run. + #[arg(long, default_value = "error")] + fail_on: String, }, /// Show a single artifact by ID @@ -753,6 +759,27 @@ enum SchemaAction { #[arg(short, long, default_value = "text")] format: String, }, + /// List JSON schemas describing `--format json` CLI outputs + /// + /// Rivet ships draft-2020-12 JSON Schemas for every `--format json` + /// output (validate, stats, coverage, list). Consumers can pipe + /// the CLI output through a JSON Schema validator to catch + /// regressions when CLI fields are added or removed. + ListJson { + /// Output format: "text" (default) or "json" + #[arg(short, long, default_value = "text")] + format: String, + }, + /// Print the path (or content) of a JSON schema for a given CLI output + /// + /// Valid output names: validate, stats, coverage, list. + GetJson { + /// Output name (validate | stats | coverage | list) + name: String, + /// Print the schema content instead of just its path + #[arg(long)] + content: bool, + }, } #[derive(Debug, Subcommand)] @@ -927,6 +954,7 @@ fn run(cli: Cli) -> Result { model, variant, binding, + fail_on, } => cmd_validate( &cli, format, @@ -937,6 +965,7 @@ fn run(cli: Cli) -> Result { model.as_deref(), variant.as_deref(), binding.as_deref(), + fail_on, ), Command::List { r#type, @@ -3091,8 +3120,10 @@ fn cmd_validate( model_path: Option<&std::path::Path>, variant_path: Option<&std::path::Path>, binding_path: Option<&std::path::Path>, + fail_on: &str, ) -> Result { validate_format(format, &["text", "json"])?; + let fail_on_threshold = parse_fail_on(fail_on)?; check_for_updates(); let ctx = ProjectContext::load_with_docs(cli)?; @@ -3511,7 +3542,33 @@ fn cmd_validate( } } - Ok(errors == 0 && cross_errors == 0) + // Exit-code gate: fail if any diagnostic at or above the configured + // severity threshold is present. Cross-repo broken refs are always + // treated as errors for this purpose (they aren't classified by + // severity today). + let has_threshold_hit = match fail_on_threshold { + Severity::Error => errors > 0 || cross_errors > 0, + Severity::Warning => errors > 0 || cross_errors > 0 || warnings > 0, + Severity::Info => { + errors > 0 || cross_errors > 0 || warnings > 0 || infos > 0 + } + }; + Ok(!has_threshold_hit) +} + +/// Parse the `--fail-on` flag into a `Severity` threshold. +/// +/// Accepts `error` (default), `warning`, `info` (case-insensitive). +fn parse_fail_on(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "error" => Ok(Severity::Error), + "warning" | "warn" => Ok(Severity::Warning), + "info" => Ok(Severity::Info), + other => anyhow::bail!( + "invalid --fail-on value '{}' — valid options: error, warning, info", + other + ), + } } /// Run core validation via the salsa incremental database. @@ -3833,6 +3890,27 @@ fn cmd_stats( // Compute stats once — both formats share the same data. let stats = compute_stats(&store, &graph); + // Diagnostic counts (errors/warnings/infos) — same shape as + // `rivet validate --format json` emits, so consumers don't need a + // second call to get the severity breakdown. + // + // We use the direct validator on the (already scoped) store so the + // counts line up with the visible artifact set when --filter or + // --baseline is in effect. + let diagnostics = validate::validate(&store, &ctx.schema, &graph); + let errors = diagnostics + .iter() + .filter(|d| d.severity == Severity::Error) + .count(); + let warnings = diagnostics + .iter() + .filter(|d| d.severity == Severity::Warning) + .count(); + let infos = diagnostics + .iter() + .filter(|d| d.severity == Severity::Info) + .count(); + if format == "json" { let mut types = serde_json::Map::new(); for (name, count) in &stats.type_counts { @@ -3844,6 +3922,9 @@ fn cmd_stats( "types": types, "orphans": stats.orphans, "broken_links": stats.broken_links, + "errors": errors, + "warnings": warnings, + "infos": infos, }); println!("{}", serde_json::to_string_pretty(&output).unwrap()); } else { @@ -3863,6 +3944,12 @@ fn cmd_stats( if stats.broken_links > 0 { println!("\nBroken links: {}", stats.broken_links); } + + // Diagnostic summary — same numbers as the JSON output. + println!( + "\nDiagnostics: {} error(s), {} warning(s), {} info(s)", + errors, warnings, infos + ); } Ok(true) @@ -3954,7 +4041,8 @@ fn cmd_coverage( let total: usize = report.entries.iter().map(|e| e.total).sum(); let covered: usize = report.entries.iter().map(|e| e.covered).sum(); let overall_pct = (report.overall_coverage() * 10.0).round() / 10.0; - let output = serde_json::json!({ + let mut output = serde_json::json!({ + "command": "coverage", "rules": rules_json, "overall": { "covered": covered, @@ -3962,6 +4050,16 @@ fn cmd_coverage( "percentage": overall_pct, }, }); + // Echo the threshold + pass/fail result when --fail-under is in + // effect so CI consumers can programmatically distinguish a + // clean run from a gated failure without parsing stderr. + if let Some(&threshold) = fail_under { + let passed = report.overall_coverage() >= threshold; + output["threshold"] = serde_json::json!({ + "fail_under": threshold, + "passed": passed, + }); + } println!("{}", serde_json::to_string_pretty(&output).unwrap()); } else { println!("Traceability Coverage Report\n"); @@ -4005,10 +4103,15 @@ fn cmd_coverage( let overall = report.overall_coverage(); if overall < threshold { eprintln!( - "\nerror: overall coverage {:.1}% is below threshold {:.1}%", + "\nerror: overall coverage {:.1}% is below threshold {:.1}% (--fail-under)", overall, threshold ); return Ok(false); + } else if format != "json" { + println!( + "\n\u{2714} coverage {:.1}% meets threshold {:.1}%", + overall, threshold + ); } } @@ -5154,6 +5257,7 @@ fn cmd_diff( model: None, variant: None, binding: None, + fail_on: "error".to_string(), }, }; let head_cli = Cli { @@ -5169,6 +5273,7 @@ fn cmd_diff( model: None, variant: None, binding: None, + fail_on: "error".to_string(), }, }; let bc = ProjectContext::load(&base_cli)?; @@ -5750,6 +5855,20 @@ fn cmd_docs( /// Introspect loaded schemas. fn cmd_schema(cli: &Cli, action: &SchemaAction) -> Result { + // `list-json` / `get-json` don't need the project schema graph — + // they describe CLI output shapes, not artifact types. Handle them + // before the expensive load. + match action { + SchemaAction::ListJson { format } => { + validate_format(format, &["text", "json"])?; + return cmd_schema_list_json(cli, format); + } + SchemaAction::GetJson { name, content } => { + return cmd_schema_get_json(cli, name, *content); + } + _ => {} + } + let schemas_dir = resolve_schemas_dir(cli); let config_path = cli.project.join("rivet.yaml"); let schema_names = if config_path.exists() { @@ -5791,11 +5910,126 @@ fn cmd_schema(cli: &Cli, action: &SchemaAction) -> Result { }; schema_cmd::cmd_info(&schema_file, format) } + SchemaAction::ListJson { .. } | SchemaAction::GetJson { .. } => unreachable!(), }; print!("{output}"); Ok(true) } +/// The four CLI subcommands that emit machine-readable JSON along with the +/// JSON schema file that describes their output. +const JSON_SCHEMA_REGISTRY: &[(&str, &str, &str)] = &[ + ( + "validate", + "schemas/json/validate-output.schema.json", + "rivet validate --format json", + ), + ( + "stats", + "schemas/json/stats-output.schema.json", + "rivet stats --format json", + ), + ( + "coverage", + "schemas/json/coverage-output.schema.json", + "rivet coverage --format json", + ), + ( + "list", + "schemas/json/list-output.schema.json", + "rivet list --format json", + ), +]; + +/// Resolve a schema file path against `--schemas` (if set) or the +/// bundled repo-relative `schemas/` directory. We need a slightly +/// different heuristic than `resolve_schemas_dir` — JSON schemas live +/// under `schemas/json/` regardless of whether the user overrode the +/// YAML schemas path. +fn resolve_json_schema(cli: &Cli, relative: &str) -> PathBuf { + // Strip the leading "schemas/" — we'll reattach it whether we use + // the override or the default. + let sub = relative.strip_prefix("schemas/").unwrap_or(relative); + if let Some(ref s) = cli.schemas { + return s.join(sub); + } + // Prefer the sibling of the current project dir (`/schemas`) + // when the project has one; fall back to repo-local. + let project_schemas = cli.project.join("schemas").join(sub); + if project_schemas.exists() { + return project_schemas; + } + PathBuf::from(relative) +} + +fn cmd_schema_list_json(cli: &Cli, format: &str) -> Result { + let entries: Vec<(String, PathBuf, String, bool)> = JSON_SCHEMA_REGISTRY + .iter() + .map(|(name, rel, desc)| { + let path = resolve_json_schema(cli, rel); + let exists = path.exists(); + (name.to_string(), path, desc.to_string(), exists) + }) + .collect(); + + if format == "json" { + let items: Vec = entries + .iter() + .map(|(name, path, desc, exists)| { + serde_json::json!({ + "name": name, + "path": path.display().to_string(), + "describes": desc, + "exists": exists, + }) + }) + .collect(); + let output = serde_json::json!({ + "command": "schema-list-json", + "count": items.len(), + "schemas": items, + }); + println!("{}", serde_json::to_string_pretty(&output).unwrap()); + } else { + println!("JSON schemas for rivet --format json outputs:\n"); + let header = format!( + " {:<12} {:<72} {}", + "Name", "Path", "Describes" + ); + println!("{header}"); + let sep = "-".repeat(110); + println!(" {sep}"); + for (name, path, desc, exists) in &entries { + let marker = if *exists { " " } else { "!" }; + let path_str = path.display().to_string(); + println!(" {marker} {name:<10} {path_str:<72} {desc}"); + } + println!("\nUse: rivet schema get-json # print path"); + println!(" rivet schema get-json --content # print schema JSON"); + } + Ok(true) +} + +fn cmd_schema_get_json(cli: &Cli, name: &str, print_content: bool) -> Result { + let Some((_, rel, _)) = JSON_SCHEMA_REGISTRY.iter().find(|(n, _, _)| *n == name) else { + let valid: Vec<&str> = JSON_SCHEMA_REGISTRY.iter().map(|(n, _, _)| *n).collect(); + anyhow::bail!( + "unknown JSON schema '{}' — valid names: {}", + name, + valid.join(", ") + ); + }; + let path = resolve_json_schema(cli, rel); + if print_content { + let content = std::fs::read_to_string(&path) + .with_context(|| format!("reading {}", path.display()))?; + print!("{content}"); + } else { + println!("{}", path.display()); + } + Ok(true) +} + /// Generate .rivet/agent-context.md from project state. fn cmd_context(cli: &Cli) -> Result { let ctx = ProjectContext::load_with_docs(cli)?; diff --git a/rivet-cli/tests/cli_commands.rs b/rivet-cli/tests/cli_commands.rs index 904fdab..247ac31 100644 --- a/rivet-cli/tests/cli_commands.rs +++ b/rivet-cli/tests/cli_commands.rs @@ -1322,3 +1322,576 @@ fn all_json_outputs_are_valid() { ); } } + +// ── rivet validate --fail-on ───────────────────────────────── + +/// Build a small project with a single requirement that has no backlink +/// from a feature. The dev schema's `requirement-coverage` rule is a +/// warning — so validation emits 0 errors and 1 warning. This is the +/// fixture used by the `--fail-on` tests. +/// +/// Returns the tempdir so the caller controls its lifetime. +fn warning_only_project() -> tempfile::TempDir { + let tmp = tempfile::tempdir().expect("create temp dir"); + let dir = tmp.path(); + + // Init the dev preset (which seeds REQ-001 satisfied by FEAT-001), + // then overwrite the sample with a requirement that has no + // satisfying feature so the coverage warning fires. + let init = Command::new(rivet_bin()) + .args(["init", "--preset", "dev", "--dir", dir.to_str().unwrap()]) + .output() + .expect("init"); + assert!( + init.status.success(), + "init must succeed: {}", + String::from_utf8_lossy(&init.stderr) + ); + + let artifacts = dir.join("artifacts").join("requirements.yaml"); + // `active` status keeps rule severity at its declared level + // (warning for `requirement-coverage`). Draft would downgrade to info. + std::fs::write( + &artifacts, + "artifacts:\n - id: REQ-001\n type: requirement\n \ + title: Orphan requirement\n status: active\n \ + description: >\n Unsatisfied — triggers \ + requirement-coverage warning.\n tags: [core]\n \ + fields:\n priority: must\n category: functional\n", + ) + .expect("write fixture"); + + tmp +} + +/// `rivet validate --fail-on error` (the default) must exit 0 on a +/// project that only emits warnings. +#[test] +fn validate_fail_on_error_ignores_warnings() { + let tmp = warning_only_project(); + let out = Command::new(rivet_bin()) + .args([ + "--project", + tmp.path().to_str().unwrap(), + "validate", + "--format", + "json", + "--fail-on", + "error", + ]) + .output() + .expect("validate"); + + let stdout = String::from_utf8_lossy(&out.stdout); + let stderr = String::from_utf8_lossy(&out.stderr); + let parsed: serde_json::Value = + serde_json::from_str(&stdout).expect("validate JSON"); + + // Sanity: 0 errors, at least 1 warning. + assert_eq!( + parsed.get("errors").and_then(|v| v.as_u64()).unwrap_or(99), + 0, + "expected 0 errors, got:\n{stdout}" + ); + assert!( + parsed + .get("warnings") + .and_then(|v| v.as_u64()) + .unwrap_or(0) + >= 1, + "expected >=1 warning, got:\n{stdout}" + ); + + assert!( + out.status.success(), + "--fail-on error must exit 0 when there are only warnings.\n\ + stdout: {stdout}\nstderr: {stderr}" + ); +} + +/// `rivet validate --fail-on warning` must exit 1 on the same project +/// (warnings promote to failures). +#[test] +fn validate_fail_on_warning_fails_on_warnings() { + let tmp = warning_only_project(); + let out = Command::new(rivet_bin()) + .args([ + "--project", + tmp.path().to_str().unwrap(), + "validate", + "--format", + "json", + "--fail-on", + "warning", + ]) + .output() + .expect("validate"); + + let stdout = String::from_utf8_lossy(&out.stdout); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + !out.status.success(), + "--fail-on warning must exit non-zero when warnings are present.\n\ + stdout: {stdout}\nstderr: {stderr}" + ); +} + +// ── rivet coverage --fail-under ───────────────────────────────────────── + +/// `rivet coverage --format json` echoes the threshold block when +/// `--fail-under` is set. Consumers can check `threshold.passed` to +/// distinguish a clean run from a gated failure without parsing stderr. +#[test] +fn coverage_json_echoes_threshold() { + let output = Command::new(rivet_bin()) + .args([ + "--project", + project_root().to_str().unwrap(), + "coverage", + "--format", + "json", + "--fail-under", + "0", + ]) + .output() + .expect("coverage"); + assert!(output.status.success()); + let parsed: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("coverage JSON"); + let threshold = parsed + .get("threshold") + .expect("threshold block present when --fail-under set"); + assert_eq!( + threshold + .get("fail_under") + .and_then(|v| v.as_f64()) + .unwrap_or(-1.0), + 0.0 + ); + assert_eq!( + threshold.get("passed").and_then(|v| v.as_bool()), + Some(true) + ); +} + +/// `rivet coverage --fail-under 0` always succeeds (any coverage ≥ 0%). +#[test] +fn coverage_fail_under_zero_passes() { + let output = Command::new(rivet_bin()) + .args([ + "--project", + project_root().to_str().unwrap(), + "coverage", + "--fail-under", + "0", + ]) + .output() + .expect("coverage"); + + assert!( + output.status.success(), + "--fail-under 0 must always pass. stderr:\n{}", + String::from_utf8_lossy(&output.stderr) + ); +} + +/// `rivet coverage --fail-under 101` always fails (no project has > 100%). +#[test] +fn coverage_fail_under_above_100_fails() { + let output = Command::new(rivet_bin()) + .args([ + "--project", + project_root().to_str().unwrap(), + "coverage", + "--fail-under", + "101", + ]) + .output() + .expect("coverage"); + + assert!( + !output.status.success(), + "--fail-under 101 must fail. stdout:\n{}", + String::from_utf8_lossy(&output.stdout) + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("below threshold") || stderr.contains("coverage"), + "error message should mention threshold, got:\n{stderr}" + ); +} + +/// Without `--fail-under`, coverage is report-only — a low-coverage +/// project still exits 0. +#[test] +fn coverage_without_fail_under_is_report_only() { + let output = Command::new(rivet_bin()) + .args([ + "--project", + project_root().to_str().unwrap(), + "coverage", + "--format", + "json", + ]) + .output() + .expect("coverage"); + + assert!( + output.status.success(), + "coverage without --fail-under must exit 0. stderr:\n{}", + String::from_utf8_lossy(&output.stderr) + ); +} + +/// `rivet stats --format json` exposes diagnostic counts so consumers +/// don't need a second `rivet validate --format json` call just to +/// get the severity breakdown. +#[test] +fn stats_json_includes_diagnostic_counts() { + let output = Command::new(rivet_bin()) + .args([ + "--project", + project_root().to_str().unwrap(), + "stats", + "--format", + "json", + ]) + .output() + .expect("stats"); + + assert!( + output.status.success(), + "rivet stats must exit 0: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let stdout = String::from_utf8_lossy(&output.stdout); + let parsed: serde_json::Value = + serde_json::from_str(&stdout).expect("stats JSON must be valid"); + + // Backward-compat: existing fields still present. + assert!(parsed.get("total").is_some(), "'total' still present"); + assert!(parsed.get("types").is_some(), "'types' still present"); + + // New fields, numeric, >=0. + for field in ["errors", "warnings", "infos"] { + let v = parsed.get(field); + assert!( + v.is_some(), + "stats JSON must include '{field}' count, got: {stdout}" + ); + assert!( + v.unwrap().is_u64(), + "'{field}' must be a number, got: {}", + v.unwrap() + ); + } +} + +/// Counts in `rivet stats --format json` must match what +/// `rivet validate --format json` reports for the same project. +#[test] +fn stats_json_counts_match_validate() { + let root = project_root(); + let root_str = root.to_str().unwrap(); + + let stats = Command::new(rivet_bin()) + .args(["--project", root_str, "stats", "--format", "json"]) + .output() + .expect("stats"); + assert!(stats.status.success()); + let stats_json: serde_json::Value = + serde_json::from_slice(&stats.stdout).expect("stats JSON"); + + let validate = Command::new(rivet_bin()) + .args(["--project", root_str, "validate", "--format", "json"]) + .output() + .expect("validate"); + let validate_json: serde_json::Value = + serde_json::from_slice(&validate.stdout).expect("validate JSON"); + + for field in ["errors", "warnings", "infos"] { + let s = stats_json.get(field).and_then(|v| v.as_u64()); + let v = validate_json.get(field).and_then(|v| v.as_u64()); + assert_eq!( + s, v, + "stats vs validate disagree on '{field}': stats={s:?} validate={v:?}" + ); + } +} + +// ── rivet schema list-json / get-json ─────────────────────────────────── + +/// `rivet schema list-json --format json` lists all shipped JSON +/// schemas describing `--format json` output shapes. +#[test] +fn schema_list_json_produces_valid_output() { + let output = Command::new(rivet_bin()) + .args([ + "--project", + project_root().to_str().unwrap(), + "schema", + "list-json", + "--format", + "json", + ]) + .output() + .expect("schema list-json"); + + assert!( + output.status.success(), + "schema list-json must succeed: {}", + String::from_utf8_lossy(&output.stderr) + ); + let parsed: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("valid JSON"); + assert_eq!( + parsed.get("command").and_then(|v| v.as_str()), + Some("schema-list-json"), + ); + let schemas = parsed + .get("schemas") + .and_then(|v| v.as_array()) + .expect("schemas array"); + let names: Vec<&str> = schemas + .iter() + .filter_map(|e| e.get("name").and_then(|v| v.as_str())) + .collect(); + for expected in ["validate", "stats", "coverage", "list"] { + assert!( + names.contains(&expected), + "expected '{expected}' in list, got {names:?}" + ); + } + // Every shipped schema must resolve to an existing file on disk. + for entry in schemas { + assert_eq!( + entry.get("exists").and_then(|v| v.as_bool()), + Some(true), + "schema entry must exist on disk: {entry}" + ); + } +} + +/// `rivet schema get-json ` prints the path to the schema file, +/// and `--content` reads the schema. +#[test] +fn schema_get_json_returns_path_and_content() { + let root_str = project_root(); + let root_str = root_str.to_str().unwrap(); + + for name in ["validate", "stats", "coverage", "list"] { + // Path mode + let out = Command::new(rivet_bin()) + .args([ + "--project", + root_str, + "schema", + "get-json", + name, + ]) + .output() + .expect("get-json path"); + assert!( + out.status.success(), + "get-json {name} must succeed: {}", + String::from_utf8_lossy(&out.stderr) + ); + let path_str = String::from_utf8_lossy(&out.stdout).trim().to_string(); + let path = std::path::PathBuf::from(&path_str); + assert!( + path.exists(), + "path '{path_str}' printed by get-json {name} must exist" + ); + + // Content mode — verify it's valid JSON and looks like a schema. + let out = Command::new(rivet_bin()) + .args([ + "--project", + root_str, + "schema", + "get-json", + name, + "--content", + ]) + .output() + .expect("get-json --content"); + assert!(out.status.success()); + let content: serde_json::Value = + serde_json::from_slice(&out.stdout).expect("schema JSON parseable"); + assert_eq!( + content.get("$schema").and_then(|v| v.as_str()), + Some("https://json-schema.org/draft/2020-12/schema"), + "{name} schema must declare draft-2020-12" + ); + assert!( + content.get("title").and_then(|v| v.as_str()).is_some(), + "{name} schema must have a title" + ); + } +} + +/// An unknown schema name is rejected with a helpful message. +#[test] +fn schema_get_json_unknown_name_rejected() { + let out = Command::new(rivet_bin()) + .args([ + "--project", + project_root().to_str().unwrap(), + "schema", + "get-json", + "bogus", + ]) + .output() + .expect("get-json"); + + assert!(!out.status.success()); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains("unknown") || stderr.contains("valid names"), + "error must list valid names, got: {stderr}" + ); +} + +/// Every shipped JSON schema file must itself be parseable as JSON +/// (catches hand-written typos at CI time). +#[test] +fn shipped_json_schemas_are_valid_json() { + let schemas_dir = project_root().join("schemas").join("json"); + for name in [ + "validate-output.schema.json", + "stats-output.schema.json", + "coverage-output.schema.json", + "list-output.schema.json", + ] { + let path = schemas_dir.join(name); + let content = std::fs::read_to_string(&path) + .unwrap_or_else(|e| panic!("read {}: {e}", path.display())); + let parsed: serde_json::Value = serde_json::from_str(&content) + .unwrap_or_else(|e| panic!("{} is not valid JSON: {e}", path.display())); + // Minimal well-formed JSON Schema: must be an object with $schema, + // title, type. + assert!(parsed.is_object(), "{name} must be a JSON object"); + for key in ["$schema", "title", "type"] { + assert!( + parsed.get(key).is_some(), + "{name} must declare '{key}'" + ); + } + } +} + +/// The `rivet validate --format json` output must conform to the shipped +/// schema — this catches drift between the CLI output shape and the +/// published schema. +#[test] +fn validate_json_output_matches_shipped_schema() { + let out = Command::new(rivet_bin()) + .args([ + "--project", + project_root().to_str().unwrap(), + "validate", + "--format", + "json", + ]) + .output() + .expect("validate"); + + let parsed: serde_json::Value = + serde_json::from_slice(&out.stdout).expect("validate JSON"); + + // Light-weight schema conformance (no external crate): check the + // required fields listed in validate-output.schema.json are all + // present with the expected types. + let schema_path = project_root() + .join("schemas") + .join("json") + .join("validate-output.schema.json"); + let schema: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(&schema_path).expect("read schema"), + ) + .expect("schema JSON"); + let required = schema + .get("required") + .and_then(|v| v.as_array()) + .expect("required array"); + for req in required { + let key = req.as_str().expect("required[] is string"); + assert!( + parsed.get(key).is_some(), + "validate JSON missing required field '{key}'" + ); + } + // `command` field must match the const in the schema. + assert_eq!( + parsed.get("command").and_then(|v| v.as_str()), + Some("validate"), + ); +} + +/// Same conformance check for `rivet stats --format json`. +#[test] +fn stats_json_output_matches_shipped_schema() { + let out = Command::new(rivet_bin()) + .args([ + "--project", + project_root().to_str().unwrap(), + "stats", + "--format", + "json", + ]) + .output() + .expect("stats"); + + let parsed: serde_json::Value = + serde_json::from_slice(&out.stdout).expect("stats JSON"); + + let schema_path = project_root() + .join("schemas") + .join("json") + .join("stats-output.schema.json"); + let schema: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(&schema_path).expect("read schema"), + ) + .expect("schema JSON"); + let required = schema + .get("required") + .and_then(|v| v.as_array()) + .expect("required array"); + for req in required { + let key = req.as_str().expect("required[] is string"); + assert!( + parsed.get(key).is_some(), + "stats JSON missing required field '{key}'" + ); + } + assert_eq!( + parsed.get("command").and_then(|v| v.as_str()), + Some("stats") + ); +} + +/// An invalid `--fail-on` value is rejected up-front. +#[test] +fn validate_fail_on_invalid_value_rejected() { + let out = Command::new(rivet_bin()) + .args([ + "--project", + project_root().to_str().unwrap(), + "validate", + "--fail-on", + "bogus", + ]) + .output() + .expect("validate"); + + assert!( + !out.status.success(), + "--fail-on bogus must fail" + ); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains("bogus") || stderr.contains("fail-on"), + "error must mention the bad value, got: {stderr}" + ); +} diff --git a/schemas/json/coverage-output.schema.json b/schemas/json/coverage-output.schema.json new file mode 100644 index 0000000..72ae6ad --- /dev/null +++ b/schemas/json/coverage-output.schema.json @@ -0,0 +1,74 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pulseengine.github.io/rivet/schemas/json/coverage-output.schema.json", + "title": "rivet coverage --format json", + "description": "Machine-readable output produced by `rivet coverage --format json`.", + "type": "object", + "required": ["command", "rules", "overall"], + "additionalProperties": true, + "properties": { + "command": { + "type": "string", + "const": "coverage" + }, + "rules": { + "type": "array", + "description": "One entry per traceability rule covered by the report.", + "items": { + "type": "object", + "required": [ + "name", + "source_type", + "covered", + "total", + "percentage" + ], + "additionalProperties": true, + "properties": { + "name": { "type": "string" }, + "description": { "type": ["string", "null"] }, + "source_type": { "type": "string" }, + "link_type": { "type": ["string", "null"] }, + "direction": { "type": ["string", "null"] }, + "covered": { "type": "integer", "minimum": 0 }, + "total": { "type": "integer", "minimum": 0 }, + "percentage": { + "type": "number", + "minimum": 0, + "maximum": 100 + }, + "uncovered_ids": { + "type": "array", + "items": { "type": "string" } + } + } + } + }, + "overall": { + "type": "object", + "required": ["covered", "total", "percentage"], + "properties": { + "covered": { "type": "integer", "minimum": 0 }, + "total": { "type": "integer", "minimum": 0 }, + "percentage": { + "type": "number", + "minimum": 0, + "maximum": 100 + } + } + }, + "threshold": { + "type": "object", + "description": "Present when --fail-under is set. Added in v0.4.1.", + "required": ["fail_under", "passed"], + "properties": { + "fail_under": { + "type": "number", + "minimum": 0, + "maximum": 100 + }, + "passed": { "type": "boolean" } + } + } + } +} diff --git a/schemas/json/list-output.schema.json b/schemas/json/list-output.schema.json new file mode 100644 index 0000000..bedb70e --- /dev/null +++ b/schemas/json/list-output.schema.json @@ -0,0 +1,41 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pulseengine.github.io/rivet/schemas/json/list-output.schema.json", + "title": "rivet list --format json", + "description": "Machine-readable output produced by `rivet list --format json`.", + "type": "object", + "required": ["command", "count", "artifacts"], + "additionalProperties": true, + "properties": { + "command": { + "type": "string", + "const": "list" + }, + "count": { + "type": "integer", + "minimum": 0 + }, + "artifacts": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "type", "title"], + "additionalProperties": true, + "properties": { + "id": { "type": "string" }, + "type": { "type": "string" }, + "title": { "type": "string" }, + "status": { + "type": "string", + "description": "Status string, or '-' if unset." + }, + "links": { + "type": "integer", + "minimum": 0, + "description": "Number of outgoing links from this artifact." + } + } + } + } + } +} diff --git a/schemas/json/stats-output.schema.json b/schemas/json/stats-output.schema.json new file mode 100644 index 0000000..d368284 --- /dev/null +++ b/schemas/json/stats-output.schema.json @@ -0,0 +1,62 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pulseengine.github.io/rivet/schemas/json/stats-output.schema.json", + "title": "rivet stats --format json", + "description": "Machine-readable output produced by `rivet stats --format json`.", + "type": "object", + "required": [ + "command", + "total", + "types", + "orphans", + "broken_links", + "errors", + "warnings", + "infos" + ], + "additionalProperties": true, + "properties": { + "command": { + "type": "string", + "const": "stats" + }, + "total": { + "type": "integer", + "minimum": 0, + "description": "Total artifacts in the scoped store (post-filter, post-baseline)." + }, + "types": { + "type": "object", + "description": "Per-artifact-type counts; keys are artifact-type names.", + "additionalProperties": { + "type": "integer", + "minimum": 0 + } + }, + "orphans": { + "type": "array", + "description": "Artifact IDs with no incoming or outgoing links.", + "items": { "type": "string" } + }, + "broken_links": { + "type": "integer", + "minimum": 0, + "description": "Count of links whose target does not exist in the store." + }, + "errors": { + "type": "integer", + "minimum": 0, + "description": "Diagnostic count, severity 'error'. Added in v0.4.1." + }, + "warnings": { + "type": "integer", + "minimum": 0, + "description": "Diagnostic count, severity 'warning'. Added in v0.4.1." + }, + "infos": { + "type": "integer", + "minimum": 0, + "description": "Diagnostic count, severity 'info'. Added in v0.4.1." + } + } +} diff --git a/schemas/json/validate-output.schema.json b/schemas/json/validate-output.schema.json new file mode 100644 index 0000000..5986cf8 --- /dev/null +++ b/schemas/json/validate-output.schema.json @@ -0,0 +1,162 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pulseengine.github.io/rivet/schemas/json/validate-output.schema.json", + "title": "rivet validate --format json", + "description": "Machine-readable output produced by `rivet validate --format json`.", + "type": "object", + "required": [ + "command", + "result", + "errors", + "warnings", + "infos", + "diagnostics" + ], + "additionalProperties": true, + "properties": { + "command": { + "type": "string", + "const": "validate" + }, + "result": { + "type": "string", + "enum": ["PASS", "FAIL"], + "description": "Overall result. FAIL iff there is at least one error or broken cross-ref." + }, + "errors": { + "type": "integer", + "minimum": 0, + "description": "Count of diagnostics with severity 'error'." + }, + "warnings": { + "type": "integer", + "minimum": 0 + }, + "infos": { + "type": "integer", + "minimum": 0 + }, + "cross_repo_broken": { + "type": "integer", + "minimum": 0 + }, + "backlinks": { + "type": "integer", + "minimum": 0 + }, + "circular_deps": { + "type": "integer", + "minimum": 0 + }, + "version_conflicts": { + "type": "integer", + "minimum": 0 + }, + "lifecycle_gaps": { + "type": "integer", + "minimum": 0 + }, + "diagnostics": { + "type": "array", + "items": { + "type": "object", + "required": ["severity", "message"], + "additionalProperties": true, + "properties": { + "severity": { + "type": "string", + "enum": ["error", "warning", "info"] + }, + "artifact_id": { + "type": ["string", "null"] + }, + "message": { + "type": "string" + } + } + } + }, + "broken_cross_refs": { + "type": "array", + "items": { + "type": "object", + "required": ["reference", "reason"], + "properties": { + "reference": { "type": "string" }, + "reason": { "type": "string" } + } + } + }, + "cross_repo_backlinks": { + "type": "array", + "items": { + "type": "object", + "required": ["source_prefix", "source_id", "target"], + "properties": { + "source_prefix": { "type": "string" }, + "source_id": { "type": "string" }, + "target": { "type": "string" } + } + } + }, + "circular_dependencies": { + "type": "array", + "items": { + "type": "object", + "required": ["chain"], + "properties": { + "chain": { + "type": "array", + "items": { "type": "string" } + } + } + } + }, + "version_conflict_details": { + "type": "array", + "items": { + "type": "object", + "required": ["repo_identifier", "versions"], + "properties": { + "repo_identifier": { "type": "string" }, + "versions": { + "type": "array", + "items": { + "type": "object", + "required": ["declared_by", "version"], + "properties": { + "declared_by": { "type": "string" }, + "version": { "type": "string" } + } + } + } + } + } + }, + "lifecycle_coverage": { + "type": "array", + "items": { + "type": "object", + "required": ["artifact_id", "artifact_type", "missing"], + "properties": { + "artifact_id": { "type": "string" }, + "artifact_type": { "type": "string" }, + "status": { "type": ["string", "null"] }, + "missing": { + "type": "array", + "items": { "type": "string" } + } + } + } + }, + "variant": { + "type": "object", + "required": ["name", "bound_artifacts", "resolved_artifacts"], + "properties": { + "name": { "type": "string" }, + "bound_artifacts": { "type": "integer", "minimum": 0 }, + "resolved_artifacts": { "type": "integer", "minimum": 0 } + } + } + } +}