diff --git a/docs-site/public/llms-full.txt b/docs-site/public/llms-full.txt index 4831515..8adc1ea 100644 --- a/docs-site/public/llms-full.txt +++ b/docs-site/public/llms-full.txt @@ -3,7 +3,7 @@ > AI rules audit engine — score, detect drift, and improve rules in any AI-assisted codebase. > Source: https://lambda-curry.github.io/anvil/ -> Generated: 2026-07-23 +> Generated: 2026-07-24 ## Installation @@ -876,10 +876,13 @@ anvil audit --target ./my-repo [options] |---|---| | `--target ` | Path to the repo to audit (required) | | `--output ` | Save report to a file | +| `--artifacts-dir ` | Write drift and bootstrap artifacts to this directory | +| `--json` | Output JSON instead of a markdown report | | `--ci` | Deterministic local-only structural lint mode (skips AI synthesis) | | `--ai-provider ` | AI provider: `auto` \| `openai` \| `codex-cli` \| `claude-code` \| `gemini-cli` \| `opencode` \| `heuristic` | | hidden alias: `--no-ai` | Deprecated compatibility alias for `--ci` | | `--ai-model ` | Model to use for synthesis (e.g., `gpt-4o`) | +| `--ai-timeout-ms ` | Set the AI synthesis timeout in milliseconds | Relative `--target` paths resolve from your current shell cwd. @@ -973,12 +976,13 @@ Rule provenance tracked. External rules reviewed before adoption. The guardrail readiness score covers: -- CI discipline -- Type safety -- Test depth -- Security -- Drift resilience -- Hook coverage +- **CI discipline** — pull-request workflows run the core lint, test, build, and typecheck gates. +- **Type safety** — TypeScript strictness and typechecking are configured and enforced. +- **Test depth** — tests cover real scenarios and run as part of the delivery path. +- **Code quality** — linting and formatting policies are configured and enforced. +- **Review ownership** — pull-request review and ownership rules protect critical paths. +- **Security** — secret, dependency, permission, and AI-specific risks have explicit guardrails. +- **Drift resilience** — automation keeps rules and documentation aligned with the codebase. Each dimension contributes to the 0–35 total. Hard gates can enforce minimum scores per dimension via `.anvil/config.yml`. diff --git a/docs-site/src/content/docs/reference/cli.md b/docs-site/src/content/docs/reference/cli.md index 198e2df..069fd28 100644 --- a/docs-site/src/content/docs/reference/cli.md +++ b/docs-site/src/content/docs/reference/cli.md @@ -22,10 +22,13 @@ anvil audit --target ./my-repo [options] |---|---| | `--target ` | Path to the repo to audit (required) | | `--output ` | Save report to a file | +| `--artifacts-dir ` | Write drift and bootstrap artifacts to this directory | +| `--json` | Output JSON instead of a markdown report | | `--ci` | Deterministic local-only structural lint mode (skips AI synthesis) | | `--ai-provider ` | AI provider: `auto` \| `openai` \| `codex-cli` \| `claude-code` \| `gemini-cli` \| `opencode` \| `heuristic` | | hidden alias: `--no-ai` | Deprecated compatibility alias for `--ci` | | `--ai-model ` | Model to use for synthesis (e.g., `gpt-4o`) | +| `--ai-timeout-ms ` | Set the AI synthesis timeout in milliseconds | Relative `--target` paths resolve from your current shell cwd. diff --git a/docs-site/src/content/docs/reference/rubric.md b/docs-site/src/content/docs/reference/rubric.md index bf850cf..b5004b2 100644 --- a/docs-site/src/content/docs/reference/rubric.md +++ b/docs-site/src/content/docs/reference/rubric.md @@ -42,12 +42,13 @@ Rule provenance tracked. External rules reviewed before adoption. The guardrail readiness score covers: -- CI discipline -- Type safety -- Test depth -- Security -- Drift resilience -- Hook coverage +- **CI discipline** — pull-request workflows run the core lint, test, build, and typecheck gates. +- **Type safety** — TypeScript strictness and typechecking are configured and enforced. +- **Test depth** — tests cover real scenarios and run as part of the delivery path. +- **Code quality** — linting and formatting policies are configured and enforced. +- **Review ownership** — pull-request review and ownership rules protect critical paths. +- **Security** — secret, dependency, permission, and AI-specific risks have explicit guardrails. +- **Drift resilience** — automation keeps rules and documentation aligned with the codebase. Each dimension contributes to the 0–35 total. Hard gates can enforce minimum scores per dimension via `.anvil/config.yml`. diff --git a/scripts/audit-cli-mode.test.ts b/scripts/audit-cli-mode.test.ts index c737dd6..aba9631 100644 --- a/scripts/audit-cli-mode.test.ts +++ b/scripts/audit-cli-mode.test.ts @@ -3,6 +3,7 @@ import { cpSync, mkdirSync, mkdtempSync, + readdirSync, readFileSync, rmSync, writeFileSync, @@ -46,7 +47,7 @@ function isExitSignal(value: unknown): value is ExitSignal { ); } -async function runAuditCli(args: string[]) { +async function runAuditCli(args: string[], cwd = repoRoot) { const stdout: string[] = []; const stderr: string[] = []; const originalArgv = [...process.argv]; @@ -75,7 +76,7 @@ async function runAuditCli(args: string[]) { applyNoProviderEnv(); process.argv = ["bun", "audit", ...args]; - process.chdir(repoRoot); + process.chdir(cwd); try { await main(); @@ -122,15 +123,39 @@ test("parseArgs supports --ci and tracks deprecated --no-ai alias", () => { expect(aliasArgs.noAiAliasUsed).toBe(true); }); -test("default audit path fails with guidance when no AI provider is available", async () => { - const run = await runAuditCli(["--target", fixture, "--skip-bootstrap"]); +test("default audit path preserves a structural report when no AI provider is available", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "anvil-full-fallback-")); - expect(run.exitCode).toBe(1); - const combined = `${run.stdout}\n${run.stderr}`; - expect(combined).toContain( - "AI synthesis is required for the default `anvil audit` path.", - ); - expect(combined).toContain("--ci"); + try { + const run = await runAuditCli( + ["--target", resolve(repoRoot, fixture), "--skip-bootstrap"], + tempDir, + ); + + expect(run.exitCode).toBe(0); + const combined = `${run.stdout}\n${run.stderr}`; + expect(combined).toContain( + "AI synthesis unavailable — showing structural results only.", + ); + expect(combined).toContain("Audit report written:"); + + const auditDir = join(tempDir, "docs", "audits"); + const reportName = readdirSync(auditDir).find((name) => + name.endsWith(".md"), + ); + expect(reportName).toBeDefined(); + + const report = readFileSync(join(auditDir, reportName!), "utf8"); + expect(report).toContain( + "AI synthesis unavailable — showing structural results only.", + ); + expect(report).toContain("Rule Quality Score"); + expect(report).toContain("Guardrail Readiness Score"); + expect(report).toContain("Drift"); + expect(report).toContain("Coverage"); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } }); test("--ci succeeds without any AI provider and reports structural lint mode", async () => { diff --git a/scripts/audit.ts b/scripts/audit.ts index 5989121..235fc81 100644 --- a/scripts/audit.ts +++ b/scripts/audit.ts @@ -741,20 +741,21 @@ function emitNoAiAliasDeprecationWarning(): void { ); } -function buildAiRequiredMessage(args: ParsedArgs): string { +const AI_FALLBACK_SUMMARY = + "AI synthesis unavailable — showing structural results only."; + +function buildAiFallbackMessage(args: ParsedArgs): string { const providerLine = args.aiProvider && args.aiProvider !== "auto" ? `Requested provider \`${args.aiProvider}\` did not produce a synthesis result.` : "No working AI provider was detected for the default audit path."; return [ - "AI synthesis is required for the default `anvil audit` path.", + AI_FALLBACK_SUMMARY, providerLine, "", - "Next steps:", - "- Install/login to Claude Code, Codex CLI, Gemini CLI, or opencode; or set `OPENAI_API_KEY`.", - "- Re-run the same audit command for the full product output.", - "- If you want deterministic local-only structural lint instead, re-run with `--ci`.", + "The local audit completed and its report will still be written.", + "Re-run with a working provider for AI-prioritized suggestions.", ].join("\n"); } @@ -4609,6 +4610,14 @@ export function buildReport( lines.push(`*Generated by: Anvil audit.ts*`); lines.push(""); + if (result.auditMode === "full" && result.aiSynthesis.mode !== "ai") { + lines.push(`> **${AI_FALLBACK_SUMMARY}**`); + lines.push( + "> The local scores, drift, coverage, guardrail, and remediation evidence below are complete. Re-run with a working provider for AI-prioritized suggestions.", + ); + lines.push(""); + } + lines.push("## Summary"); lines.push(""); @@ -5938,7 +5947,7 @@ export async function runAudit(args: ParsedArgs): Promise { }); if (auditMode === "full" && aiSynthesis.mode !== "ai") { - throw new Error(buildAiRequiredMessage(args)); + logProgress(`⚠️ ${buildAiFallbackMessage(args)}`); } logProgress( diff --git a/scripts/mine-pr-cli.test.ts b/scripts/mine-pr-cli.test.ts new file mode 100644 index 0000000..50eed2a --- /dev/null +++ b/scripts/mine-pr-cli.test.ts @@ -0,0 +1,11 @@ +import { expect, test } from "bun:test"; + +import { assertGitHubCliAvailable } from "./mine-pr-rules.ts"; + +test("missing GitHub CLI produces actionable setup guidance", () => { + expect(() => + assertGitHubCliAvailable("/definitely/missing-anvil-gh"), + ).toThrow( + "GitHub CLI (gh) was not found or could not run. Install it from https://cli.github.com and run `gh auth login`.", + ); +}); diff --git a/scripts/mine-pr-rules.ts b/scripts/mine-pr-rules.ts index b157260..ec921a6 100644 --- a/scripts/mine-pr-rules.ts +++ b/scripts/mine-pr-rules.ts @@ -1,6 +1,8 @@ #!/usr/bin/env bun /* eslint-disable no-console */ +import { spawnSync } from "node:child_process"; + export type CliArgs = { repo: string; limit: number; @@ -267,6 +269,19 @@ export function printUsageAndExit(code: number): never { process.exit(code); } +export function assertGitHubCliAvailable(ghCommand = "gh"): void { + const check = spawnSync(ghCommand, ["--version"], { + encoding: "utf8", + stdio: "pipe", + }); + + if (check.error || check.status !== 0) { + throw new Error( + "GitHub CLI (gh) was not found or could not run. Install it from https://cli.github.com and run `gh auth login`.", + ); + } +} + const QUERY = ` query($owner: String!, $repo: String!, $pageSize: Int!, $cursor: String) { repository(owner: $owner, name: $repo) { @@ -1127,6 +1142,7 @@ export async function main() { const [owner, name] = args.repo.split("/"); try { + assertGitHubCliAvailable(); const { prs, comments } = await fetchMergedPRComments( owner, name, diff --git a/scripts/public-cli-docs-contract.test.ts b/scripts/public-cli-docs-contract.test.ts index 13ab534..a9329be 100644 --- a/scripts/public-cli-docs-contract.test.ts +++ b/scripts/public-cli-docs-contract.test.ts @@ -59,3 +59,43 @@ test("BYOK docs describe provider auto-detection without an opt-in claim", () => expect(doc).toContain("use `--ci` to stay local"); } }); + +test("CLI reference documents every public audit output and timeout flag", () => { + const cliReference = readRepoFile( + "docs-site/src/content/docs/reference/cli.md", + ); + + expect(cliReference).toContain("`--json`"); + expect(cliReference).toContain("`--artifacts-dir `"); + expect(cliReference).toContain("`--ai-timeout-ms `"); +}); + +test("rubric reference matches the seven guardrail dimensions in code", () => { + const rubric = readRepoFile("docs-site/src/content/docs/reference/rubric.md"); + const guardrailSource = readRepoFile("scripts/lib/guardrail-score.ts"); + + for (const dimension of [ + "ciDiscipline", + "typeSafety", + "testDepth", + "codeQuality", + "reviewOwnership", + "security", + "driftResilience", + ]) { + expect(guardrailSource).toContain(`"${dimension}"`); + } + + for (const heading of [ + "CI discipline", + "Type safety", + "Test depth", + "Code quality", + "Review ownership", + "Security", + "Drift resilience", + ]) { + expect(rubric).toContain(`**${heading}**`); + } + expect(rubric).not.toContain("Hook coverage"); +});