From 7775b481d3e5e0951bd783de2d357c0060c5c3ca Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 28 Jul 2026 14:21:04 +0000 Subject: [PATCH] =?UTF-8?q?feat(spec,lint):=20ADR-0109=20revised=20+=20Pha?= =?UTF-8?q?se=201=20=E2=80=94=20skills-only=20default=20path,=20tool-name?= =?UTF-8?q?=20registry,=20advisory=20skill.tools=20lint=20(#3820=20R7)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR-0109 revision: the first draft framed third-party tools as authored binding records; review surfaced the stronger conclusion — the runtime already materialises a tool per declarative action (action_), so the DEFAULT third-party path needs no tool records at all. Skills are the only required authoring artifact; tool records are demoted to an optional AI-presentation refinement layer (Phase 2, gated on acceptance and a real refinement need). AI capability ≡ application capability: same executable, same authz, same audit. Phase 1, implemented here: - spec: PLATFORM_PROVIDED_TOOL_NAMES / PLATFORM_TOOL_FAMILY_PREFIXES / isPlatformProvidedToolName in system/constants/platform-tool-names.ts — the PLATFORM_PROVIDED_OBJECT_NAMES precedent applied to tools. Owning packages live in the cloud repo, so (like CLOUD_PROVIDED_OBJECT_NAMES) the conformance half of the contract lives there; this repo pins the list's internal invariants. - lint: validate-ai-tool-references — the #3820 R7 skill.tools branch, wildcard-aware, resolving against stack.tools ∪ registry ∪ materialised action_ family. Warning severity (ADR-0078 advisory-first): runtime plugins outside the registry are statically invisible. Appended to REFERENCE_INTEGRITY_RULES → validate/lint/compile pick it up. Exactly 10 findings / 0 false positives on the HotCRM corpus; a dedicated near-miss suggestion catches the raw-action-name-instead-of-action_ mistake edit distance cannot. - spec: 'tools' joins composeStacks' CONCAT_ARRAY_FIELDS (declared records survive composition) and the stack.tools / AI-slot docs teach the ADR-0109 model. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BHjroNkLkajskKbJaidko4 --- .changeset/ai-tool-registry-and-lint.md | 34 +++ docs/adr/0109-ai-tool-authoring-model.md | 134 ++++++++++-- packages/lint/src/index.ts | 9 + .../src/reference-integrity-suite.test.ts | 10 +- .../lint/src/reference-integrity-suite.ts | 2 + .../src/validate-ai-tool-references.test.ts | 106 ++++++++++ .../lint/src/validate-ai-tool-references.ts | 194 ++++++++++++++++++ packages/spec/api-surface.json | 4 + packages/spec/src/stack.zod.ts | 16 +- packages/spec/src/system/constants/index.ts | 1 + .../constants/platform-tool-names.test.ts | 57 +++++ .../system/constants/platform-tool-names.ts | 116 +++++++++++ 12 files changed, 655 insertions(+), 28 deletions(-) create mode 100644 .changeset/ai-tool-registry-and-lint.md create mode 100644 packages/lint/src/validate-ai-tool-references.test.ts create mode 100644 packages/lint/src/validate-ai-tool-references.ts create mode 100644 packages/spec/src/system/constants/platform-tool-names.test.ts create mode 100644 packages/spec/src/system/constants/platform-tool-names.ts diff --git a/.changeset/ai-tool-registry-and-lint.md b/.changeset/ai-tool-registry-and-lint.md new file mode 100644 index 0000000000..87e3cd8183 --- /dev/null +++ b/.changeset/ai-tool-registry-and-lint.md @@ -0,0 +1,34 @@ +--- +"@objectstack/spec": minor +"@objectstack/lint": minor +--- + +feat(spec,lint): ADR-0109 Phase 1 — platform tool-name registry + advisory `skill.tools[]` reference lint (#3820 R7) + +ADR-0109 (revised) settles the AI tool authoring model: **the default +third-party path needs no tool records at all.** A skill's `tools[]` names +either a platform-registered tool or a tool the runtime materialises from the +app's own declarative actions (`action_`) — the executable, its authz, +and its audit trail stay on the action/flow the app already ships. Tool +records are demoted to an optional AI-presentation refinement layer (Phase 2, +gated on acceptance). + +Phase 1, shipped here: + +- **`PLATFORM_PROVIDED_TOOL_NAMES`** (`@objectstack/spec/system`) — curated + registry of every statically-named tool the cloud AI runtime registers, + grouped by owning package, plus `PLATFORM_TOOL_FAMILY_PREFIXES` for the + materialised `action_` family and `isPlatformProvidedToolName()`. The + `PLATFORM_PROVIDED_OBJECT_NAMES` precedent, applied to tools; conformance + tests live in the owning cloud packages. +- **`validate-ai-tool-references`** (`@objectstack/lint`) — the #3820 R7 + `skill.tools` branch, wildcard-aware, resolving against declared + `stack.tools` ∪ the registry ∪ the materialised action family. Severity + **warning** (ADR-0078 advisory-first ratchet): the registry cannot see + third-party runtime plugins. Joins `REFERENCE_INTEGRITY_RULES`, so + `validate`, `lint`, and `compile` all pick it up. On the HotCRM corpus it + reports exactly the 10 fictional tool references (0 false positives on the + 6 that resolve). +- **`composeStacks` no longer drops `tools`** — the slot joins the + concatenated array fields, so a declared record survives composition. +- `stack.tools` / AI-slot docs updated to the ADR-0109 model. diff --git a/docs/adr/0109-ai-tool-authoring-model.md b/docs/adr/0109-ai-tool-authoring-model.md index 34d3bc3551..c8881f9151 100644 --- a/docs/adr/0109-ai-tool-authoring-model.md +++ b/docs/adr/0109-ai-tool-authoring-model.md @@ -1,38 +1,134 @@ -# ADR-0109: The AI tool authoring model — third-party tools are bindings to executable primitives +# ADR-0109: The AI tool authoring model — the default third-party path needs no tool records -**Status**: Proposed (2026-07-28) — the "D0" decision of issue #3820; blocks the `skill.tools` branch of the R7 reference-integrity rule. +**Status**: Proposed (2026-07-28; revised same day — see Revision note). Phase 1 (platform tool-name registry + advisory reference lint) is implemented alongside this revision; Phase 2 (the optional refinement layer) awaits acceptance. **Deciders**: ObjectStack Protocol Architects -**Builds on**: [ADR-0063](./0063-two-kernel-agents-skills-are-the-extension-primitive.md) (skills + tools are the third-party extension primitive), [ADR-0064](./0064-tool-scoping-to-agent.md) (an agent's tools are its skills' tools), [ADR-0078](./0078-no-silently-inert-metadata.md) (no silently inert metadata) -**Consumers**: `@objectstack/spec` (`ai/tool.zod.ts`, `stack.zod.ts`), `@objectstack/lint` (the R7 rule), `../cloud/service-ai` (ToolRegistry), `../cloud/service-ai-studio` +**Builds on**: [ADR-0063](./0063-two-kernel-agents-skills-are-the-extension-primitive.md) (skills + tools are the third-party extension primitive), [ADR-0064](./0064-tool-scoping-to-agent.md) (an agent's tools are its skills' tools), [ADR-0078](./0078-no-silently-inert-metadata.md) (no silently inert metadata), [ADR-0049](./0049-no-unenforced-security-properties.md) (enforce-or-remove) +**Consumers**: `@objectstack/spec` (`system/constants/platform-tool-names.ts`, `ai/tool.zod.ts`, `stack.zod.ts`), `@objectstack/lint` (`validate-ai-tool-references`), `../cloud/service-ai` + `service-ai-studio` (registry conformance) + +> **Revision note.** The first draft framed third-party tools as *bindings to +> actions/flows that authors declare as tool records*. Review surfaced the +> stronger conclusion: the runtime **already materialises** a tool per +> declarative action (`action_`, the family `actions_executor` +> subscribes to with `action_*`), so in the default path a third party needs +> **no tool record at all** — the binding the draft proposed already exists +> implicitly for every action. The tool record is thereby demoted from "the +> authoring artifact" to an *optional refinement layer*, and Phase 1 shrinks +> to what today's mechanisms already support. --- ## TL;DR -`stack.tools` today is a **declaration with no executable half and no reader**: `ToolSchema` carries no handler binding, the metadata `tool` type is written once at boot (Studio visibility mirror, `service-ai-studio/plugin.ts`) and read by **nothing**, `'tools'` is missing from `composeStacks`' `CONCAT_ARRAY_FIELDS`, and the executable tool set lives exclusively in the runtime-registered in-process `ToolRegistry`. ADR-0063 §2 says third parties extend via "skills **and tools**" — but only the skills half is real. That is the ADR-0078 prohibited state, and it makes any reference-integrity rule over `skill.tools[]` unbuildable: there is no registry of facts to resolve against (16/16 of HotCRM's skill→tool references miss `stack.tools`; 6 of those 16 nonetheless resolve — against the runtime registry lint cannot see). +**Third parties extend AI by authoring skills — and only skills.** A skill's +`tools[]` names either a platform-registered tool (curated in +`PLATFORM_PROVIDED_TOOL_NAMES`) or a tool the runtime materialises from the +app's own declarative actions (`action_`). The executable, its authz, +and its audit trail are the action/flow the app already ships for its UI — +the AI can do exactly what the application can already do, under the same +permissions, and nothing else. + +`stack.tools` records are an **optional AI-presentation refinement layer** +(Phase 2): needed only when the AI-facing surface must differ from the raw +executable. They are never required, and never a place business logic lives. -Decision: **a third-party tool is a *binding* to an executable primitive the platform already has — an action or a flow — never a free-floating executable of its own.** Kernel/plugin tools stay runtime-registered, and their names become facts via a curated, conformance-tested registry in spec. +## Context + +`stack.tools` before this ADR was the ADR-0078 prohibited state: a +declaration with no executable half and no reader. `ToolSchema` carries no +handler binding; the metadata `tool` type is written once at boot (Studio +visibility mirror) and read by nothing; `'tools'` was missing from +`composeStacks`' concat list; the executable tool set lives exclusively in +the runtime-registered in-process `ToolRegistry`. Meanwhile ADR-0063 §2 +promised "skills **and tools**" as the third-party primitive — only the +skills half was real. On the HotCRM corpus, 16/16 `skill.tools` references +missed `stack.tools` while 6 of them resolved against the runtime registry +lint could not see: no reference rule could be correct in either direction +(#3820 D0/D2). ## Decision -1. **Third-party tools bind, they don't implement.** `ToolSchema` gains a required-for-app-packages `binding` (`{ type: 'action' | 'flow', name: string }`). The runtime materialises a callable from the bound primitive (the same mechanism that already materialises `action_` tools from object action lists); `parameters` may narrow, but never widen, what the bound primitive accepts. Handlers, authz, and audit stay where they already are — on the action/flow. A tool record without a `binding` is legal only for runtime-registered (kernel/plugin) tools, whose record is a visibility mirror, not a definition. -2. **Platform tool names become facts, not guesses.** Spec exports a curated `PLATFORM_PROVIDED_TOOL_NAMES: ReadonlySet` (the `PLATFORM_PROVIDED_OBJECT_NAMES` / #3657 precedent), kept honest by conformance tests in the *owning* packages (service-ai / service-ai-studio assert "every tool I register is listed; every listed name with my prefix is registered"). The lint → spec dependency direction is preserved. -3. **Reference integrity becomes decidable, and R7's tool branch unblocks.** `skill.tools[]` (wildcard-aware) resolves against: bound tools declared in-stack (→ checkable through to the action/flow, which existing rules already resolve) ∪ `PLATFORM_PROVIDED_TOOL_NAMES` ∪ wildcard families. Resolution ladder per ADR-0072: unknown name → **warning** at first (ADR-0078 ratchet), **error** once the registry has soaked one release. -4. **The inert surface is closed.** `'tools'` joins `CONCAT_ARRAY_FIELDS`; the boot-time mirror writes gain provenance so a stack-declared record cannot shadow a platform tool's mirror (today `if (exists) skip` lets it); the runtime's "silently dropped" posture for unresolved references stays (graceful degradation) but logs, because the *authoring-time* signal now comes from lint — loud at the producer, tolerant at the consumer (Prime Directive #12). +1. **The default path is skills-only.** A third party gives the AI a new + capability by (a) declaring the executable as a normal action/flow — which + the app needs for its UI anyway — and (b) authoring a skill whose + `tools[]` names the materialised `action_` tool or a platform tool. + Zero tool records; permissions and audit are inherited from the executable + by construction. This is also one less namespace an AI author can + hallucinate into. + +2. **Platform tool names are facts, not guesses.** + `PLATFORM_PROVIDED_TOOL_NAMES` in `@objectstack/spec` + (`system/constants/platform-tool-names.ts`, the + `PLATFORM_PROVIDED_OBJECT_NAMES` precedent) curates every statically-named + tool the cloud AI runtime registers, grouped by owning package; + `PLATFORM_TOOL_FAMILY_PREFIXES` names the dynamically-materialised + families (today exactly `action_`). The owning cloud packages carry + conformance tests — the same split as `CLOUD_PROVIDED_OBJECT_NAMES`, + whose conformance half also lives outside this repo. + +3. **`skill.tools[]` reference integrity is decidable and linted** — + `validate-ai-tool-references` in `@objectstack/lint`, a member of + `REFERENCE_INTEGRITY_RULES`. Resolution universe: `stack.tools` names ∪ + the platform registry ∪ the materialised `action_` family (from + `stack.actions` and every object's `actions`), wildcard-aware. Severity + **warning** first (ADR-0078 ratchet): a runtime plugin outside the + registry remains statically invisible, and the runtime deliberately + tolerates unresolved names. On the HotCRM corpus the rule yields exactly + the 10 fictional references and 0 false positives. + +4. **Tool records become the optional refinement layer (Phase 2).** A tool + record earns its place only when the AI-facing surface must differ from + the raw executable: LLM-directed description, parameter narrowing (expose + 3 of 20 params, tighten enums), exposing a *flow* (no materialised family + yet), a stable AI-facing name decoupled from the action's, or execution + policy (a confirm-before-run flag — note `requiresConfirmation` was + *removed* in #3715/#3876 as unenforced per ADR-0049; it returns only + together with its enforcement, as part of this layer). When Phase 2 + lands, a third-party tool record MUST carry a `binding` + (`{ type: 'action' | 'flow', name }`) to the executable it refines — + handlers never live on the tool. Until Phase 2 lands, third-party tool + records remain inert-and-documented (the `stack.tools` doc says so), and + the refinement needs above are the acceptance trigger. + +5. **The inert-surface holes close.** `'tools'` joins `composeStacks`' + concat list (this revision — a declared record must at least survive + composition). Phase 2 adds provenance to the boot-time metadata mirror so + a stack-declared record cannot shadow a platform tool's entry (today + `if (exists) skip` lets it), and revisits the `tool` metadata-type flags + (`allowRuntimeCreate`/`allowOrgOverride`) which currently advertise an + authoring surface with no reader. -## Rejected alternative +## Rejected alternatives -**Tools as first-class executables** (a `handler` on ToolSchema): re-invents actions/flows inside the AI layer, splits authz/audit across two systems, and turns every tool into a second place to define business logic. The platform's ontology is already executable; the AI layer needs a *view* of it, not a rival. +- **Tools as first-class executables** (a `handler` on ToolSchema): + re-invents actions/flows inside the AI layer, splits authz/audit across two + systems, and turns every tool into a second place to define business logic. + The platform's ontology is already executable; the AI layer needs a *view* + of it, not a rival. +- **A required tool record per exposed action** (the first draft's implicit + shape): the materialised `action_` family already provides the + binding implicitly, so a mandatory record would add a second authoring + step, a second namespace to keep consistent, and a second surface for AI + authors to hallucinate into — for zero added capability. ## Consequences -- "Skills + tools" (ADR-0063 §2) becomes a shipped contract: a third party can author a skill whose tools are real, checkable, and governed by the same authz as the UI button that runs the same action. -- AI-authored metadata gets a closed vocabulary: a hallucinated tool name fails lint at draft time instead of shipping as a dead capability the copilot claims to have (the HotCRM failure, 10 fictional tools across 6 skills). -- Costs: a spec change (`binding`), the registry + conformance tests, a materialisation path in the runtime; the R7 tool-branch rule stays blocked until 1–2 land. +- "Third-party AI extension = write a skill" becomes the whole story — one + concept to document, teach, and generate. ADR-0063 §2's "skills and tools" + reads as: skills are the primitive, tools are its optional refinement. +- AI capability ≡ application capability: same executables, same + permissions, same audit. There is no second security system to review and + no way for the AI to do what the app cannot. +- A hallucinated tool name fails lint at draft time (advisory now, gating + after the ratchet) instead of shipping as a copilot that claims abilities + it does not have — the HotCRM failure, 10 fictional tools across 6 skills. +- Cost honestly stated: the registry is a curated list that can go stale; + the conformance tests in the owning packages are the mechanism that keeps + it true, and they live in a different repository than the list. ## Follow-up -- [ ] `binding` on `ToolSchema` + `'tools'` into `CONCAT_ARRAY_FIELDS` (spec). -- [ ] `PLATFORM_PROVIDED_TOOL_NAMES` + owning-package conformance tests (spec, cloud). -- [ ] Materialise bound tools in the ToolRegistry; provenance-guard the boot mirror (cloud). -- [ ] R7 tool branch: `validate-ai-references` skill→tool resolution, warning-first (lint; #3820). +- [x] `PLATFORM_PROVIDED_TOOL_NAMES` + `PLATFORM_TOOL_FAMILY_PREFIXES` + invariant tests (spec — this change). +- [x] `validate-ai-tool-references`, advisory, in `REFERENCE_INTEGRITY_RULES` (lint — this change). +- [x] `'tools'` into `composeStacks`' concat list (spec — this change). +- [ ] Registry conformance tests in `../cloud` `service-ai` / `service-ai-studio` (cloud PR; activates fully when the cloud `.objectstack-sha` pin advances past this change). +- [ ] **Phase 2 (on acceptance + a real refinement need):** `binding` on `ToolSchema`; flow exposure; boot-mirror provenance guard; revisit `tool` metadata-type flags; ratchet the lint rule to error per ADR-0078 once the registry has soaked a release. diff --git a/packages/lint/src/index.ts b/packages/lint/src/index.ts index 3f9734f675..ef745dfc02 100644 --- a/packages/lint/src/index.ts +++ b/packages/lint/src/index.ts @@ -231,6 +231,15 @@ export type { AiSurfaceAffinitySeverity, } from './validate-ai-surface-affinity.js'; +export { + validateAiToolReferences, + AI_SKILL_TOOL_UNRESOLVED, +} from './validate-ai-tool-references.js'; +export type { + AiToolRefFinding, + AiToolRefSeverity, +} from './validate-ai-tool-references.js'; + // One entry point for the reference-resolution rules above (#3583 §5 D5). // Adding a rule to `REFERENCE_INTEGRITY_RULES` runs it on `validate`, `lint` // and `compile` at once — the CLI call sites do not change. diff --git a/packages/lint/src/reference-integrity-suite.test.ts b/packages/lint/src/reference-integrity-suite.test.ts index 9828338890..e4d0883b42 100644 --- a/packages/lint/src/reference-integrity-suite.test.ts +++ b/packages/lint/src/reference-integrity-suite.test.ts @@ -23,6 +23,7 @@ describe('reference-integrity suite — membership', () => { 'validateTranslationReferences', 'validateFlowTemplatePaths', 'validateAiSurfaceAffinity', + 'validateAiToolReferences', ]); }); @@ -111,7 +112,9 @@ describe('reference-integrity suite — every member actually runs', () => { // validateAiSurfaceAffinity: an 'ask' agent binding a 'build' skill — the // runtime throws on this at chat time (ADR-0064 §3). agents: [{ name: 'helper', surface: 'ask', skills: ['metadata_authoring'] }], - skills: [{ name: 'metadata_authoring', surface: 'build', tools: [] }], + // validateAiToolReferences: a tool name nothing declares, registers, or + // materialises (the HotCRM fictional-tool class). + skills: [{ name: 'metadata_authoring', surface: 'build', tools: ['forecast_revenue'] }], flows: [ { name: 'lead_followup', @@ -145,6 +148,7 @@ describe('reference-integrity suite — every member actually runs', () => { expect(rules).toContain('translation-target-unknown'); expect(rules).toContain('flow-template-unknown-field'); expect(rules).toContain('ai-skill-surface-mismatch'); + expect(rules).toContain('ai-skill-tool-unresolved'); }); it('carries a gating flow-template finding through the suite (#3810)', () => { @@ -166,9 +170,9 @@ describe('reference-integrity suite — every member actually runs', () => { expect(typeof f.message).toBe('string'); expect(typeof f.hint).toBe('string'); } - // Object references run first, AI surface affinity last. + // Object references run first, AI tool references last. expect(findings[0].rule).toBe('object-reference-unknown'); - expect(findings[findings.length - 1].rule).toBe('ai-skill-surface-mismatch'); + expect(findings[findings.length - 1].rule).toBe('ai-skill-tool-unresolved'); }); it('returns nothing for an empty stack', () => { diff --git a/packages/lint/src/reference-integrity-suite.ts b/packages/lint/src/reference-integrity-suite.ts index 396a374071..ce435fad43 100644 --- a/packages/lint/src/reference-integrity-suite.ts +++ b/packages/lint/src/reference-integrity-suite.ts @@ -53,6 +53,7 @@ import { validateNavAccess } from './validate-nav-access.js'; import { validateTranslationReferences } from './validate-translation-references.js'; import { validateFlowTemplatePaths } from './validate-flow-template-paths.js'; import { validateAiSurfaceAffinity } from './validate-ai-surface-affinity.js'; +import { validateAiToolReferences } from './validate-ai-tool-references.js'; export type ReferenceIntegritySeverity = 'error' | 'warning'; @@ -96,6 +97,7 @@ export const REFERENCE_INTEGRITY_RULES: readonly ReferenceIntegrityRule[] = [ { name: 'validateTranslationReferences', run: validateTranslationReferences }, { name: 'validateFlowTemplatePaths', run: validateFlowTemplatePaths }, { name: 'validateAiSurfaceAffinity', run: validateAiSurfaceAffinity }, + { name: 'validateAiToolReferences', run: validateAiToolReferences }, ]; /** diff --git a/packages/lint/src/validate-ai-tool-references.test.ts b/packages/lint/src/validate-ai-tool-references.test.ts new file mode 100644 index 0000000000..afecf15c42 --- /dev/null +++ b/packages/lint/src/validate-ai-tool-references.test.ts @@ -0,0 +1,106 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { describe, it, expect } from 'vitest'; +import { + validateAiToolReferences, + AI_SKILL_TOOL_UNRESOLVED, +} from './validate-ai-tool-references.js'; + +describe('validate-ai-tool-references', () => { + // ── The motivating corpus, measured per branch (#3806 / #3820 §3) ───────── + // HotCRM's six skills carry 16 tool references: 6 resolve via the platform + // registry, 10 are fictional. The rule must find exactly the 10 — a single + // false positive on the resolvable 6 is the failure mode that got this + // branch blocked on D0 in the first place. + it('yields exactly the 10 fictional references on the HotCRM corpus, 0 false positives', () => { + const stack = { + skills: [ + { name: 'live_data', tools: ['describe_object', 'list_objects', 'query_records', 'get_record', 'aggregate_data'] }, + { name: 'customer_360', tools: ['search_knowledge'] }, + { name: 'case_triage', tools: ['triage_case'] }, + { name: 'email_drafting', tools: ['generate_email_copy', 'optimize_subject_line', 'personalize_content', 'generate_email'] }, + { name: 'lead_qualification', tools: ['analyze_lead', 'suggest_next_action'] }, + { name: 'revenue_forecasting', tools: ['analyze_pipeline', 'identify_at_risk', 'forecast_revenue'] }, + ], + }; + const findings = validateAiToolReferences(stack); + expect(findings).toHaveLength(10); + expect(findings.every((f) => f.severity === 'warning')).toBe(true); + expect(findings.every((f) => f.rule === AI_SKILL_TOOL_UNRESOLVED)).toBe(true); + const flagged = findings.map((f) => f.message.match(/references tool "([a-z_]+)"/)?.[1]); + expect(flagged).toEqual([ + 'triage_case', + 'generate_email_copy', + 'optimize_subject_line', + 'personalize_content', + 'generate_email', + 'analyze_lead', + 'suggest_next_action', + 'analyze_pipeline', + 'identify_at_risk', + 'forecast_revenue', + ]); + }); + + it('resolves declared stack.tools records (the optional refinement layer)', () => { + const stack = { + tools: [{ name: 'quote_discount', label: 'Quote', description: 'x' }], + skills: [{ name: 's', tools: ['quote_discount'] }], + }; + expect(validateAiToolReferences(stack)).toEqual([]); + }); + + it('resolves materialised action tools from stack-level and object-level actions', () => { + const stack = { + actions: [{ name: 'send_invoice', label: 'Send' }], + objects: [{ name: 'crm_case', actions: [{ name: 'triage_case', label: 'Triage' }] }], + skills: [{ name: 's', tools: ['action_send_invoice', 'action_triage_case'] }], + }; + expect(validateAiToolReferences(stack)).toEqual([]); + }); + + it('does NOT resolve a bare action name — the tool is the materialised action_', () => { + // The ADR-0109 default path is `action_`; naming the raw action is + // the near-miss the hint should catch, via the suggestion. + const stack = { + objects: [{ name: 'crm_case', actions: [{ name: 'triage_case', label: 'Triage' }] }], + skills: [{ name: 's', tools: ['triage_case'] }], + }; + const findings = validateAiToolReferences(stack); + expect(findings).toHaveLength(1); + expect(findings[0].message).toContain('Did you mean "action_triage_case"?'); + }); + + it('resolves trailing-wildcard families against the universe', () => { + const withActions = { + objects: [{ name: 'crm_case', actions: [{ name: 'triage_case', label: 'T' }] }], + skills: [{ name: 's', tools: ['action_*'] }], + }; + expect(validateAiToolReferences(withActions)).toEqual([]); + + // A family subscription that matches nothing contributes zero tools. + const withoutActions = { skills: [{ name: 's', tools: ['crm_*'] }] }; + const findings = validateAiToolReferences(withoutActions); + expect(findings).toHaveLength(1); + expect(findings[0].message).toContain('tool family "crm_*"'); + }); + + it('resolves platform registry names without any stack declarations', () => { + const stack = { skills: [{ name: 's', tools: ['query_records', 'verify_build', 'suggest_builder'] }] }; + expect(validateAiToolReferences(stack)).toEqual([]); + }); + + it('reports stable paths and tolerates junk shapes', () => { + const stack = { + skills: [{ name: 's', tools: ['query_records', 'nope_tool'] }], + }; + const findings = validateAiToolReferences(stack); + expect(findings.map((f) => f.path)).toEqual(['skills[0].tools[1]']); + + expect(validateAiToolReferences({})).toEqual([]); + expect(validateAiToolReferences({ skills: 'nope', tools: 42 } as never)).toEqual([]); + expect( + validateAiToolReferences({ skills: [null, { tools: [7, null, 'query_records'] }] } as never), + ).toEqual([]); + }); +}); diff --git a/packages/lint/src/validate-ai-tool-references.ts b/packages/lint/src/validate-ai-tool-references.ts new file mode 100644 index 0000000000..931ab57dd5 --- /dev/null +++ b/packages/lint/src/validate-ai-tool-references.ts @@ -0,0 +1,194 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [ADR-0109 / issue #3820 R7] `skill.tools[]` reference integrity — the tool + * branch of the AI reference rules. + * + * Under ADR-0109's authoring model the DEFAULT third-party path declares no + * tool records at all: a skill names either a platform-registered tool or an + * auto-materialised action tool. A `skill.tools[]` entry therefore resolves + * against, in order: + * + * 1. the stack's own `stack.tools[]` names (the optional refinement layer); + * 2. `PLATFORM_PROVIDED_TOOL_NAMES` — the curated registry of tools the + * cloud AI runtime registers at boot (spec owns the list; the owning + * cloud packages carry the conformance tests); + * 3. the materialised `action_` family — one tool per declarative + * action (`stack.actions` ∪ every object's `actions`), the mechanism the + * built-in `actions_executor` subscribes to with `action_*`. + * + * Trailing-wildcard entries (`action_*`, `foo_*`) resolve when ANY member of + * that universe matches the prefix. + * + * Severity is **warning** (ADR-0078 advisory-first ratchet), not error, + * because the universe has a known blind spot: a runtime plugin outside the + * registry can legitimately register tools no static analysis can see, and + * the runtime deliberately tolerates unresolved names (skills may be authored + * before their tools exist — `skill-registry.ts`). What the warning buys: the + * HotCRM failure — 10 fictional tools across 6 skills, every one shipping + * through `validate`/`lint` clean and surfacing as a copilot that claims + * abilities it does not have — now surfaces at authoring time. On that same + * corpus the resolution ladder above yields exactly 10 findings and 0 false + * positives (6 references resolve via the registry). + */ + +import { PLATFORM_PROVIDED_TOOL_NAMES, PLATFORM_TOOL_FAMILY_PREFIXES } from '@objectstack/spec/system'; + +export const AI_SKILL_TOOL_UNRESOLVED = 'ai-skill-tool-unresolved'; + +export type AiToolRefSeverity = 'error' | 'warning'; + +export interface AiToolRefFinding { + /** Always `warning` — see the header for why this rule starts advisory. */ + severity: AiToolRefSeverity; + /** Diagnostic rule id. */ + rule: string; + /** Human-readable location, e.g. `skill "revenue_forecasting" · tools`. */ + where: string; + /** Config path, e.g. `skills[3].tools[1]`. */ + path: string; + /** What is wrong. */ + message: string; + /** How to fix it. */ + hint: string; +} + +type AnyRec = Record; + +function asArray(v: unknown): AnyRec[] { + if (Array.isArray(v)) return v.filter((x): x is AnyRec => !!x && typeof x === 'object'); + if (v && typeof v === 'object') { + return Object.entries(v as AnyRec).map(([name, def]) => ({ name, ...(def as AnyRec) })); + } + return []; +} + +function strName(v: unknown): string | undefined { + return typeof v === 'string' && v.length > 0 ? v : undefined; +} + +function distance(a: string, b: string): number { + const m = a.length; + const n = b.length; + if (m === 0) return n; + if (n === 0) return m; + let prev = Array.from({ length: n + 1 }, (_, j) => j); + for (let i = 1; i <= m; i++) { + const curr = [i, ...new Array(n).fill(0)]; + for (let j = 1; j <= n; j++) { + const cost = a[i - 1] === b[j - 1] ? 0 : 1; + curr[j] = Math.min(curr[j - 1] + 1, prev[j] + 1, prev[j - 1] + cost); + } + prev = curr; + } + return prev[n]; +} + +function suggest(target: string, known: Set): string { + // The high-frequency near-miss first: naming the raw ACTION where the + // materialised TOOL (`action_`) is meant. Edit distance cannot catch + // it (the prefix alone is 7 edits), and it is exactly the mistake the + // ADR-0109 default path invites from authors who know their action names. + for (const prefix of PLATFORM_TOOL_FAMILY_PREFIXES) { + if (known.has(`${prefix}${target}`)) return ` Did you mean "${prefix}${target}"?`; + } + + let best: string | undefined; + let bestScore = Infinity; + for (const candidate of known) { + const d = distance(target, candidate); + if (d < bestScore) { + bestScore = d; + best = candidate; + } + } + const limit = Math.max(2, Math.floor(target.length / 3)); + return best && bestScore <= limit ? ` Did you mean "${best}"?` : ''; +} + +/** + * The full set of tool names resolvable from this stack: declared tool + * records ∪ the platform registry ∪ the materialised action family. + */ +function collectToolUniverse(stack: AnyRec): Set { + const universe = new Set(PLATFORM_PROVIDED_TOOL_NAMES); + + for (const tool of asArray(stack.tools)) { + const n = strName(tool.name); + if (n) universe.add(n); + } + + const addActionFamily = (actions: unknown) => { + for (const action of asArray(actions)) { + const n = strName(action.name); + if (n) universe.add(`action_${n}`); + } + }; + addActionFamily(stack.actions); + for (const obj of asArray(stack.objects)) { + addActionFamily(obj.actions); + } + + return universe; +} + +/** + * Validate every `skill.tools[]` reference in a stack. Returns findings + * (empty = clean). + */ +export function validateAiToolReferences(stack: AnyRec): AiToolRefFinding[] { + const findings: AiToolRefFinding[] = []; + if (!stack || typeof stack !== 'object') return findings; + + const universe = collectToolUniverse(stack); + + const resolves = (ref: string): boolean => { + if (ref.endsWith('*')) { + const prefix = ref.slice(0, -1); + for (const name of universe) { + if (name.startsWith(prefix)) return true; + } + return false; + } + return universe.has(ref); + }; + + const skills = asArray(stack.skills); + for (let si = 0; si < skills.length; si++) { + const skill = skills[si]; + const skillName = strName(skill.name) ?? `#${si}`; + const refs = Array.isArray(skill.tools) ? skill.tools : []; + + for (let ti = 0; ti < refs.length; ti++) { + const ref = strName(refs[ti]); + if (!ref || resolves(ref)) continue; + + const isPattern = ref.endsWith('*'); + findings.push({ + severity: 'warning', + rule: AI_SKILL_TOOL_UNRESOLVED, + where: `skill "${skillName}" · tools`, + path: `skills[${si}].tools[${ti}]`, + message: isPattern + ? `Skill "${skillName}" subscribes to tool family "${ref}", which matches nothing this ` + + `stack can resolve (no declared tool, no platform tool, and no declarative action ` + + `materialises into it). The subscription contributes zero tools at runtime.` + : `Skill "${skillName}" references tool "${ref}", which resolves to nothing this stack ` + + `can see: not a \`stack.tools\` record, not a platform-registered tool, and not a ` + + `materialised action tool (\`action_\`). The runtime silently drops the ` + + `reference, so the skill's instructions claim a capability the agent does not have — ` + + `the assistant will improvise or fail when asked to use it.` + + suggest(ref, universe), + hint: + `Back "${ref}" with a real executable: declare a declarative action (or flow) and ` + + `reference its materialised tool (\`action_\` — the ADR-0109 default path, no ` + + `tool record needed), reference a platform tool by its registered name, or remove the ` + + `reference and the instructions that mention it. Ignore this only if a runtime plugin ` + + `outside the platform registry provides "${ref}". Family prefixes materialised by the ` + + `runtime: ${PLATFORM_TOOL_FAMILY_PREFIXES.join(', ')}.`, + }); + } + } + + return findings; +} diff --git a/packages/spec/api-surface.json b/packages/spec/api-surface.json index faa324aeb4..7f1ae98b3e 100644 --- a/packages/spec/api-surface.json +++ b/packages/spec/api-surface.json @@ -1031,6 +1031,9 @@ "PLATFORM_OBJECTS_BY_PACKAGE (const)", "PLATFORM_OBJECT_PREFIXES (const)", "PLATFORM_PROVIDED_OBJECT_NAMES (const)", + "PLATFORM_PROVIDED_TOOL_NAMES (const)", + "PLATFORM_TOOLS_BY_PACKAGE (const)", + "PLATFORM_TOOL_FAMILY_PREFIXES (const)", "PNCounter (type)", "PNCounterSchema (const)", "PackageDirectory (type)", @@ -1295,6 +1298,7 @@ "hasPlatformObjectPrefix (function)", "isDataMigrationFlagVerified (function)", "isPlatformProvidedObjectName (function)", + "isPlatformProvidedToolName (function)", "isPublicAudience (function)", "minioStorageExample (const)", "resolveActionConfirm (function)", diff --git a/packages/spec/src/stack.zod.ts b/packages/spec/src/stack.zod.ts index 75c2574d0f..2ca6442dda 100644 --- a/packages/spec/src/stack.zod.ts +++ b/packages/spec/src/stack.zod.ts @@ -306,14 +306,17 @@ export const ObjectStackDefinitionSchema = lazySchema(() => z.object({ * declares its agent surface affinity (`'ask' | 'build' | 'both'`, * ADR-0063 §3 — checked by lint, enforced at load), trigger phrases for * intent matching, and trigger conditions for context-aware activation. - * - **tools**: declaration-only metadata records today. The EXECUTABLE tool - * set is runtime-registered (kernel + plugins); a `stack.tools` entry has - * no handler binding and no runtime reader yet. The third-party tool - * authoring model is an open decision (#3820 D0) — until it lands, a - * skill's `tools[]` can only name runtime-registered tools. + * - **tools**: OPTIONAL refinement layer, never required (ADR-0109). The + * default third-party path declares no tool records: a skill's `tools[]` + * names a platform-registered tool (`PLATFORM_PROVIDED_TOOL_NAMES`) or a + * tool the runtime materialises from the app's own declarative actions + * (`action_`) — the executable, its authz and its audit stay on the + * action/flow the app already ships. A `stack.tools` record exists only + * for AI-presentation refinement (Phase 2: LLM description, parameter + * narrowing, flow exposure) and has no runtime reader until that lands. */ agents: z.array(AgentSchema).optional().describe('AI Agents — platform-internal (ADR-0063 §2): the kernel ships exactly two (ask/build); third parties extend via skills, not agents'), - tools: z.array(ToolSchema).optional().describe('AI Tool metadata records (declaration-only; the executable tool set is runtime-registered — see #3820 D0)'), + tools: z.array(ToolSchema).optional().describe('AI Tool metadata records — optional refinement layer, never required: the default path is skills referencing platform tools or materialised action_ tools (ADR-0109)'), skills: z.array(SkillSchema).optional().describe('AI Skills (reusable capability bundles — the third-party AI extension primitive, ADR-0063)'), /** @@ -1260,6 +1263,7 @@ const CONCAT_ARRAY_FIELDS = [ 'webhooks', 'agents', 'skills', + 'tools', 'hooks', 'mappings', 'analyticsCubes', diff --git a/packages/spec/src/system/constants/index.ts b/packages/spec/src/system/constants/index.ts index d48a2cbc83..bc49cbe649 100644 --- a/packages/spec/src/system/constants/index.ts +++ b/packages/spec/src/system/constants/index.ts @@ -13,3 +13,4 @@ export * from './paths'; export * from './system-names'; export * from './platform-object-names'; +export * from './platform-tool-names'; diff --git a/packages/spec/src/system/constants/platform-tool-names.test.ts b/packages/spec/src/system/constants/platform-tool-names.test.ts new file mode 100644 index 0000000000..2aef6e11f9 --- /dev/null +++ b/packages/spec/src/system/constants/platform-tool-names.test.ts @@ -0,0 +1,57 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// Shape guarantees for PLATFORM_PROVIDED_TOOL_NAMES (issue #3820, ADR-0109). +// +// Unlike platform-object-names, the OWNING packages live in the separate +// `cloud` repository (`service-ai` / `service-ai-studio`), so the "does the +// list match what is actually registered" conformance half of the contract is +// tested THERE — same split as CLOUD_PROVIDED_OBJECT_NAMES. What this repo can +// and does pin: the list's internal invariants, so a bad entry cannot poison +// every consumer silently. + +import { describe, it, expect } from 'vitest'; +import { + PLATFORM_TOOLS_BY_PACKAGE, + PLATFORM_PROVIDED_TOOL_NAMES, + PLATFORM_TOOL_FAMILY_PREFIXES, + isPlatformProvidedToolName, +} from './platform-tool-names'; + +describe('PLATFORM_PROVIDED_TOOL_NAMES — internal invariants', () => { + it('every name is snake_case (ToolSchema name grammar)', () => { + for (const name of PLATFORM_PROVIDED_TOOL_NAMES) { + expect(name).toMatch(/^[a-z_][a-z0-9_]*$/); + } + }); + + it('no name collides across owning packages', () => { + const all = Object.values(PLATFORM_TOOLS_BY_PACKAGE).flat(); + expect(new Set(all).size).toBe(all.length); + }); + + it('no static name sits inside a materialised family namespace', () => { + // A statically-listed `action_x` would shadow the dynamic family and make + // the resolution ladder ambiguous — families resolve against the stack's + // own actions, never this registry. + for (const name of PLATFORM_PROVIDED_TOOL_NAMES) { + for (const prefix of PLATFORM_TOOL_FAMILY_PREFIXES) { + expect(name.startsWith(prefix)).toBe(false); + } + } + }); + + it('the membership helper answers from the flattened set', () => { + expect(isPlatformProvidedToolName('describe_object')).toBe(true); + expect(isPlatformProvidedToolName('query_records')).toBe(true); + // The HotCRM audit's fictional reference — the case the registry exists + // to catch. + expect(isPlatformProvidedToolName('forecast_revenue')).toBe(false); + expect(isPlatformProvidedToolName('action_create_case')).toBe(false); + }); + + it('each group is sorted, so diffs stay reviewable', () => { + for (const [pkg, names] of Object.entries(PLATFORM_TOOLS_BY_PACKAGE)) { + expect([...names].sort(), pkg).toEqual([...names]); + } + }); +}); diff --git a/packages/spec/src/system/constants/platform-tool-names.ts b/packages/spec/src/system/constants/platform-tool-names.ts new file mode 100644 index 0000000000..aeaa3a67a4 --- /dev/null +++ b/packages/spec/src/system/constants/platform-tool-names.ts @@ -0,0 +1,116 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * Canonical registry of PLATFORM-PROVIDED AI tool names (issue #3820, ADR-0109). + * + * A skill legitimately references tools its stack does not declare: the + * executable tool set is RUNTIME-REGISTERED by the cloud AI runtime + * (`../cloud/service-ai` for the `ask` data product, `service-ai-studio` for + * the `build` authoring product), not read from `stack.tools`. Until this + * module existed, no static check could tell `describe_object` (real, + * registered at boot) from `forecast_revenue` (fictional — the HotCRM audit's + * "11 tools with no definitions", shipping through `validate`/`lint` clean). + * + * Structure mirrors {@link PLATFORM_PROVIDED_OBJECT_NAMES} exactly: the + * contract package owns the curated list; the OWNING packages carry + * conformance tests asserting the list matches what they actually register. + * The tools live in the separate `cloud` repository, so — like + * `CLOUD_PROVIDED_OBJECT_NAMES` — the conformance half of the contract lives + * there (`service-ai` / `service-ai-studio` test suites), not here. + * + * ADR-0109's authoring model makes this list the load-bearing half of + * reference integrity for `skill.tools[]`: the DEFAULT third-party path + * declares no tool records at all — a skill names either a platform tool + * (this registry) or an auto-materialised action tool (the + * {@link PLATFORM_TOOL_FAMILY_PREFIXES} families, e.g. `action_` from + * the stack's own declarative actions). + * + * Maintenance contract: registering a new tool in a cloud AI package means + * adding its name here. The owning package's conformance test fails + * otherwise — an out-of-date registry is worse than no registry, because + * consumers now trust it. + */ + +/** + * Statically-named tools registered by the cloud AI runtime, grouped by + * owning package. Each group is the exact set that package registers. + */ +export const PLATFORM_TOOLS_BY_PACKAGE: Readonly> = { + /** + * `@objectstack/service-ai` (cloud) — the `ask` data product's shared + * data/knowledge/visualization tools. + */ + 'service-ai': [ + 'aggregate_data', + 'get_record', + 'query_data', + 'query_records', + 'search_knowledge', + 'visualize_data', + ], + /** + * `@objectstack/service-ai-studio` (cloud) — the shared schema-reader pair + * (`describe_object`/`list_objects`) plus the `build` authoring product's + * metadata/blueprint/package/verify tools. + */ + 'service-ai-studio': [ + 'add_field', + 'apply_blueprint', + 'apply_edit', + 'create_metadata', + 'create_object', + 'create_package', + 'create_seed', + 'delete_field', + 'describe_metadata', + 'describe_object', + 'get_active_package', + 'get_metadata_schema', + 'get_package', + 'list_metadata', + 'list_objects', + 'list_packages', + 'modify_field', + 'propose_blueprint', + 'set_active_package', + 'suggest_builder', + 'todo_write', + 'update_metadata', + 'validate_expression', + 'verify_build', + ], +} as const; + +/** + * Every statically-named platform tool, for fast membership checks in lint + * rules and cross-reference validation. + */ +export const PLATFORM_PROVIDED_TOOL_NAMES: ReadonlySet = new Set( + Object.values(PLATFORM_TOOLS_BY_PACKAGE).flat(), +); + +/** + * Name prefixes of tool FAMILIES the runtime materialises dynamically — names + * that are real without being statically enumerable. Today that is exactly + * one family: `action_`, one tool per declarative Action + * (`service-ai`'s action-tools generator; the built-in `actions_executor` + * skill subscribes via the `action_*` wildcard). + * + * A `skill.tools[]` entry under one of these prefixes resolves against the + * stack's own actions (`action_` + the action's name), not this registry. + */ +export const PLATFORM_TOOL_FAMILY_PREFIXES: readonly string[] = ['action_']; + +/** + * Is `name` a KNOWN platform-provided tool (statically registered by a cloud + * AI package)? + * + * `false` does NOT mean fictional on its own: the name may be a materialised + * family member (check {@link PLATFORM_TOOL_FAMILY_PREFIXES} against the + * stack's actions) or contributed by a third-party runtime plugin the + * registry cannot know about — the reason the R7 lint rule reports misses at + * ADVISORY severity (ADR-0078 ratchet). + */ +export function isPlatformProvidedToolName(name: string): boolean { + return PLATFORM_PROVIDED_TOOL_NAMES.has(name); +}